Loading src/common/crud_table.py +92 −60 Original line number Diff line number Diff line Loading @@ -13,6 +13,7 @@ basic crud operations to database tables. """ from abc import ABC from common.logz import create_logger Loading Loading @@ -73,19 +74,23 @@ class CRUDTable(ABC): :param kwargs: column_name=value for every column """ # assure that all columns are defined assert all(arg in self.columns.keys() for arg in kwargs.keys()),\ ("Must supply values for all columns to create an entry. " + f"Columns: {self.columns}") # FIXME - asserts should only be used inside tests assert all(arg in self.columns.keys() for arg in kwargs.keys()), ( "Must supply values for all columns to create an entry. " + f"Columns: {self.columns}" ) # get a 'pretty' string of column names column_names = str(list(kwargs.keys()))[1:-1].\ replace('\'', '') column_names = str(list(kwargs.keys()))[1:-1].replace("'", "") # build the query with the class's name and the kwargs that were passed query = (f"INSERT INTO {self.schema}.{self.__class__.__name__} " + f"({column_names}) " + f"VALUES ({', '.join(['%s']*len(kwargs.keys()))})") query = ( f"INSERT INTO {self.schema}.{self.__class__.__name__} " f"({column_names}) " f"VALUES ({', '.join(['%s']*len(kwargs.keys()))})" ) # tell the user that we are executing an insert query self.logger.info(f"Executing query: {query} " + f"params: {tuple(kwargs.values())}") self.logger.info( f"Executing query: {query} " + f"params: {tuple(kwargs.values())}" ) try: # if the db is not open.. if not self.db.is_open(): Loading @@ -99,9 +104,11 @@ class CRUDTable(ABC): self.db.connection.commit() except Exception as err: # close the connection to the database self.logger.error("Exception occured when trying to execute " + f"query: {query} with " + f"parameters: {tuple(kwargs.values())}") self.logger.error( "Exception occured when trying to execute " + f"query: {query} with " + f"parameters: {tuple(kwargs.values())}" ) self.logger.error(f"Exception Message: {err}") finally: self.db.close() Loading Loading @@ -130,9 +137,10 @@ class CRUDTable(ABC): if len(kwargs) > 0: # assure that every column specified in kwargs are define in # self.columns assert all(column in self.columns for column in kwargs), \ 'Column(s) specified in kwargs could not be found. ' +\ 'Please check kwargs definition and try again.' assert all(column in self.columns for column in kwargs), ( "Column(s) specified in kwargs could not be found. " + "Please check kwargs definition and try again." ) # construct and assign the where clause with the # conversion function where_clause = convert_to_where(kwargs) Loading @@ -142,29 +150,36 @@ class CRUDTable(ABC): if isinstance(columns, list): # assure that every specfied column in columns are defined in # self.columns assert all(column in self.columns for column in columns), \ 'Column(s) specified in columns could not be found. ' +\ 'Please check columns definition and try again.' assert all(column in self.columns for column in columns), ( "Column(s) specified in columns could not be found. " + "Please check columns definition and try again." ) # construct a 'select' clause from the list select_clause = str(columns)[1:-1].replace("'", "") # is columns is a string... elif isinstance(columns, str): # assure that the column specified is defined in self.columns assert columns in self.columns, 'Could not find column ' +\ f'{columns} in self.columns definition: {self.columns}' assert columns in self.columns, ( "Could not find column " + f"{columns} in self.columns definition: {self.columns}" ) # use the column specified as the select clause select_clause = columns # if columns is of any other type... else: # raise an exception as we do not know what to do raise TypeError("column argument should be of type list or" + f" str not {type(columns)}") raise TypeError( "column argument should be of type list or" + f" str not {type(columns)}" ) # if the where clause was not set/specified if where_clause is None: try: # make a query to select the columns with no where clause query = (f"SELECT {select_clause} " + f"FROM {self.schema}.{self.__class__.__name__}") query = ( f"SELECT {select_clause} " + f"FROM {self.schema}.{self.__class__.__name__}" ) # inform the user we are executing the query.. self.logger.info(f"Executing query: {query}") # if the db is not already opened.. Loading @@ -176,20 +191,24 @@ class CRUDTable(ABC): # execute the query curr.execute(query) except Exception as err: self.logger.error("Exception occured when trying to execute " + f"query: {query}") self.logger.error( "Exception occured when trying to execute " + f"query: {query}" ) self.logger.error(f"Exception Message: {err}") # if there is a where clause... else: try: # make a query to select the columns with the where clause query = (f"SELECT {select_clause} " + f"FROM {self.schema}.{self.__class__.__name__} " + f"{where_clause[0]}") query = ( f"SELECT {select_clause} " + f"FROM {self.schema}.{self.__class__.__name__} " + f"{where_clause[0]}" ) # tell the user we are executing their query self.logger.info(f"Executing query: {query} " + f"params: {where_clause[1]}") self.logger.info( f"Executing query: {query} " + f"params: {where_clause[1]}" ) # if the db is not already opened.. if not self.db.is_open(): # open the connection to the database Loading @@ -199,9 +218,11 @@ class CRUDTable(ABC): # execute the query with the where clause params curr.execute(query, where_clause[1]) except Exception as err: self.logger.error("Exception occured when trying to execute " + f"query: {query} with " + f"parameters: {where_clause[1]}") self.logger.error( "Exception occured when trying to execute " + f"query: {query} with " + f"parameters: {where_clause[1]}" ) self.logger.error(f"Exception Message: {err}") # close the connection to the database self.db.close() Loading @@ -216,9 +237,11 @@ class CRUDTable(ABC): # simply fetch the results from the db data = curr.fetchall() except Exception as err: self.logger.error("Exception occured when trying to fetch " + f"results from query: {query} with " + f"parameters: {where_clause[1]}") self.logger.error( "Exception occured when trying to fetch " + f"results from query: {query} with " + f"parameters: {where_clause[1]}" ) self.logger.error(f"Exception Message: {err}") # return the results from the database finally: Loading @@ -235,13 +258,13 @@ class CRUDTable(ABC): :param kwargs: keys and values to update in the database """ # ensure there is a where clause assert where is not None and len(where) > 0,\ "No where clause found." +\ "\nUpdate must have a where clause!" assert where is not None and len(where) > 0, ( "No where clause found." + "\nUpdate must have a where clause!" ) # ensure that some update was specified in the kwargs assert kwargs is not None and len(kwargs) > 0,\ "No keyword arguments supplied." +\ "\nUpdate must have a field to update!" assert kwargs is not None and len(kwargs) > 0, ( "No keyword arguments supplied." + "\nUpdate must have a field to update!" ) try: # construct the where clause with the conversion method Loading @@ -251,9 +274,11 @@ class CRUDTable(ABC): # construct the parms for the update statement params = tuple([*update_clause[1], *where_clause[1]]) # build a query to update the specified values in the db query = (f"UPDATE {self.schema}.{self.__class__.__name__} " + f"SET {update_clause[0]} " + f"{where_clause[0]}") query = ( f"UPDATE {self.schema}.{self.__class__.__name__} " + f"SET {update_clause[0]} " + f"{where_clause[0]}" ) # tell the user we are executing their query self.logger.info(f"Executing query: {query} params: {params}") # if the db is not open.. Loading @@ -267,9 +292,11 @@ class CRUDTable(ABC): # commit the changes to the database self.db.connection.commit() except Exception as err: self.logger.error("Exception occured when trying to execute " + f"query: {query} with " + f"parameters: {params}") self.logger.error( "Exception occured when trying to execute " + f"query: {query} with " + f"parameters: {params}" ) self.logger.error(f"Exception Message: {err}") finally: # close the connection to the database Loading @@ -281,18 +308,21 @@ class CRUDTable(ABC): :param kwargs: where clause to delete on """ # ensure that some kwargs were passed assert kwargs is not None and len(kwargs) > 0,\ "No keyword arguments supplied." +\ "\nDelete must have a where clause!" assert kwargs is not None and len(kwargs) > 0, ( "No keyword arguments supplied." + "\nDelete must have a where clause!" ) try: # construct the where clause with the conversion method where_clause = convert_to_where(kwargs) # build a delete query with the specified values query = (f"DELETE FROM {self.schema}.{self.__class__.__name__} " + f"{where_clause[0]}") query = ( f"DELETE FROM {self.schema}.{self.__class__.__name__} " + f"{where_clause[0]}" ) # tell the user that we are executing their query self.logger.info(f"Executing query: {query}, " + f"params: {where_clause[1]}") self.logger.info( f"Executing query: {query}, " + f"params: {where_clause[1]}" ) # if the db is not open.. if not self.db.is_open(): # open a connection to the database Loading @@ -304,9 +334,11 @@ class CRUDTable(ABC): # commit the changes to the database self.db.connection.commit() except Exception as err: self.logger.error("Exception occured when trying to execute " + f"query: {query} with " + f"parameters: {where_clause[1]}") self.logger.error( "Exception occured when trying to execute " + f"query: {query} with " + f"parameters: {where_clause[1]}" ) self.logger.error(f"Exception Message: {err}") finally: # close the connection Loading src/common/database.py +103 −92 Original line number Diff line number Diff line Loading @@ -7,8 +7,9 @@ any type of connection through implementation of this abstract class. """ import traceback from abc import ABC from common.logz import create_logger from common.env import check_environment as ce from common.logz import create_logger class Database(ABC): Loading Loading @@ -50,50 +51,60 @@ class Database(ABC): :note: File logging is disabled by default. """ DEFAULT_DB = ce('DATABASE_DB', ce('PG_DATABASE', 'postgres')) DEFAULT_USER = ce('DATABASE_USER', ce('PG_USER', 'postgres')) DEFAULT_PW = ce('DATABASE_PW', ce('PG_PASSWORD', 'postgres')) DEFAULT_HOST = ce('DATABASE_HOST', ce('PG_HOST', "localhost")) DEFAULT_PORT = int(ce('DATABASE_PORT', ce('PG_PORT', 5432))) DEFAULT_SCHEMA = ce('DATABASE_SCHEMA', ce('PG_SCHEMA', 'public')) DEFAULT_ENGINE = ce('DATABASE_ENGINE', 'postgresql') DEFAULT_TIMEOUT = ce('DATABASE_TIMEOUT', ce('PG_TIMEOUT', 60)) DEFAULT_LOG_PATH = ce('DATABASE_LOG_FILE', '') DEFAULT_LOG_MODE = ce('DATABASE_LOG_MODE', 'a') DEFAULT_LOG_ENCODING = ce('DATABASE_LOG_ENCODING', 'utf-8') DEFAULT_DB = ce("DATABASE_DB", ce("PG_DATABASE", "postgres")) DEFAULT_USER = ce("DATABASE_USER", ce("PG_USER", "postgres")) DEFAULT_PW = ce("DATABASE_PW", ce("PG_PASSWORD", "postgres")) DEFAULT_HOST = ce("DATABASE_HOST", ce("PG_HOST", "localhost")) DEFAULT_PORT = int(ce("DATABASE_PORT", ce("PG_PORT", 5432))) DEFAULT_SCHEMA = ce("DATABASE_SCHEMA", ce("PG_SCHEMA", "public")) DEFAULT_ENGINE = ce("DATABASE_ENGINE", "postgresql") DEFAULT_TIMEOUT = ce("DATABASE_TIMEOUT", ce("PG_TIMEOUT", 60)) DEFAULT_LOG_PATH = ce("DATABASE_LOG_FILE", "") DEFAULT_LOG_MODE = ce("DATABASE_LOG_MODE", "a") DEFAULT_LOG_ENCODING = ce("DATABASE_LOG_ENCODING", "utf-8") # define a URI string if URI is perferred to connect DEFAULT_URI = (DEFAULT_ENGINE + '://' + DEFAULT_USER + ':' + str(DEFAULT_PW) + '@' + DEFAULT_HOST + '/' + DEFAULT_DB) def __init__(self, log_file_info={'path': DEFAULT_LOG_PATH, 'mode': DEFAULT_LOG_MODE, 'encoding': DEFAULT_LOG_ENCODING}, connection_info={'dbName': DEFAULT_DB, 'dbUser': DEFAULT_USER, 'dbPassword': DEFAULT_PW, 'dbSchema': DEFAULT_SCHEMA, 'dbHost': DEFAULT_HOST, 'dbPort': DEFAULT_PORT, 'dbTimeout': DEFAULT_TIMEOUT, 'dbEngine': DEFAULT_ENGINE, 'uri': DEFAULT_URI}, **kwargs): DEFAULT_URI = ( f"{DEFAULT_ENGINE}://{DEFAULT_USER}:{str(DEFAULT_PW)}" + f"@{DEFAULT_HOST}/{DEFAULT_DB}" ) def __init__( self, log_file_info={ "path": DEFAULT_LOG_PATH, "mode": DEFAULT_LOG_MODE, "encoding": DEFAULT_LOG_ENCODING, }, connection_info={ "dbName": DEFAULT_DB, "dbUser": DEFAULT_USER, "dbPassword": DEFAULT_PW, "dbSchema": DEFAULT_SCHEMA, "dbHost": DEFAULT_HOST, "dbPort": DEFAULT_PORT, "dbTimeout": DEFAULT_TIMEOUT, "dbEngine": DEFAULT_ENGINE, "uri": DEFAULT_URI, }, **kwargs, ): """Create a Database object. This *does not* open a connection to the database. Use open() or `with` to establish a database connection. :param connection_info: a dictionary containing connection info """ if log_file_info['path'] == '': if log_file_info["path"] == "": self.logger = create_logger() else: self.logger = create_logger(log_file_info['path'], log_file_info['mode'], log_file_info['encoding']) self.logger = create_logger( log_file_info["path"], log_file_info["mode"], log_file_info["encoding"] ) self.connection_info = connection_info self.logger.debug(connection_info) self.connection = None self.cursor = None self.logger.debug('Database object successfully initialized') self.logger.debug("Database object successfully initialized") def __del__(self): """Make any pending database commits and close the connection Loading @@ -104,7 +115,7 @@ class Database(ABC): execution scope would sever the last reference to a Database object, nor even when the script finishes execution. """ self.logger.debug('Deleting Database Object') self.logger.debug("Deleting Database Object") self.close() def __enter__(self): Loading @@ -122,14 +133,14 @@ class Database(ABC): state = self.open() except BaseException as e: traceback.print_stack() self.logger.error(f'Error {e} occurred while entering Database') self.logger.error(f"Error {e} occurred while entering Database") state = False if state is True: return if state is False: traceback.print_stack() self.logger.critical('Not possible to enter Database') raise BaseException('Not possible to enter Database') self.logger.critical("Not possible to enter Database") raise BaseException("Not possible to enter Database") def close(self): """Explicitly close the connection Loading src/common/env.py +12 −12 Original line number Diff line number Diff line Loading @@ -9,11 +9,11 @@ def boolify(var): :param var: the variable to check to see if it can be converted to bool """ if var in [0, '0', 'FALSE', 'False', 'false', False]: if var in [0, "0", "FALSE", "False", "false", False]: return False if var in [1, '1', 'TRUE', 'True', 'true', True]: if var in [1, "1", "TRUE", "True", "true", True]: return True raise TypeError('unable to evaluate expected boolean') raise TypeError("unable to evaluate expected boolean") def check_environment(env_var, default=None): Loading src/common/file_operations.py +22 −12 Original line number Diff line number Diff line Loading @@ -12,6 +12,7 @@ from pathlib import Path # for Json files - use json module # append a key/dict/etc to the JSON would be nice def path_exists(dirc_or_file): """ # function to check if directory/file exists Loading @@ -20,13 +21,14 @@ def path_exists(dirc_or_file): """ return Path(dirc_or_file).exists() def create_directory(dirc=None): """ # function to create directories at the specified path # @params : directory to create # @returns : boolean / None """ if dirc != None: if dirc is not None: try: if path_exists(dirc): return f"{dirc} already exists." Loading @@ -37,13 +39,14 @@ def create_directory(dirc=None): print(f"Error: {e}") return None def delete_directory(dirc=None): """ # function to delete directories at the specified path # @params : directory to delete # @returns : str / None """ if dirc != None: if dirc is not None: try: if path_exists(dirc): Path(dirc).rmdir() Loading @@ -54,24 +57,28 @@ def delete_directory(dirc=None): print(f"Error: {e}") return None def tree(dirc=None): """ # function to walk a directory # @params : directory to walk # @returns : None """ if dirc != None: if dirc is not None: try: if path_exists(dirc): for item in Path(dirc).glob("*"): if item.is_file(): print(f"File: {item}") elif item.is_dir(): print(f"Directory: {item}") if item.is_file(): print(f"File: {item}") elif item.is_dir(): print(f"Directory: {item}") else: return f"{dirc} does not exist." except Exception as e: print(f"Error: {e}") return None def read_file(file_name): """ # function to read a file Loading @@ -91,6 +98,7 @@ def read_file(file_name): print(f"Invalid JSON file: {e}") return None def append_file(file_name, file_data): """ # function to append contents to a file Loading @@ -108,6 +116,7 @@ def append_file(file_name, file_data): print(f"Error: {te}, please provide data as string") return None def write_file(file_name, file_data=""): """ # function to write to a file Loading @@ -130,6 +139,7 @@ def write_file(file_name, file_data=""): print(f"Error: {te}, please provide data as string") return None def delete_file(file_name): """ # function to delete a file Loading src/common/logz.py +10 −9 Original line number Diff line number Diff line #!/usr/bin/env python3 # -*- coding: utf-8 -*- """Provide a standard logger for all use cases.""" import logging # standard python imports import os import logging from rich.logging import RichHandler from rich.traceback import install Loading @@ -28,24 +29,24 @@ def create_logger(file_out=None, mode=None, encoding=None): logging.root.removeHandler(handler) install() log_level = os.environ.get('LOGLEVEL', 'INFO').upper() log_level = os.environ.get("LOGLEVEL", "INFO").upper() rich_handler = RichHandler(rich_tracebacks=True, markup=True) if file_out is not None: file_handler = logging.FileHandler(file_out, mode, encoding) file_handler_fmt = logging.Formatter('[%(asctime)s]' + '%(levelname)8s - ' + ' - %(message)s') file_handler_fmt = logging.Formatter( "[%(asctime)s]" + "%(levelname)8s - " + " - %(message)s" ) file_handler.setFormatter(file_handler_fmt) handlers = [rich_handler, file_handler] else: file_handler = None handlers = [rich_handler] logging.basicConfig(level=log_level, format='%(message)s', datefmt="[%Y/%m/%d %H:%M;%S]", handlers=handlers) logging.basicConfig( level=log_level, format="%(message)s", datefmt="[%Y/%m/%d %H:%M;%S]", handlers=handlers ) # if there is a file_handler set, close it before leaving :) # This prevents leaving open files if file_handler is not None: file_handler.close() rich_handler.close() return logging.getLogger('rich') return logging.getLogger("rich") Loading
src/common/crud_table.py +92 −60 Original line number Diff line number Diff line Loading @@ -13,6 +13,7 @@ basic crud operations to database tables. """ from abc import ABC from common.logz import create_logger Loading Loading @@ -73,19 +74,23 @@ class CRUDTable(ABC): :param kwargs: column_name=value for every column """ # assure that all columns are defined assert all(arg in self.columns.keys() for arg in kwargs.keys()),\ ("Must supply values for all columns to create an entry. " + f"Columns: {self.columns}") # FIXME - asserts should only be used inside tests assert all(arg in self.columns.keys() for arg in kwargs.keys()), ( "Must supply values for all columns to create an entry. " + f"Columns: {self.columns}" ) # get a 'pretty' string of column names column_names = str(list(kwargs.keys()))[1:-1].\ replace('\'', '') column_names = str(list(kwargs.keys()))[1:-1].replace("'", "") # build the query with the class's name and the kwargs that were passed query = (f"INSERT INTO {self.schema}.{self.__class__.__name__} " + f"({column_names}) " + f"VALUES ({', '.join(['%s']*len(kwargs.keys()))})") query = ( f"INSERT INTO {self.schema}.{self.__class__.__name__} " f"({column_names}) " f"VALUES ({', '.join(['%s']*len(kwargs.keys()))})" ) # tell the user that we are executing an insert query self.logger.info(f"Executing query: {query} " + f"params: {tuple(kwargs.values())}") self.logger.info( f"Executing query: {query} " + f"params: {tuple(kwargs.values())}" ) try: # if the db is not open.. if not self.db.is_open(): Loading @@ -99,9 +104,11 @@ class CRUDTable(ABC): self.db.connection.commit() except Exception as err: # close the connection to the database self.logger.error("Exception occured when trying to execute " + f"query: {query} with " + f"parameters: {tuple(kwargs.values())}") self.logger.error( "Exception occured when trying to execute " + f"query: {query} with " + f"parameters: {tuple(kwargs.values())}" ) self.logger.error(f"Exception Message: {err}") finally: self.db.close() Loading Loading @@ -130,9 +137,10 @@ class CRUDTable(ABC): if len(kwargs) > 0: # assure that every column specified in kwargs are define in # self.columns assert all(column in self.columns for column in kwargs), \ 'Column(s) specified in kwargs could not be found. ' +\ 'Please check kwargs definition and try again.' assert all(column in self.columns for column in kwargs), ( "Column(s) specified in kwargs could not be found. " + "Please check kwargs definition and try again." ) # construct and assign the where clause with the # conversion function where_clause = convert_to_where(kwargs) Loading @@ -142,29 +150,36 @@ class CRUDTable(ABC): if isinstance(columns, list): # assure that every specfied column in columns are defined in # self.columns assert all(column in self.columns for column in columns), \ 'Column(s) specified in columns could not be found. ' +\ 'Please check columns definition and try again.' assert all(column in self.columns for column in columns), ( "Column(s) specified in columns could not be found. " + "Please check columns definition and try again." ) # construct a 'select' clause from the list select_clause = str(columns)[1:-1].replace("'", "") # is columns is a string... elif isinstance(columns, str): # assure that the column specified is defined in self.columns assert columns in self.columns, 'Could not find column ' +\ f'{columns} in self.columns definition: {self.columns}' assert columns in self.columns, ( "Could not find column " + f"{columns} in self.columns definition: {self.columns}" ) # use the column specified as the select clause select_clause = columns # if columns is of any other type... else: # raise an exception as we do not know what to do raise TypeError("column argument should be of type list or" + f" str not {type(columns)}") raise TypeError( "column argument should be of type list or" + f" str not {type(columns)}" ) # if the where clause was not set/specified if where_clause is None: try: # make a query to select the columns with no where clause query = (f"SELECT {select_clause} " + f"FROM {self.schema}.{self.__class__.__name__}") query = ( f"SELECT {select_clause} " + f"FROM {self.schema}.{self.__class__.__name__}" ) # inform the user we are executing the query.. self.logger.info(f"Executing query: {query}") # if the db is not already opened.. Loading @@ -176,20 +191,24 @@ class CRUDTable(ABC): # execute the query curr.execute(query) except Exception as err: self.logger.error("Exception occured when trying to execute " + f"query: {query}") self.logger.error( "Exception occured when trying to execute " + f"query: {query}" ) self.logger.error(f"Exception Message: {err}") # if there is a where clause... else: try: # make a query to select the columns with the where clause query = (f"SELECT {select_clause} " + f"FROM {self.schema}.{self.__class__.__name__} " + f"{where_clause[0]}") query = ( f"SELECT {select_clause} " + f"FROM {self.schema}.{self.__class__.__name__} " + f"{where_clause[0]}" ) # tell the user we are executing their query self.logger.info(f"Executing query: {query} " + f"params: {where_clause[1]}") self.logger.info( f"Executing query: {query} " + f"params: {where_clause[1]}" ) # if the db is not already opened.. if not self.db.is_open(): # open the connection to the database Loading @@ -199,9 +218,11 @@ class CRUDTable(ABC): # execute the query with the where clause params curr.execute(query, where_clause[1]) except Exception as err: self.logger.error("Exception occured when trying to execute " + f"query: {query} with " + f"parameters: {where_clause[1]}") self.logger.error( "Exception occured when trying to execute " + f"query: {query} with " + f"parameters: {where_clause[1]}" ) self.logger.error(f"Exception Message: {err}") # close the connection to the database self.db.close() Loading @@ -216,9 +237,11 @@ class CRUDTable(ABC): # simply fetch the results from the db data = curr.fetchall() except Exception as err: self.logger.error("Exception occured when trying to fetch " + f"results from query: {query} with " + f"parameters: {where_clause[1]}") self.logger.error( "Exception occured when trying to fetch " + f"results from query: {query} with " + f"parameters: {where_clause[1]}" ) self.logger.error(f"Exception Message: {err}") # return the results from the database finally: Loading @@ -235,13 +258,13 @@ class CRUDTable(ABC): :param kwargs: keys and values to update in the database """ # ensure there is a where clause assert where is not None and len(where) > 0,\ "No where clause found." +\ "\nUpdate must have a where clause!" assert where is not None and len(where) > 0, ( "No where clause found." + "\nUpdate must have a where clause!" ) # ensure that some update was specified in the kwargs assert kwargs is not None and len(kwargs) > 0,\ "No keyword arguments supplied." +\ "\nUpdate must have a field to update!" assert kwargs is not None and len(kwargs) > 0, ( "No keyword arguments supplied." + "\nUpdate must have a field to update!" ) try: # construct the where clause with the conversion method Loading @@ -251,9 +274,11 @@ class CRUDTable(ABC): # construct the parms for the update statement params = tuple([*update_clause[1], *where_clause[1]]) # build a query to update the specified values in the db query = (f"UPDATE {self.schema}.{self.__class__.__name__} " + f"SET {update_clause[0]} " + f"{where_clause[0]}") query = ( f"UPDATE {self.schema}.{self.__class__.__name__} " + f"SET {update_clause[0]} " + f"{where_clause[0]}" ) # tell the user we are executing their query self.logger.info(f"Executing query: {query} params: {params}") # if the db is not open.. Loading @@ -267,9 +292,11 @@ class CRUDTable(ABC): # commit the changes to the database self.db.connection.commit() except Exception as err: self.logger.error("Exception occured when trying to execute " + f"query: {query} with " + f"parameters: {params}") self.logger.error( "Exception occured when trying to execute " + f"query: {query} with " + f"parameters: {params}" ) self.logger.error(f"Exception Message: {err}") finally: # close the connection to the database Loading @@ -281,18 +308,21 @@ class CRUDTable(ABC): :param kwargs: where clause to delete on """ # ensure that some kwargs were passed assert kwargs is not None and len(kwargs) > 0,\ "No keyword arguments supplied." +\ "\nDelete must have a where clause!" assert kwargs is not None and len(kwargs) > 0, ( "No keyword arguments supplied." + "\nDelete must have a where clause!" ) try: # construct the where clause with the conversion method where_clause = convert_to_where(kwargs) # build a delete query with the specified values query = (f"DELETE FROM {self.schema}.{self.__class__.__name__} " + f"{where_clause[0]}") query = ( f"DELETE FROM {self.schema}.{self.__class__.__name__} " + f"{where_clause[0]}" ) # tell the user that we are executing their query self.logger.info(f"Executing query: {query}, " + f"params: {where_clause[1]}") self.logger.info( f"Executing query: {query}, " + f"params: {where_clause[1]}" ) # if the db is not open.. if not self.db.is_open(): # open a connection to the database Loading @@ -304,9 +334,11 @@ class CRUDTable(ABC): # commit the changes to the database self.db.connection.commit() except Exception as err: self.logger.error("Exception occured when trying to execute " + f"query: {query} with " + f"parameters: {where_clause[1]}") self.logger.error( "Exception occured when trying to execute " + f"query: {query} with " + f"parameters: {where_clause[1]}" ) self.logger.error(f"Exception Message: {err}") finally: # close the connection Loading
src/common/database.py +103 −92 Original line number Diff line number Diff line Loading @@ -7,8 +7,9 @@ any type of connection through implementation of this abstract class. """ import traceback from abc import ABC from common.logz import create_logger from common.env import check_environment as ce from common.logz import create_logger class Database(ABC): Loading Loading @@ -50,50 +51,60 @@ class Database(ABC): :note: File logging is disabled by default. """ DEFAULT_DB = ce('DATABASE_DB', ce('PG_DATABASE', 'postgres')) DEFAULT_USER = ce('DATABASE_USER', ce('PG_USER', 'postgres')) DEFAULT_PW = ce('DATABASE_PW', ce('PG_PASSWORD', 'postgres')) DEFAULT_HOST = ce('DATABASE_HOST', ce('PG_HOST', "localhost")) DEFAULT_PORT = int(ce('DATABASE_PORT', ce('PG_PORT', 5432))) DEFAULT_SCHEMA = ce('DATABASE_SCHEMA', ce('PG_SCHEMA', 'public')) DEFAULT_ENGINE = ce('DATABASE_ENGINE', 'postgresql') DEFAULT_TIMEOUT = ce('DATABASE_TIMEOUT', ce('PG_TIMEOUT', 60)) DEFAULT_LOG_PATH = ce('DATABASE_LOG_FILE', '') DEFAULT_LOG_MODE = ce('DATABASE_LOG_MODE', 'a') DEFAULT_LOG_ENCODING = ce('DATABASE_LOG_ENCODING', 'utf-8') DEFAULT_DB = ce("DATABASE_DB", ce("PG_DATABASE", "postgres")) DEFAULT_USER = ce("DATABASE_USER", ce("PG_USER", "postgres")) DEFAULT_PW = ce("DATABASE_PW", ce("PG_PASSWORD", "postgres")) DEFAULT_HOST = ce("DATABASE_HOST", ce("PG_HOST", "localhost")) DEFAULT_PORT = int(ce("DATABASE_PORT", ce("PG_PORT", 5432))) DEFAULT_SCHEMA = ce("DATABASE_SCHEMA", ce("PG_SCHEMA", "public")) DEFAULT_ENGINE = ce("DATABASE_ENGINE", "postgresql") DEFAULT_TIMEOUT = ce("DATABASE_TIMEOUT", ce("PG_TIMEOUT", 60)) DEFAULT_LOG_PATH = ce("DATABASE_LOG_FILE", "") DEFAULT_LOG_MODE = ce("DATABASE_LOG_MODE", "a") DEFAULT_LOG_ENCODING = ce("DATABASE_LOG_ENCODING", "utf-8") # define a URI string if URI is perferred to connect DEFAULT_URI = (DEFAULT_ENGINE + '://' + DEFAULT_USER + ':' + str(DEFAULT_PW) + '@' + DEFAULT_HOST + '/' + DEFAULT_DB) def __init__(self, log_file_info={'path': DEFAULT_LOG_PATH, 'mode': DEFAULT_LOG_MODE, 'encoding': DEFAULT_LOG_ENCODING}, connection_info={'dbName': DEFAULT_DB, 'dbUser': DEFAULT_USER, 'dbPassword': DEFAULT_PW, 'dbSchema': DEFAULT_SCHEMA, 'dbHost': DEFAULT_HOST, 'dbPort': DEFAULT_PORT, 'dbTimeout': DEFAULT_TIMEOUT, 'dbEngine': DEFAULT_ENGINE, 'uri': DEFAULT_URI}, **kwargs): DEFAULT_URI = ( f"{DEFAULT_ENGINE}://{DEFAULT_USER}:{str(DEFAULT_PW)}" + f"@{DEFAULT_HOST}/{DEFAULT_DB}" ) def __init__( self, log_file_info={ "path": DEFAULT_LOG_PATH, "mode": DEFAULT_LOG_MODE, "encoding": DEFAULT_LOG_ENCODING, }, connection_info={ "dbName": DEFAULT_DB, "dbUser": DEFAULT_USER, "dbPassword": DEFAULT_PW, "dbSchema": DEFAULT_SCHEMA, "dbHost": DEFAULT_HOST, "dbPort": DEFAULT_PORT, "dbTimeout": DEFAULT_TIMEOUT, "dbEngine": DEFAULT_ENGINE, "uri": DEFAULT_URI, }, **kwargs, ): """Create a Database object. This *does not* open a connection to the database. Use open() or `with` to establish a database connection. :param connection_info: a dictionary containing connection info """ if log_file_info['path'] == '': if log_file_info["path"] == "": self.logger = create_logger() else: self.logger = create_logger(log_file_info['path'], log_file_info['mode'], log_file_info['encoding']) self.logger = create_logger( log_file_info["path"], log_file_info["mode"], log_file_info["encoding"] ) self.connection_info = connection_info self.logger.debug(connection_info) self.connection = None self.cursor = None self.logger.debug('Database object successfully initialized') self.logger.debug("Database object successfully initialized") def __del__(self): """Make any pending database commits and close the connection Loading @@ -104,7 +115,7 @@ class Database(ABC): execution scope would sever the last reference to a Database object, nor even when the script finishes execution. """ self.logger.debug('Deleting Database Object') self.logger.debug("Deleting Database Object") self.close() def __enter__(self): Loading @@ -122,14 +133,14 @@ class Database(ABC): state = self.open() except BaseException as e: traceback.print_stack() self.logger.error(f'Error {e} occurred while entering Database') self.logger.error(f"Error {e} occurred while entering Database") state = False if state is True: return if state is False: traceback.print_stack() self.logger.critical('Not possible to enter Database') raise BaseException('Not possible to enter Database') self.logger.critical("Not possible to enter Database") raise BaseException("Not possible to enter Database") def close(self): """Explicitly close the connection Loading
src/common/env.py +12 −12 Original line number Diff line number Diff line Loading @@ -9,11 +9,11 @@ def boolify(var): :param var: the variable to check to see if it can be converted to bool """ if var in [0, '0', 'FALSE', 'False', 'false', False]: if var in [0, "0", "FALSE", "False", "false", False]: return False if var in [1, '1', 'TRUE', 'True', 'true', True]: if var in [1, "1", "TRUE", "True", "true", True]: return True raise TypeError('unable to evaluate expected boolean') raise TypeError("unable to evaluate expected boolean") def check_environment(env_var, default=None): Loading
src/common/file_operations.py +22 −12 Original line number Diff line number Diff line Loading @@ -12,6 +12,7 @@ from pathlib import Path # for Json files - use json module # append a key/dict/etc to the JSON would be nice def path_exists(dirc_or_file): """ # function to check if directory/file exists Loading @@ -20,13 +21,14 @@ def path_exists(dirc_or_file): """ return Path(dirc_or_file).exists() def create_directory(dirc=None): """ # function to create directories at the specified path # @params : directory to create # @returns : boolean / None """ if dirc != None: if dirc is not None: try: if path_exists(dirc): return f"{dirc} already exists." Loading @@ -37,13 +39,14 @@ def create_directory(dirc=None): print(f"Error: {e}") return None def delete_directory(dirc=None): """ # function to delete directories at the specified path # @params : directory to delete # @returns : str / None """ if dirc != None: if dirc is not None: try: if path_exists(dirc): Path(dirc).rmdir() Loading @@ -54,24 +57,28 @@ def delete_directory(dirc=None): print(f"Error: {e}") return None def tree(dirc=None): """ # function to walk a directory # @params : directory to walk # @returns : None """ if dirc != None: if dirc is not None: try: if path_exists(dirc): for item in Path(dirc).glob("*"): if item.is_file(): print(f"File: {item}") elif item.is_dir(): print(f"Directory: {item}") if item.is_file(): print(f"File: {item}") elif item.is_dir(): print(f"Directory: {item}") else: return f"{dirc} does not exist." except Exception as e: print(f"Error: {e}") return None def read_file(file_name): """ # function to read a file Loading @@ -91,6 +98,7 @@ def read_file(file_name): print(f"Invalid JSON file: {e}") return None def append_file(file_name, file_data): """ # function to append contents to a file Loading @@ -108,6 +116,7 @@ def append_file(file_name, file_data): print(f"Error: {te}, please provide data as string") return None def write_file(file_name, file_data=""): """ # function to write to a file Loading @@ -130,6 +139,7 @@ def write_file(file_name, file_data=""): print(f"Error: {te}, please provide data as string") return None def delete_file(file_name): """ # function to delete a file Loading
src/common/logz.py +10 −9 Original line number Diff line number Diff line #!/usr/bin/env python3 # -*- coding: utf-8 -*- """Provide a standard logger for all use cases.""" import logging # standard python imports import os import logging from rich.logging import RichHandler from rich.traceback import install Loading @@ -28,24 +29,24 @@ def create_logger(file_out=None, mode=None, encoding=None): logging.root.removeHandler(handler) install() log_level = os.environ.get('LOGLEVEL', 'INFO').upper() log_level = os.environ.get("LOGLEVEL", "INFO").upper() rich_handler = RichHandler(rich_tracebacks=True, markup=True) if file_out is not None: file_handler = logging.FileHandler(file_out, mode, encoding) file_handler_fmt = logging.Formatter('[%(asctime)s]' + '%(levelname)8s - ' + ' - %(message)s') file_handler_fmt = logging.Formatter( "[%(asctime)s]" + "%(levelname)8s - " + " - %(message)s" ) file_handler.setFormatter(file_handler_fmt) handlers = [rich_handler, file_handler] else: file_handler = None handlers = [rich_handler] logging.basicConfig(level=log_level, format='%(message)s', datefmt="[%Y/%m/%d %H:%M;%S]", handlers=handlers) logging.basicConfig( level=log_level, format="%(message)s", datefmt="[%Y/%m/%d %H:%M;%S]", handlers=handlers ) # if there is a file_handler set, close it before leaving :) # This prevents leaving open files if file_handler is not None: file_handler.close() rich_handler.close() return logging.getLogger('rich') return logging.getLogger("rich")
