Commit 1e03c6b4 authored by Cage, Gregory's avatar Cage, Gregory
Browse files

Merge branch '28-connection-store-refactor' into 'main' 

Resolve "Simplify job management"

Closes #28

See merge request ndip/public-packages/nova-galaxy!18
parents 68ef2c43 ca9d3b9e

CHANGELOG.md

+22 −0
Original line number Diff line number Diff line 
### Nova Galaxy, 0.8.0

- `get_data_store()` has been added to the ConnectionHelper class. This functionally does the same thing as create_data_store, but users can choose whether to only use existing upstream data stores. `create_data_store` creates data stores by default and connects to existing ones as well automatically (thanks to Gregory Cage).  [Merge Request 18](https://code.ornl.gov/ndip/public-packages/nova-galaxy/-/merge_requests/18)

- `Connections.connect()` can now be used with or without the `with` keyword. Consequently, stores can also be created outside a `with` block. `Connection.close()` performs the clean up that exiting the `with` block provides (thanks to Gregory Cage). [Merge Request 18](https://code.ornl.gov/ndip/public-packages/nova-galaxy/-/merge_requests/18)

- Data stores can be cleaned up manually (thanks to Gregory Cage). [Merge Request 18](https://code.ornl.gov/ndip/public-packages/nova-galaxy/-/merge_requests/18)

- Can now wait for the result of a running tool (thanks to Gregory Cage). [Merge Request 18](https://code.ornl.gov/ndip/public-packages/nova-galaxy/-/merge_requests/18)





### Nova Galaxy, 0.7.4

- Allow users to choose to check URL when calling get_url() from a Tool (thanks to Gregory Cage). [Merge Request 17](https://code.ornl.gov/ndip/public-packages/nova-galaxy/-/merge_requests/17)

- Return more detailed information when getting the content of a DatasetCollection (thanks to Gregory Cage). [Merge Request 17](https://code.ornl.gov/ndip/public-packages/nova-galaxy/-/merge_requests/17)

- Data stores are now persisted by default. A new mark_for_cleanup method has been provided to clean up data stores after usage. The persist method's behavior remains unchanged (thanks to Gregory Cage). [Merge Request 17](https://code.ornl.gov/ndip/public-packages/nova-galaxy/-/merge_requests/17)



### Nova Galaxy, 0.7.3

- Allow Dataset content to be set manually in memory rather than only loading from a file or downloading from Galaxy (thanks to Gregory Cage). [Merge Request 16](https://code.ornl.gov/ndip/public-packages/nova-galaxy/-/merge_requests/16)

- Add file type (extensions) to Datasets (thanks to Gregory Cage). [Merge Request 16](https://code.ornl.gov/ndip/public-packages/nova-galaxy/-/merge_requests/16)



### Nova Galaxy, 0.7.2

- Add more states to Work State enum (thanks to Gregory Cage). [Merge Request 15](https://code.ornl.gov/ndip/public-packages/nova-galaxy/-/merge_requests/15)



### Nova Galaxy, 0.7.1

- Speeds ups recovering tools from data stores. (thanks to Gregory Cage). [Merge Request 14](https://code.ornl.gov/ndip/public-packages/nova-galaxy/-/merge_requests/14)



### Nova Galaxy, 0.7.0

- Reworks some issues where the url was trying to be fetched in scenarios where it would take the full timeout (thanks to Gregory Cage).  [Merge Request 13](https://code.ornl.gov/ndip/public-packages/nova-galaxy/-/merge_requests/13)

- Added a lot more user documentation (thanks to Gregory Cage).  [Merge Request 13](https://code.ornl.gov/ndip/public-packages/nova-galaxy/-/merge_requests/13)

poetry.lock

+926 −778

File changed.

Preview size limit exceeded, changes collapsed.

pyproject.toml

+3 −2
Original line number Diff line number Diff line 
[tool.poetry]

name = "nova-galaxy"

version = "0.7.4"

version = "0.8.0"

description = "Utilties for accessing the ORNL Galaxy instance"

authors = ["Greg Watson <watsongr@ornl.gov>", "Gregory Cage <cagege@ornl.gov>"]

readme = "README.md"
@@ -13,8 +13,9 @@ packages = [ 


[tool.poetry.dependencies]

python = "^3.10"

bioblend = "^1.3.0"

bioblend = "^1.5.0"

tomli = "^2.0.2"

deprecated = "^1.2.18"



[build-system]

requires = ["poetry-core"]

src/nova/galaxy/connection.py

+74 −29
Original line number Diff line number Diff line 
"""The NOVA class is responsible for managing interactions with a Galaxy server instance."""



from contextlib import contextmanager

from typing import Generator, List, Optional

from typing import Any, List



from bioblend import galaxy

from deprecated import deprecated



from .data_store import Datastore

from .tool import stop_all_tools_in_store
@@ -26,7 +26,7 @@ class ConnectionHelper: 
    """Manages datastore for current connection.



    Should not be instantiated manually. Use Connection.connect() instead. Any stores created using the connection will

    be automatically purged after connection is closed, unless Datastore.persist() is called for that store.

    be persisted after connection is closed, unless Datastore.mark_for_cleanup() is called for that store.

    """



    def __init__(self, galaxy_instance: galaxy.GalaxyInstance, galaxy_url: str):
@@ -34,22 +34,70 @@ class ConnectionHelper: 
        self.galaxy_url = galaxy_url

        self.datastores: List[Datastore] = []



    def __enter__(self) -> Any:

        """Enter method for use with "with" keyword."""

        return self



    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:

        """Exit method for use with "with" keyword."""

        self.close()



    @deprecated(version="0.8.0", reason="Should use `get_data_store() instead.")

    def create_data_store(self, name: str) -> Datastore:

        """Creates a datastore with the given name."""

        """DEPRECATED. Creates a datastore with the given name or returns an existing data store with that name.



        Parameters

        ----------

        name: str

            Name of the data store.

        """

        return self.get_data_store(name=name, create=True)



    def get_data_store(self, name: str, create: bool = True) -> Datastore:

        """Fetches a datastore with the given name.



        Parameters

        ----------

        name: str

            Name of the data store.

        create: bool

            If true, creates a data store if one does not exist with the specified name.



        Returns

        -------

        Datastore

            Returns the specified or newly created data store.

        """

        histories = self.galaxy_instance.histories.get_histories(name=name)

        if len(histories) > 0:

            store = Datastore(name, self, histories[0]["id"])

            self.datastores.append(store)

            return store

        if create:

            history_id = self.galaxy_instance.histories.create_history(name=name)["id"]

            store = Datastore(name, self, history_id)

            self.datastores.append(store)

            return store

        else:

            raise Exception("Data store does not exist and auto creation is set to false.")



    def remove_data_store(self, store: Datastore) -> None:

        """Permanently deletes the data store with the given name."""

        history = self.galaxy_instance.histories.get_histories(name=store.name)[0]["id"]

        self.galaxy_instance.histories.delete_history(history_id=history, purge=True)

        """Permanently deletes the data store with the given name.



        Parameters

        ----------

        store: Datastore

            The data store to remove from this connection.

        """

        store.cleanup()

        self.datastores.remove(store)



    def close(self) -> None:

        # Remove all data stores after execution

        for store in self.datastores:

            if not store.persist_store:

                stop_all_tools_in_store(store)

                self.remove_data_store(store)





class Connection:
@@ -64,22 +112,29 @@ class Connection: 


    def __init__(

        self,

        galaxy_url: Optional[str] = None,

        galaxy_key: Optional[str] = None,

        galaxy_url: str,

        galaxy_key: str,

    ) -> None:

        """

        Initializes the Connection instance with the provided URL and API key.



        Args:

            galaxy_url (Optional[str]): URL of the Galaxy instance.

            galaxy_key (Optional[str]): API key for the Galaxy instance.

            galaxy_url str: URL of the Galaxy instance.

            galaxy_key str: API key for the Galaxy instance.

        """

        self.galaxy_url = galaxy_url

        self.galaxy_api_key = galaxy_key

        self.galaxy_instance: galaxy.GalaxyInstance



    @contextmanager

    def connect(self) -> Generator:

    def _init_galaxy_instance(self) -> None:

        if not self.galaxy_url or not self.galaxy_api_key:

            raise ValueError("Galaxy URL and API key must be provided.")

        if not isinstance(self.galaxy_url, str):

            raise ValueError("Galaxy URL must be a string")

        self.galaxy_instance = galaxy.GalaxyInstance(url=self.galaxy_url, key=self.galaxy_api_key)

        self.galaxy_instance.config.get_version()



    def connect(self) -> ConnectionHelper:

        """

        Connects to the Galaxy instance using the provided URL and API key.
@@ -89,16 +144,6 @@ class Connection: 
        ------

            ValueError: If the Galaxy URL or API key is not provided.

        """

        if not self.galaxy_url or not self.galaxy_api_key:

            raise ValueError("Galaxy URL and API key must be provided or set in environment variables.")

        if not isinstance(self.galaxy_url, str):

            raise ValueError("Galaxy URL must be a string")

        self.galaxy_instance = galaxy.GalaxyInstance(url=self.galaxy_url, key=self.galaxy_api_key)

        self.galaxy_instance.config.get_version()

        self._init_galaxy_instance()

        conn = ConnectionHelper(self.galaxy_instance, self.galaxy_url)

        yield conn

        # Remove all data stores after execution

        for store in conn.datastores:

            if not store.persist_store:

                stop_all_tools_in_store(store)

                conn.remove_data_store(store)
 
        return conn

src/nova/galaxy/data_store.py

+4 −0
Original line number Diff line number Diff line
@@ -32,6 +32,10 @@ class Datastore: 
        """Clean up and delete all content related to this Data Store after the associated connection is closed."""

        self.persist_store = False



    def cleanup(self) -> None:

        history = self.nova_connection.galaxy_instance.histories.get_histories(name=self.name)[0]["id"]

        self.nova_connection.galaxy_instance.histories.delete_history(history_id=history, purge=True)



    def recover_tools(self, filter_running: bool = True) -> List[Tool]:

        """Recovers all running tools in this data_store.