Commit c5bf63bc authored by John Chilton's avatar John Chilton
Browse files

Add DEFERRED datasets to Galaxy 

- Improves options and display around discarded datasets.
- Adds new APIs for importing model stores.
- Adds support to model store layer for importing/exporting workflow invocations and APIs and tests to support this.
- Add new DEFERRED dataset state - with APIs for materializing them or the option to defer them on the worker node if using tool_evaluation_strategy=remote on the job destination.
- UI and API options for uploading deferred datasets individually or into collections.
parent d2204b65

client/src/components/Dataset/DatasetStorage/DatasetStorage.vue

+31 −0
Original line number Diff line number Diff line
@@ -3,6 +3,18 @@ 
        <h3 v-if="includeTitle">Dataset Storage</h3>

        <div v-if="errorMessage" class="error">{{ errorMessage }}</div>

        <loading-span v-else-if="storageInfo == null"> </loading-span>

        <div v-else-if="discarded">

            <p>This dataset has been discarded and its files are not available to Galaxy.</p>

        </div>

        <div v-else-if="deferred">

            <p>

                This dataset is remote and deferred. The dataset's files are not available to Galaxy.

                <span v-if="sourceUri">

                    This dataset will be downloaded from <b class="deferred-dataset-source-uri">{{ sourceUri }}</b> when

                    jobs use this dataset.

                </span>

            </p>

        </div>

        <div v-else>

            <p>

                This dataset is stored in
@@ -50,6 +62,25 @@ export default { 
            errorMessage: null,

        };

    },

    computed: {

        discarded() {

            return this.storageInfo.dataset_state == "discarded";

        },

        deferred() {

            return this.storageInfo.dataset_state == "deferred";

        },

        sourceUri() {

            const sources = this.storageInfo.sources;

            if (!sources) {

                return null;

            }

            const rootSources = sources.filter((source) => !source.extra_files_path);

            if (rootSources.length == 0) {

                return null;

            }

            return rootSources[0].source_uri;

        },

    },

    created() {

        const datasetId = this.datasetId;

        const datasetType = this.datasetType;

client/src/components/History/Content/model/states.js

+6 −1
Original line number Diff line number Diff line
@@ -6,7 +6,12 @@ export const STATES = { 
    /** deleted while uploading */

    discarded: {

        status: "danger",

        text: "The job creating this dataset was cancelled before completion.",

        text: "This dataset is discarded - the job creating it may have been cancelled or it may have been imported without file data.",

        icon: "exclamation-triangle",

    },

    deferred: {

        status: "danger",

        text: "This dataset is remote, has not be ingested by Galaxy, and full metadata may not be available.",

        icon: "exclamation-triangle",

    },

    /** has no data */

client/src/components/RuleCollectionBuilder.vue

+23 −5
Original line number Diff line number Diff line
@@ -980,8 +980,8 @@ export default { 
            } else if (this.elementsType == "ftp") {

                metadataOptions["path"] = _l("Path");

            } else if (this.elementsType == "remote_files") {

                // IS THIS NEEDED?

                metadataOptions["url"] = _l("URL");

                metadataOptions["url_deferred"] = _l("URL (deferred)");

            } else if (this.elementsType == "library_datasets") {

                metadataOptions["name"] = _l("Name");

            } else if (this.elementsType == "datasets") {
@@ -1034,7 +1034,17 @@ export default { 
        validOnlyOnePath() {

            let valid = true;

            const mappingAsDict = this.mappingAsDict;

            if (mappingAsDict.ftp_path && mappingAsDict.url) {

            let pathSourceCount = 0;

            if (mappingAsDict.ftp_path) {

                pathSourceCount += 1;

            }

            if (mappingAsDict.url) {

                pathSourceCount += 1;

            }

            if (mappingAsDict.url_deferred) {

                pathSourceCount += 1;

            }

            if (pathSourceCount > 1) {

                // Can only specify one of these.

                valid = false;

            }
@@ -1045,7 +1055,7 @@ export default { 
            const mappingAsDict = this.mappingAsDict;

            const requiresSourceColumn =

                this.elementsType == "ftp" || this.elementsType == "raw" || this.elementsType == "remote_files";

            if (requiresSourceColumn && !mappingAsDict.ftp_path && !mappingAsDict.url) {

            if (requiresSourceColumn && !mappingAsDict.ftp_path && !mappingAsDict.url && !mappingAsDict.url_deferred) {

                valid = false;

            }

            return valid;
@@ -1662,8 +1672,13 @@ export default { 
        },

        _datasetFor(dataIndex, data, mappingAsDict) {

            const res = {};

            if (mappingAsDict.url || mappingAsDict.url_deferred) {

                let urlColumn;

                if (mappingAsDict.url) {

                const urlColumn = mappingAsDict.url.columns[0];

                    urlColumn = mappingAsDict.url.columns[0];

                } else {

                    urlColumn = mappingAsDict.url_deferred.columns[0];

                }

                let url = data[dataIndex][urlColumn];

                url = url.trim();

                if (url.indexOf("://") == -1) {
@@ -1677,6 +1692,9 @@ export default { 
                }

                res["url"] = url;

                res["src"] = "url";

                if (mappingAsDict.url_deferred) {

                    res["deferred"] = true;

                }

            } else if (mappingAsDict.ftp_path) {

                const ftpPathColumn = mappingAsDict.ftp_path.columns[0];

                const ftpPath = data[dataIndex][ftpPathColumn];

client/src/components/Upload/helpers.js

+1 −1
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ export function uploadModelsToPayload(items, history_id, composite = false) { 
                ext: item.get("extension", "auto"),

                space_to_tab: item.get("space_to_tab"),

                to_posix_lines: item.get("to_posix_lines"),

                deferred: item.get("deferred"),

            };

            switch (item.get("file_mode")) {

                case "new":
@@ -84,7 +85,6 @@ export function uploadModelsToPayload(items, history_id, composite = false) { 
        })

        .filter((item) => item)

        .flat();



    const target = {

        destination: { type: "hdas" },

        elements: elements,

client/src/mvc/dataset/dataset-li.js

+15 −1
Original line number Diff line number Diff line
@@ -525,7 +525,21 @@ DatasetListItemView.prototype.templates = (() => { 
        "dataset"

    );

    summaryTemplates[STATES.DISCARDED] = BASE_MVC.wrapTemplate(

        ["<div>", _l("The job creating this dataset was cancelled before completion"), "</div>"],

        [

            "<div>",

            _l(

                "This dataset is discarded - the job creating it may have been cancelled or it may have been imported with file data"

            ),

            "</div>",

        ],

        "dataset"

    );

    summaryTemplates[STATES.DEFERRED] = BASE_MVC.wrapTemplate(

        [

            "<div>",

            _l("This dataset is remote, has not be ingested by Galaxy, and full metadata may not be available"),

            "</div>",

        ],

        "dataset"

    );

    summaryTemplates[STATES.QUEUED] = BASE_MVC.wrapTemplate(