Unverified Commit b85297c0 authored by John Chilton's avatar John Chilton Committed by GitHub
Browse files

Merge pull request #13945 from mvdbeek/cleanup_tus_uploads 

[22.01] Remove tus uploads during fetch data job
parents e6b04a8d bd8f7e89

client/src/utils/uploadbox.js

+9 −0
Original line number Diff line number Diff line
@@ -19,6 +19,14 @@ function submitPayload(payload, cnf) { 
        });

}



function buildFingerprint(cnf) {

    async function customFingerprint(file, options) {

        return ["tus-br", file.name, file.type, file.size, file.lastModified, cnf.data.history_id].join("-");

    }



    return customFingerprint;

}



function tusUpload(data, index, tusEndpoint, cnf) {

    const startTime = performance.now();

    const chunkSize = cnf.chunkSize;
@@ -31,6 +39,7 @@ function tusUpload(data, index, tusEndpoint, cnf) { 
    console.debug(`Starting chunked upload for ${file.name} [chunkSize=${chunkSize}].`);

    const upload = new tus.Upload(file, {

        endpoint: tusEndpoint,

        fingerprint: buildFingerprint(cnf),

        chunkSize: chunkSize,

        metadata: data.payload,

        onError: function (error) {

lib/galaxy/tools/data_fetch.py

+5 −0
Original line number Diff line number Diff line
@@ -502,6 +502,11 @@ class UploadConfig: 
        if purge_source:

            try:

                shutil.move(path, new_path)

                # Drop .info file if it exists

                try:

                    os.remove(f"{path}.info")

                except FileNotFoundError:

                    pass

            except OSError as e:

                # We may not have permission to remove converted_path

                if e.errno != errno.EACCES:

lib/galaxy/webapps/galaxy/api/_fetch_util.py

+1 −0
Original line number Diff line number Diff line
@@ -185,6 +185,7 @@ def validate_and_normalize_targets(trans, payload): 
            item["in_place"] = run_as_real_user

        elif src == "files":

            item["in_place"] = run_as_real_user

            item["purge_source"] = True



        # Small disagreement with traditional uploads - we purge less by default since whether purging

        # happens varies based on upload options in non-obvious ways.