Merge pull request #12656 from mvdbeek/resumable_upload (64329459) · Commits · NDIP / Galaxy

client/package.json

+1 −0

Original line number	Diff line number	Diff line
		@@ -77,6 +77,7 @@
		"threads": "^1.6.5",
		"timers-browserify": "^2.0.12",
		"toastr": "^2.1.4",
		"tus-js-client": "^2.3.0",
		"underscore": "^1.10.2",
		"underscore.string": "^3.3.5",
		"vue": "^2.6.14",

client/src/components/Upload/Collection.vue

+1 −0

Original line number	Diff line number	Diff line
		@@ -202,6 +202,7 @@ export default {
		ondragleave: () => {
		this.highlightBox = false;
		},
		chunkSize: this.app.chunkUploadSize,
		});
		this.collection.on("remove", (model) => {
		this._eventRemove(model);

client/src/components/Upload/Default.vue

+2 −6

Original line number	Diff line number	Diff line
		@@ -112,7 +112,6 @@
		<script>
		import _l from "utils/localization";
		import _ from "underscore";
		import { getGalaxyInstance } from "app";
		import UploadRow from "mvc/upload/default/default-row";
		import UploadBoxMixin from "./UploadBoxMixin";
		import { BButton } from "bootstrap-vue";
		@@ -199,6 +198,7 @@ export default {
		ondragleave: () => {
		this.highlightBox = false;
		},
		chunkSize: this.app.chunkUploadSize,
		});
		this.collection.on("remove", (model) => {
		this._eventRemove(model);
		@@ -282,11 +282,7 @@ export default {
		this._uploadFtp();

		// queue remaining files
		const Galaxy = getGalaxyInstance();
		this.uploadbox.start({
		id: Galaxy.user.id,
		chunk_upload_size: this.app.chunkUploadSize,
		});
		this.uploadbox.start();
		this._updateStateForCounters();
		}
		},

client/src/utils/uploadbox.js

+59 −87

Original line number	Diff line number	Diff line
		@@ -5,6 +5,7 @@
		import _ from "underscore";
		import jQuery from "jquery";
		import { getAppRoot } from "onload/loadConfig";
		import * as tus from "tus-js-client";

		(($) => {
		// add event properties
		@@ -79,6 +80,7 @@ import { getAppRoot } from "onload/loadConfig";
		error_file: "File not provided.",
		error_attempt: "Maximum number of attempts reached.",
		error_tool: "Tool submission failed.",
		chunkSize: 10485760,
		},
		config
		);
		@@ -94,43 +96,38 @@ import { getAppRoot } from "onload/loadConfig";
		cnf.error(cnf.error_file);
		return;
		}
		var file = file_data.file;
		var attempts = cnf.attempts;
		var session_id = `${cnf.session.id}-${new Date().valueOf()}-${file.size}`;
		var chunk_size = cnf.session.chunk_upload_size;
		console.debug(`Starting chunked uploads [size=${chunk_size}].`);

		// chunk processing helper
		function process(start) {
		start = start \|\| 0;
		var slicer = file.mozSlice \|\| file.webkitSlice \|\| file.slice;
		if (!slicer) {
		cnf.error("Browser does not support chunked uploads.");
		return;
		}
		var end = Math.min(start + chunk_size, file.size);
		var size = file.size;
		console.debug(`Submitting chunk at ${start} bytes...`);
		var form = new FormData();
		form.append("session_id", session_id);
		form.append("session_start", start);
		form.append("session_chunk", slicer.bind(file)(start, end));
		_uploadrequest({
		url: `${getAppRoot()}api/uploads`,
		data: form,
		success: (upload_response) => {
		var new_start = start + chunk_size;
		if (new_start < size) {
		attempts = cnf.attempts;
		process(new_start);
		} else {
		console.debug("Upload completed.");
		data.payload.inputs = JSON.parse(data.payload.inputs);
		data.payload.inputs["files_0\|file_data"] = {
		session_id: session_id,
		const startTime = performance.now();
		const file = file_data.file;
		const tusEndpoint = `${getAppRoot()}api/upload/resumable_upload/`;
		const chunkSize = cnf.chunkSize;
		console.debug(`Starting chunked uploads [size=${chunkSize}].`);

		const upload = new tus.Upload(file, {
		endpoint: tusEndpoint,
		chunkSize: chunkSize,
		metadata: data.payload,
		onError: function (error) {
		console.log("Failed because: " + error);
		cnf.error(error);
		},
		onProgress: function (bytesUploaded, bytesTotal) {
		var percentage = ((bytesUploaded / bytesTotal) * 100).toFixed(2);
		console.log(bytesUploaded, bytesTotal, percentage + "%");
		cnf.progress(percentage);
		},
		onSuccess: function () {
		console.log(
		`Upload of ${upload.file.name} to ${upload.url} took ${
		(performance.now() - startTime) / 1000
		} seconds`
		);

		const toolInputs = JSON.parse(data.payload.inputs);
		toolInputs["files_0\|file_data"] = {
		session_id: upload.url.split("/").at(-1),
		name: file.name,
		};
		data.payload.inputs = JSON.stringify(data.payload.inputs);
		data.payload.inputs = JSON.stringify(toolInputs);
		$.ajax({
		url: `${getAppRoot()}api/tools`,
		method: "POST",
		@@ -139,37 +136,23 @@ import { getAppRoot } from "onload/loadConfig";
		cnf.success(tool_response);
		},
		error: (tool_response) => {
		var err_msg =
		tool_response && tool_response.responseJSON && tool_response.responseJSON.err_msg;
		var err_msg = tool_response && tool_response.responseJSON && tool_response.responseJSON.err_msg;
		cnf.error(err_msg \|\| cnf.error_tool);
		},
		});
		}
		},
		warning: (upload_response) => {
		if (--attempts > 0) {
		console.debug("Retrying last chunk...");
		cnf.warning(upload_response);
		setTimeout(() => process(start), cnf.timeout);
		} else {
		console.debug(cnf.error_attempt);
		cnf.error(cnf.error_attempt);
		}
		},
		error: (upload_response) => {
		console.debug(upload_response);
		cnf.error(upload_response);
		},
		progress: (e) => {
		if (e.lengthComputable) {
		cnf.progress(Math.min(Math.round(((start + e.loaded) * 100) / file.size), 100));
		}
		},
		});
		// Check if there are any previous uploads to continue.
		upload.findPreviousUploads().then(function (previousUploads) {
		// Found previous uploads so we select the first one.
		if (previousUploads.length) {
		console.log("previous Upload", previousUploads);
		upload.resumeFromPreviousUpload(previousUploads[0]);
		}

		// initiate processing queue for chunks
		process();
		// Start the upload
		upload.start();
		});
		};

		/**
		@@ -343,9 +326,6 @@ import { getAppRoot } from "onload/loadConfig";
		// file queue
		var queue = {};

		// session options
		var session = null;

		// queue index/length counter
		var queue_index = 0;
		var queue_length = 0;
		@@ -427,13 +407,7 @@ import { getAppRoot } from "onload/loadConfig";
		// create and submit data
		var submitter = $.uploadpost;
		var requestData = opts.initialize(index);
		if (
		file.chunk_mode &&
		session &&
		session.id &&
		session.chunk_upload_size &&
		session.chunk_upload_size > 0
		) {
		if (file.chunk_mode && opts.chunkSize > 0) {
		submitter = $.uploadchunk;
		} else if (requestData.fetchRequest) {
		submitter = $.datafetchpost;
		@@ -442,7 +416,6 @@ import { getAppRoot } from "onload/loadConfig";
		submitter({
		url: opts.initUrl(index),
		data: requestData.fetchRequest ? requestData.fetchRequest : requestData.uploadRequest,
		session: session,
		success: (message) => {
		opts.success(index, message);
		process();
		@@ -477,8 +450,7 @@ import { getAppRoot } from "onload/loadConfig";
		}

		// initiate upload process
		function start(_session) {
		session = _session;
		function start() {
		if (!queue_running) {
		queue_running = true;
		process();

client/yarn.lock

+99 −0

Original line number	Diff line number	Diff line
		@@ -3061,6 +3061,11 @@ buffer-from@1.1.1:
		resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"
		integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==

		buffer-from@^0.1.1:
		version "0.1.2"
		resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-0.1.2.tgz#15f4b9bcef012044df31142c14333caf6e0260d0"
		integrity sha512-RiWIenusJsmI2KcvqQABB83tLxCByE3upSP8QU3rJDMVFGPWLvPQJt/O1Su9moRWeH7d+Q2HYb68f6+v+tw2vg==

		buffer-from@^1.0.0:
		version "1.1.2"
		resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5"
		@@ -3620,6 +3625,14 @@ colors@~0.6.0-1:
		resolved "https://registry.yarnpkg.com/colors/-/colors-0.6.2.tgz#2423fe6678ac0c5dae8852e5d0e5be08c997abcc"
		integrity sha1-JCP+ZnisDF2uiFLl0OW+CMmXq8w=

		combine-errors@^3.0.3:
		version "3.0.3"
		resolved "https://registry.yarnpkg.com/combine-errors/-/combine-errors-3.0.3.tgz#f4df6740083e5703a3181110c2b10551f003da86"
		integrity sha1-9N9nQAg+VwOjGBEQwrEFUfAD2oY=
		dependencies:
		custom-error-instance "2.1.1"
		lodash.uniqby "4.5.0"

		combined-stream@^1.0.8:
		version "1.0.8"
		resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f"
		@@ -4132,6 +4145,11 @@ csstype@^3.0.2:
		resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.0.9.tgz#6410af31b26bd0520933d02cbc64fce9ce3fbf0b"
		integrity sha512-rpw6JPxK6Rfg1zLOYCSwle2GFOOsnjmDYDaBwEcwoOg4qlsIVCN789VkBZDJAGi4T07gI4YSutR43t9Zz4Lzuw==

		custom-error-instance@2.1.1:
		version "2.1.1"
		resolved "https://registry.yarnpkg.com/custom-error-instance/-/custom-error-instance-2.1.1.tgz#3cf6391487a6629a6247eb0ca0ce00081b7e361a"
		integrity sha1-PPY5FIemYppiR+sMoM4ACBt+Nho=

		custom-event@~1.0.0:
		version "1.0.1"
		resolved "https://registry.yarnpkg.com/custom-event/-/custom-event-1.0.1.tgz#5d02a46850adf1b4a317946a3928fccb5bfd0425"
		@@ -7534,6 +7552,11 @@ jquery@>=1.12.0, "jquery@>=1.8.0 <4.0.0", jquery@^3.0.0:
		resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.6.0.tgz#c72a09f15c1bdce142f49dbf1170bdf8adac2470"
		integrity sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw==

		js-base64@^2.6.1:
		version "2.6.4"
		resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.6.4.tgz#f4e686c5de1ea1f867dbcad3d46d969428df98c4"
		integrity sha512-pZe//GGmwJndub7ZghVHz7vjb2LgC1m8B07Au3eYqeqv9emhESByMXxaEgkUkEqJe87oBbSniGYoQNIBklc7IQ==

		js-beautify@^1.6.12, js-beautify@^1.6.14:
		version "1.14.0"
		resolved "https://registry.yarnpkg.com/js-beautify/-/js-beautify-1.14.0.tgz#2ce790c555d53ce1e3d7363227acf5dc69024c2d"
		@@ -8105,6 +8128,43 @@ locate-path@^6.0.0:
		dependencies:
		p-locate "^5.0.0"

		lodash._baseiteratee@~4.7.0:
		version "4.7.0"
		resolved "https://registry.yarnpkg.com/lodash._baseiteratee/-/lodash._baseiteratee-4.7.0.tgz#34a9b5543572727c3db2e78edae3c0e9e66bd102"
		integrity sha1-NKm1VDVycnw9sueO2uPA6eZr0QI=
		dependencies:
		lodash._stringtopath "~4.8.0"

		lodash._basetostring@~4.12.0:
		version "4.12.0"
		resolved "https://registry.yarnpkg.com/lodash._basetostring/-/lodash._basetostring-4.12.0.tgz#9327c9dc5158866b7fa4b9d42f4638e5766dd9df"
		integrity sha1-kyfJ3FFYhmt/pLnUL0Y45XZt2d8=

		lodash._baseuniq@~4.6.0:
		version "4.6.0"
		resolved "https://registry.yarnpkg.com/lodash._baseuniq/-/lodash._baseuniq-4.6.0.tgz#0ebb44e456814af7905c6212fa2c9b2d51b841e8"
		integrity sha1-DrtE5FaBSveQXGIS+iybLVG4Qeg=
		dependencies:
		lodash._createset "~4.0.0"
		lodash._root "~3.0.0"

		lodash._createset@~4.0.0:
		version "4.0.3"
		resolved "https://registry.yarnpkg.com/lodash._createset/-/lodash._createset-4.0.3.tgz#0f4659fbb09d75194fa9e2b88a6644d363c9fe26"
		integrity sha1-D0ZZ+7CddRlPqeK4imZE02PJ/iY=

		lodash._root@~3.0.0:
		version "3.0.1"
		resolved "https://registry.yarnpkg.com/lodash._root/-/lodash._root-3.0.1.tgz#fba1c4524c19ee9a5f8136b4609f017cf4ded692"
		integrity sha1-+6HEUkwZ7ppfgTa0YJ8BfPTe1pI=

		lodash._stringtopath@~4.8.0:
		version "4.8.0"
		resolved "https://registry.yarnpkg.com/lodash._stringtopath/-/lodash._stringtopath-4.8.0.tgz#941bcf0e64266e5fc1d66fed0a6959544c576824"
		integrity sha1-lBvPDmQmbl/B1m/tCmlZVExXaCQ=
		dependencies:
		lodash._basetostring "~4.12.0"

		lodash.clonedeep@^4.5.0:
		version "4.5.0"
		resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef"
		@@ -8135,6 +8195,11 @@ lodash.merge@^4.6.2:
		resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a"
		integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==

		lodash.throttle@^4.1.1:
		version "4.1.1"
		resolved "https://registry.yarnpkg.com/lodash.throttle/-/lodash.throttle-4.1.1.tgz#c23e91b710242ac70c37f1e1cda9274cc39bf2f4"
		integrity sha1-wj6RtxAkKscMN/HhzaknTMOb8vQ=

		lodash.truncate@^4.4.2:
		version "4.4.2"
		resolved "https://registry.yarnpkg.com/lodash.truncate/-/lodash.truncate-4.4.2.tgz#5a350da0b1113b837ecfffd5812cbe58d6eae193"
		@@ -8145,6 +8210,14 @@ lodash.uniq@^4.5.0:
		resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
		integrity sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=

		lodash.uniqby@4.5.0:
		version "4.5.0"
		resolved "https://registry.yarnpkg.com/lodash.uniqby/-/lodash.uniqby-4.5.0.tgz#a3a17bbf62eeb6240f491846e97c1c4e2a5e1e21"
		integrity sha1-o6F7v2LutiQPSRhG6XwcTipeHiE=
		dependencies:
		lodash._baseiteratee "~4.7.0"
		lodash._baseuniq "~4.6.0"

		lodash@^4.13.1, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.21, lodash@^4.17.4, lodash@^4.7.0:
		version "4.17.21"
		resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
		@@ -10096,6 +10169,14 @@ prop-types@^15.6.2, prop-types@^15.7.2:
		object-assign "^4.1.1"
		react-is "^16.8.1"

		proper-lockfile@^2.0.1:
		version "2.0.1"
		resolved "https://registry.yarnpkg.com/proper-lockfile/-/proper-lockfile-2.0.1.tgz#159fb06193d32003f4b3691dd2ec1a634aa80d1d"
		integrity sha1-FZ+wYZPTIAP0s2kd0uwaY0qoDR0=
		dependencies:
		graceful-fs "^4.1.2"
		retry "^0.10.0"

		proper-skip-list@^4.0.2:
		version "4.1.0"
		resolved "https://registry.yarnpkg.com/proper-skip-list/-/proper-skip-list-4.1.0.tgz#d23bbb844591e1e05ee66b910a740c595248da79"
		@@ -10927,6 +11008,11 @@ ret@~0.1.10:
		resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc"
		integrity sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==

		retry@^0.10.0:
		version "0.10.1"
		resolved "https://registry.yarnpkg.com/retry/-/retry-0.10.1.tgz#e76388d217992c252750241d3d3956fed98d8ff4"
		integrity sha1-52OI0heZLCUnUCQdPTlW/tmNj/Q=

		retry@^0.12.0:
		version "0.12.0"
		resolved "https://registry.yarnpkg.com/retry/-/retry-0.12.0.tgz#1b42a6266a21f07421d1b0b54b7dc167b01c013b"
		@@ -12297,6 +12383,19 @@ tslib@^2.0.1:
		resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01"
		integrity sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==

		tus-js-client@^2.3.0:
		version "2.3.0"
		resolved "https://registry.yarnpkg.com/tus-js-client/-/tus-js-client-2.3.0.tgz#5d76145476cea46a4e7c045a0054637cddf8dc39"
		integrity sha512-I4cSwm6N5qxqCmBqenvutwSHe9ntf81lLrtf6BmLpG2v4wTl89atCQKqGgqvkodE6Lx+iKIjMbaXmfvStTg01g==
		dependencies:
		buffer-from "^0.1.1"
		combine-errors "^3.0.3"
		is-stream "^2.0.0"
		js-base64 "^2.6.1"
		lodash.throttle "^4.1.1"
		proper-lockfile "^2.0.1"
		url-parse "^1.4.3"

		type-check@^0.4.0, type-check@~0.4.0:
		version "0.4.0"
		resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1"

Admin message