Skip to content

Commit 29f6a77

Browse files
author
Joonatan Mäkinen
committed
Gnu tar (merge commit)
Merge branch 'feature/gnu-tar' into 'main' * Add folder paths to array as strings to fix adding tar subfolders * Fix downloading a single file instead of whole container due to misplaced bracket * Update changelog * Restore the size of long name header block * fix unit test issues after buffering changes * don't use TextEncoder for tar headers * improve download buffering * Add file size to tar header for files over 8gib * Refactor code * Initial gnu tar header implementation. Enable unlimited container and object names * Remove ustar limit checks Closes #1219 and #1162 See merge request https://gitlab.ci.csc.fi/sds-dev/sd-connect/swift-browser-ui/-/merge_requests/223 Approved-by: Hang Le <[email protected]> Co-authored-by: Monika Radaviciute <[email protected]> Co-authored-by: Sampsa Penna <[email protected]> Co-authored-by: Monika Radaviciute <[email protected]> Merged by Joonatan Mäkinen <[email protected]>
2 parents ee8d37c + d43bda6 commit 29f6a77

File tree

14 files changed

+277
-168
lines changed

14 files changed

+277
-168
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
2323
- Cypress test related
2424
- Refactor entries' files, IndexPage and IndexOIDCPage
2525
- (GL #1200) New terms for sharing permissions
26+
- (GL #1162) Change from ustar to gnu tar
2627

2728
### Removed
2829
- (GL #1204) Remove old upload code from frontend

‎swift_browser_ui/ui/api.py

+8
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,14 @@ async def swift_download_object(request: aiohttp.web.Request) -> aiohttp.web.Res
299299
"GET",
300300
)
301301

302+
# Append query string with query parameters from the request,
303+
# mainly to allow ranged requests.
304+
parsed = urllib.parse.urlparse(url)
305+
qs = urllib.parse.parse_qs(parsed.query)
306+
qs.update(request.query) # type: ignore
307+
parsed._replace(query=urllib.parse.urlencode(qs, doseq=True)) # type: ignore
308+
url = parsed.geturl()
309+
302310
async with client.head(
303311
f"{endpoint}/{container}/{object_name}",
304312
headers={

‎swift_browser_ui_frontend/.eslintrc.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"env": {
33
"browser": true,
4-
"es6": true,
4+
"es2020": true,
55
"node": true,
66
"jest": true
77
},

‎swift_browser_ui_frontend/src/common/globalFunctions.js

-22
Original file line numberDiff line numberDiff line change
@@ -243,28 +243,6 @@ export async function updateObjectsAndObjectTags(
243243
}
244244
}
245245

246-
export function checkIfCanDownloadTar(objs, isSubfolder) {
247-
if (!objs) {
248-
return true;
249-
}
250-
else if (isSubfolder && objs.length === 1) {
251-
//no tar when single file in a subfolder
252-
return true;
253-
}
254-
else {
255-
//file or subfolder name max 99 chars, prefix max 154
256-
const pathOk = objs.every(
257-
(path) => {
258-
const elements = path.split("/");
259-
if (elements.find(el => el.length > 99)) return false;
260-
if (elements.length > 2 &&
261-
path.length - elements.slice(-1)[0].length > 154) return false;
262-
return true;
263-
});
264-
return pathOk;
265-
}
266-
}
267-
268246
export function timeout(ms) {
269247
return new Promise(resolve => setTimeout(resolve, ms));
270248
}

‎swift_browser_ui_frontend/src/components/CObjectTable.vue

+10-15
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ import {
4242
getPrefix,
4343
getPaginationOptions,
4444
checkIfItemIsLastOnPage,
45-
checkIfCanDownloadTar,
4645
addErrorToastOnMain,
4746
} from "@/common/globalFunctions";
4847
import {
@@ -418,20 +417,16 @@ export default {
418417
return obj.name.startsWith(object.name);
419418
})
420419
.map(item => item.name);
421-
const canDownload = checkIfCanDownloadTar(subfolderFiles, true);
422-
if (canDownload) {
423-
this.$store.state.socket.addDownload(
424-
this.$route.params.container,
425-
subfolderFiles,
426-
this.$route.params.owner ? this.$route.params.owner : "",
427-
).then(() => {
428-
if (DEV) console.log(`Started downloading subfolder ${object.name}`);
429-
}).catch(() => {
430-
addErrorToastOnMain(this.$t("message.download.error"));
431-
});
432-
} else {
433-
addErrorToastOnMain(this.$t("message.download.files"));
434-
}
420+
421+
this.$store.state.socket.addDownload(
422+
this.$route.params.container,
423+
subfolderFiles,
424+
this.$route.params.owner ? this.$route.params.owner : "",
425+
).then(() => {
426+
if (DEV) console.log(`Started downloading subfolder ${object.name}`);
427+
}).catch(() => {
428+
addErrorToastOnMain(this.$t("message.download.error"));
429+
});
435430
} else {
436431
this.$store.state.socket.addDownload(
437432
this.$route.params.container,

‎swift_browser_ui_frontend/src/components/ContainerTable.vue

+2-25
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ import {
4242
getPaginationOptions,
4343
toggleCopyFolderModal,
4444
checkIfItemIsLastOnPage,
45-
checkIfCanDownloadTar,
4645
addErrorToastOnMain,
4746
} from "@/common/globalFunctions";
4847
import {
@@ -280,8 +279,8 @@ export default {
280279
text: true,
281280
size: "small",
282281
title: this.$t("message.download.download"),
283-
onClick: async () => {
284-
await this.containerDownload(
282+
onClick: () => {
283+
this.beginDownload(
285284
item.name,
286285
item.owner ? item.owner : "",
287286
);
@@ -522,28 +521,6 @@ export default {
522521
addErrorToastOnMain(this.$t("message.download.error"));
523522
});
524523
},
525-
async containerDownload(containerName, owner) {
526-
let containerObjs = [];
527-
if (owner) containerObjs = await getObjects(
528-
this.active.id,
529-
containerName,
530-
"",
531-
this.abortController.signal,
532-
true,
533-
owner,
534-
);
535-
else containerObjs = await getObjects(this.active.id, containerName);
536-
537-
const containerObjNames = containerObjs.map(obj => obj.name);
538-
539-
const canDownload = checkIfCanDownloadTar(
540-
containerObjNames, false);
541-
if (canDownload) {
542-
this.beginDownload(containerName, owner);
543-
} else {
544-
addErrorToastOnMain(this.$t("message.download.files"));
545-
}
546-
},
547524
getEmptyText() {
548525
if (this.$route.name == "SharedFrom") {
549526
return this.$t("message.emptyProject.sharedFrom");

‎swift_browser_ui_frontend/wasm/Makefile

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ build/upworker.js: src/upinit.c src/streamingupload.c src/upcommon.c
1616
-s LLD_REPORT_UNDEFINED \
1717
-s FORCE_FILESYSTEM=1 \
1818
-s EXPORTED_RUNTIME_METHODS=ccall \
19-
-s EXPORTED_FUNCTIONS=_read_in_recv_keys_path,_wrap_chunk_content,_wrap_chunk_len,_rmrecv,_create_keypair,_create_session_key,_get_keypair_private_key,_create_crypt4gh_header,_free_chunk,_free_keypair,_encrypt_chunk,_libinit,_free_crypt4gh_session_key \
19+
-s EXPORTED_FUNCTIONS=_read_in_recv_keys_path,_wrap_chunk_content,_wrap_chunk_len,_rmrecv,_create_keypair,_create_session_key,_get_keypair_private_key,_create_crypt4gh_header,_free_chunk,_free_chunk_nobuf,_free_keypair,_encrypt_chunk,_libinit,_free_crypt4gh_session_key \
2020
-O3 \
2121
-o \
2222
$@ \
@@ -41,7 +41,7 @@ build/downworker.js: src/upinit.c src/streamingdownload.c src/upcommon.c
4141
-s LLD_REPORT_UNDEFINED \
4242
-s FORCE_FILESYSTEM=1 \
4343
-s EXPORTED_RUNTIME_METHODS=ccall \
44-
-s EXPORTED_FUNCTIONS=_read_in_recv_keys_path,_wrap_chunk_content,_wrap_chunk_len,_rmrecv,_create_keypair,_get_session_key_from_header,_get_keypair_public_key,_free_chunk,_free_keypair,_decrypt_chunk,_libinit,_free_crypt4gh_session_key \
44+
-s EXPORTED_FUNCTIONS=_read_in_recv_keys_path,_wrap_chunk_content,_wrap_chunk_len,_rmrecv,_create_keypair,_get_session_key_from_header,_get_keypair_public_key,_free_chunk,_free_chunk_nobuf,_free_keypair,_decrypt_chunk,_libinit,_free_crypt4gh_session_key \
4545
-O3 \
4646
-o \
4747
$@ \

‎swift_browser_ui_frontend/wasm/js/crypt-post-downworker.js

+51-56
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ function decryptChunk(container, path, enChunk) {
150150
let ret = new Uint8Array(HEAPU8.subarray(chunkPtr, chunkPtr + chunkLen));
151151
totalDone += chunkLen;
152152
Module.ccall(
153-
"free_chunk",
153+
"free_chunk_nobuf",
154154
"number",
155155
["number"],
156156
[chunk],
@@ -288,6 +288,9 @@ class FileSlicer {
288288
} else {
289289
// Otherwise queue to the streamController since we're using a
290290
// ServiceWorker for downloading
291+
while(this.output.desiredSize <= 0) {
292+
await timeout(10);
293+
}
291294
this.output.enqueue(decryptChunk(
292295
this.container,
293296
this.path,
@@ -307,7 +310,6 @@ class FileSlicer {
307310
["number"],
308311
[downloads[this.container].files[this.path].key],
309312
);
310-
311313
return true;
312314
}
313315
}
@@ -374,12 +376,13 @@ async function beginDownloadInSession(
374376
container,
375377
) {
376378
aborted = false; //reset with download start
377-
let fileStream = undefined;
378379

380+
let fileHandle = downloads[container].handle;
381+
let fileStream;
379382
if (downloads[container].direct) {
380-
fileStream = await downloads[container].handle.createWritable();
383+
fileStream = await fileHandle.createWritable();
381384
} else {
382-
fileStream = downloads[container].handle;
385+
fileStream = fileHandle;
383386
}
384387

385388
// Add the archive folder structure
@@ -390,20 +393,20 @@ async function beginDownloadInSession(
390393
.filter(path => path.length > 0) // remove empty paths (root level files)
391394
.sort((a, b) => a.length - b.length) // sort by path length as levels
392395
.reduce((unique, path) => { // strip paths down to just the unique ones
393-
let check = unique.find(item => item.join("/") === path.join("/"));
396+
let check = unique.find(item => item === path.join("/"));
394397
if (check === undefined) {
395-
unique.push(path);
398+
unique.push(path.join("/"));
396399
}
397400
return unique;
398401
}, []);
399402

400403
for (const path of folderPaths) {
401404
if (downloads[container].direct) {
402405
await fileStream.write(
403-
enc.encode(addTarFolder(path.slice(-1)[0], path.slice(0, -1).join("/"))),
406+
addTarFolder(path),
404407
);
405408
} else {
406-
fileStream.enqueue(enc.encode(addTarFolder(path.slice(-1)[0], path.slice(0, -1).join("/"))));
409+
fileStream.enqueue(addTarFolder(path));
407410
}
408411
}
409412
}
@@ -431,23 +434,15 @@ async function beginDownloadInSession(
431434
}
432435

433436
const response = await fetch(downloads[container].files[file].url);
437+
let path = file.replace(".c4gh", "");
434438

435439
if (downloads[container].archive) {
436440
const size = getFileSize(response, downloads[container].files[file].key);
437-
let path = file.split("/");
438-
let name = path.slice(-1)[0];
439-
let prefix;
440-
if (path.length > 1) {
441-
prefix = path.slice(0, -1).join("/");
442-
} else {
443-
prefix = "";
444-
}
445441

446-
let fileHeader = enc.encode(addTarFile(
447-
downloads[container].files[file].key != 0 ? name.replace(".c4gh", ""): name,
448-
prefix,
442+
let fileHeader = addTarFile(
443+
downloads[container].files[file].key != 0 ? path : file,
449444
size,
450-
));
445+
);
451446

452447
if (downloads[container].direct) {
453448
await fileStream.write(fileHeader);
@@ -476,47 +471,47 @@ async function beginDownloadInSession(
476471
return;
477472
}
478473
}
474+
}
479475

480-
if (downloads[container].archive) {
481-
// Write the end of the archive
482-
if (downloads[container].direct) {
483-
await fileStream.write(enc.encode("\x00".repeat(1024)));
484-
} else {
485-
fileStream.enqueue(enc.encode("\x00".repeat(1024)));
486-
}
487-
}
488-
489-
// Sync the file if downloading directly into file, otherwise finish
490-
// the fetch request.
476+
if (downloads[container].archive) {
477+
// Write the end of the archive
491478
if (downloads[container].direct) {
492-
await fileStream.close();
493-
// downloads[container].handle.flush();
494-
// downloads[container].handle.close();
479+
await fileStream.write(enc.encode("\x00".repeat(1024)));
495480
} else {
496-
fileStream.close();
481+
fileStream.enqueue(enc.encode("\x00".repeat(1024)));
497482
}
483+
}
498484

499-
if (downloads[container].direct) {
500-
// Direct downloads need no further action, the resulting archive is
501-
// already in the filesystem.
502-
postMessage({
503-
eventType: "finished",
504-
direct: true,
505-
container: container,
506-
});
507-
} else {
508-
// Inform download with service worker finished
509-
self.clients.matchAll().then(clients => {
510-
clients.forEach(client =>
511-
client.postMessage({
512-
eventType: "downloadProgressFinished",
513-
container: container,
514-
}));
515-
});
516-
}
517-
finishDownloadSession(container);
518-
return;
485+
// Sync the file if downloading directly into file, otherwise finish
486+
// the fetch request.
487+
if (downloads[container].direct) {
488+
await fileStream.close();
489+
// downloads[container].handle.flush();
490+
// downloads[container].handle.close();
491+
} else {
492+
fileStream.close();
493+
}
494+
495+
if (downloads[container].direct) {
496+
// Direct downloads need no further action, the resulting archive is
497+
// already in the filesystem.
498+
postMessage({
499+
eventType: "finished",
500+
direct: true,
501+
container: container,
502+
});
503+
} else {
504+
// Inform download with service worker finished
505+
self.clients.matchAll().then(clients => {
506+
clients.forEach(client =>
507+
client.postMessage({
508+
eventType: "downloadProgressFinished",
509+
container: container,
510+
}));
511+
});
519512
}
513+
finishDownloadSession(container);
514+
return;
520515
}
521516

522517
if (inServiceWorker) {

0 commit comments

Comments
 (0)