Skip to content

Commit d935372

Browse files
authored
Fix download of large files in Console (#2773)
1 parent 22ec87d commit d935372

File tree

9 files changed

+174
-191
lines changed

9 files changed

+174
-191
lines changed

portal-ui/package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
"@mui/styles": "^5.12.0",
1414
"@mui/x-date-pickers": "^5.0.20",
1515
"@reduxjs/toolkit": "^1.9.5",
16+
"@types/streamsaver": "^2.0.1",
1617
"@uiw/react-textarea-code-editor": "^2.1.1",
1718
"kbar": "^0.1.0-beta.39",
1819
"local-storage-fallback": "^4.1.1",
@@ -31,6 +32,7 @@
3132
"react-window": "^1.8.9",
3233
"react-window-infinite-loader": "^1.0.9",
3334
"recharts": "^2.4.3",
35+
"streamsaver": "^2.0.6",
3436
"styled-components": "^5.3.10",
3537
"superagent": "^8.0.8",
3638
"tinycolor2": "^1.6.0",

portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjects.tsx

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -668,9 +668,7 @@ const ListObjects = () => {
668668
errorMessage: "",
669669
})
670670
);
671-
672671
storeFormDataWithID(ID, formData);
673-
storeCallForObjectWithID(ID, xhr);
674672
}
675673
});
676674
};

portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/VersionsNavigator.tsx

Lines changed: 3 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,10 @@ import {
3333
textStyleUtils,
3434
} from "../../../../Common/FormComponents/common/styleLibrary";
3535
import { IFileInfo } from "./types";
36-
import { download } from "../utils";
3736
import api from "../../../../../../common/api";
3837
import { ErrorResponseHandler } from "../../../../../../common/types";
3938

40-
import {
41-
decodeURLString,
42-
encodeURLString,
43-
niceBytesInt,
44-
} from "../../../../../../common/utils";
39+
import { decodeURLString, niceBytesInt } from "../../../../../../common/utils";
4540
import RestoreFileVersion from "./RestoreFileVersion";
4641

4742
import { AppState, useAppDispatch } from "../../../../../../store";
@@ -64,21 +59,13 @@ import {
6459
setErrorSnackMessage,
6560
} from "../../../../../../systemSlice";
6661
import {
67-
makeid,
68-
storeCallForObjectWithID,
69-
} from "../../../../ObjectBrowser/transferManager";
70-
import {
71-
cancelObjectInList,
72-
completeObject,
73-
failObject,
7462
setLoadingObjectInfo,
7563
setLoadingVersions,
76-
setNewObject,
7764
setSelectedVersion,
78-
updateProgress,
7965
} from "../../../../ObjectBrowser/objectBrowserSlice";
8066
import { List, ListRowProps } from "react-virtualized";
8167
import TooltipWrapper from "../../../../Common/TooltipWrapper/TooltipWrapper";
68+
import { downloadObject } from "../../../../ObjectBrowser/utils";
8269

8370
const styles = (theme: Theme) =>
8471
createStyles({
@@ -237,57 +224,6 @@ const VersionsNavigator = ({
237224
setPreviewOpen(false);
238225
};
239226

240-
const downloadObject = (object: IFileInfo) => {
241-
const identityDownload = encodeURLString(
242-
`${bucketName}-${object.name}-${new Date().getTime()}-${Math.random()}`
243-
);
244-
245-
const ID = makeid(8);
246-
247-
const downloadCall = download(
248-
bucketName,
249-
internalPaths,
250-
object.version_id,
251-
parseInt(object.size || "0"),
252-
null,
253-
ID,
254-
(progress) => {
255-
dispatch(
256-
updateProgress({
257-
instanceID: identityDownload,
258-
progress: progress,
259-
})
260-
);
261-
},
262-
() => {
263-
dispatch(completeObject(identityDownload));
264-
},
265-
(msg: string) => {
266-
dispatch(failObject({ instanceID: identityDownload, msg }));
267-
},
268-
() => {
269-
dispatch(cancelObjectInList(identityDownload));
270-
}
271-
);
272-
273-
storeCallForObjectWithID(ID, downloadCall);
274-
dispatch(
275-
setNewObject({
276-
ID,
277-
bucketName,
278-
done: false,
279-
instanceID: identityDownload,
280-
percentage: 0,
281-
prefix: object.name,
282-
type: "download",
283-
waitingForFile: true,
284-
failed: false,
285-
cancelled: false,
286-
errorMessage: "",
287-
})
288-
);
289-
};
290-
291227
const onShareItem = (item: IFileInfo) => {
292228
setObjectToShare(item);
293229
shareObject();
@@ -304,7 +240,7 @@ const VersionsNavigator = ({
304240
};
305241

306242
const onDownloadItem = (item: IFileInfo) => {
307-
downloadObject(item);
243+
downloadObject(dispatch, bucketName, internalPaths, item);
308244
};
309245

310246
const onGlobalClick = (item: IFileInfo) => {

portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/utils.ts

Lines changed: 139 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import { BucketObjectItem } from "./ListObjects/types";
1818
import { IAllowResources } from "../../../types";
1919
import { encodeURLString } from "../../../../../common/utils";
2020
import { removeTrace } from "../../../ObjectBrowser/transferManager";
21+
import streamSaver from "streamsaver";
2122
import store from "../../../../../store";
2223

2324
export const download = (
@@ -30,7 +31,8 @@ export const download = (
3031
progressCallback: (progress: number) => void,
3132
completeCallback: () => void,
3233
errorCallback: (msg: string) => void,
33-
abortCallback: () => void
34+
abortCallback: () => void,
35+
toastCallback: () => void
3436
) => {
3537
const anchor = document.createElement("a");
3638
document.body.appendChild(anchor);
@@ -48,75 +50,153 @@ export const download = (
4850
if (versionID) {
4951
path = path.concat(`&version_id=${versionID}`);
5052
}
51-
52-
var req = new XMLHttpRequest();
53-
req.open("GET", path, true);
54-
if (anonymousMode) {
55-
req.setRequestHeader("X-Anonymous", "1");
56-
}
57-
req.addEventListener(
58-
"progress",
59-
function (evt) {
60-
let percentComplete = Math.round((evt.loaded / fileSize) * 100);
61-
62-
if (progressCallback) {
63-
progressCallback(percentComplete);
64-
}
65-
},
66-
false
53+
return new DownloadHelper(
54+
path,
55+
id,
56+
anonymousMode,
57+
fileSize,
58+
progressCallback,
59+
completeCallback,
60+
errorCallback,
61+
abortCallback,
62+
toastCallback
6763
);
64+
};
6865

69-
req.responseType = "blob";
70-
req.onreadystatechange = () => {
71-
if (req.readyState === 4) {
72-
if (req.status === 200) {
73-
const rspHeader = req.getResponseHeader("Content-Disposition");
66+
class DownloadHelper {
67+
aborter: AbortController;
68+
path: string;
69+
id: string;
70+
filename: string = "";
71+
anonymousMode: boolean;
72+
fileSize: number = 0;
73+
writer: any = null;
74+
progressCallback: (progress: number) => void;
75+
completeCallback: () => void;
76+
errorCallback: (msg: string) => void;
77+
abortCallback: () => void;
78+
toastCallback: () => void;
79+
80+
constructor(
81+
path: string,
82+
id: string,
83+
anonymousMode: boolean,
84+
fileSize: number,
85+
progressCallback: (progress: number) => void,
86+
completeCallback: () => void,
87+
errorCallback: (msg: string) => void,
88+
abortCallback: () => void,
89+
toastCallback: () => void
90+
) {
91+
this.aborter = new AbortController();
92+
this.path = path;
93+
this.id = id;
94+
this.anonymousMode = anonymousMode;
95+
this.fileSize = fileSize;
96+
this.progressCallback = progressCallback;
97+
this.completeCallback = completeCallback;
98+
this.errorCallback = errorCallback;
99+
this.abortCallback = abortCallback;
100+
this.toastCallback = toastCallback;
101+
}
74102

75-
let filename = "download";
76-
if (rspHeader) {
77-
let rspHeaderDecoded = decodeURIComponent(rspHeader);
78-
filename = rspHeaderDecoded.split('"')[1];
79-
}
103+
abort(): void {
104+
this.aborter.abort();
105+
this.abortCallback();
106+
if (this.writer) {
107+
this.writer.abort();
108+
}
109+
}
80110

81-
if (completeCallback) {
82-
completeCallback();
83-
}
111+
send(): void {
112+
let isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
113+
if (isSafari) {
114+
this.toastCallback();
115+
this.downloadSafari();
116+
} else {
117+
this.download({
118+
url: this.path,
119+
chunkSize: 1024 * 1024 * 1024 * 1.5,
120+
});
121+
}
122+
}
84123

85-
removeTrace(id);
86-
87-
var link = document.createElement("a");
88-
link.href = window.URL.createObjectURL(req.response);
89-
link.download = filename;
90-
document.body.appendChild(link);
91-
link.click();
92-
document.body.removeChild(link);
93-
} else {
94-
if (req.getResponseHeader("Content-Type") === "application/json") {
95-
const rspBody: { detailedMessage?: string } = JSON.parse(
96-
req.response
97-
);
98-
if (rspBody.detailedMessage) {
99-
errorCallback(rspBody.detailedMessage);
100-
return;
101-
}
124+
async getRangeContent(url: string, start: number, end: number) {
125+
const info = this.getRequestInfo(start, end);
126+
const response = await fetch(url, info);
127+
if (response.ok && response.body) {
128+
if (!this.filename) {
129+
this.filename = this.getFilename(response);
130+
}
131+
if (!this.writer) {
132+
this.writer = streamSaver.createWriteStream(this.filename).getWriter();
133+
}
134+
const reader = response.body.getReader();
135+
let done, value;
136+
while (!done) {
137+
({ value, done } = await reader.read());
138+
if (done) {
139+
break;
102140
}
103-
errorCallback(`Unexpected response status code (${req.status}).`);
141+
await this.writer.write(value);
104142
}
143+
} else {
144+
throw new Error(`Unexpected response status code (${response.status}).`);
105145
}
106-
};
107-
req.onerror = () => {
108-
if (errorCallback) {
109-
errorCallback("A network error occurred.");
146+
}
147+
148+
getRequestInfo(start: number, end: number) {
149+
const info: RequestInit = {
150+
signal: this.aborter.signal,
151+
headers: { range: `bytes=${start}-${end}` },
152+
};
153+
if (this.anonymousMode) {
154+
info.headers = { ...info.headers, "X-Anonymous": "1" };
110155
}
111-
};
112-
req.onabort = () => {
113-
if (abortCallback) {
114-
abortCallback();
156+
return info;
157+
}
158+
159+
getFilename(response: Response) {
160+
const rspHeader = response.headers.get("Content-Disposition");
161+
if (rspHeader) {
162+
let rspHeaderDecoded = decodeURIComponent(rspHeader);
163+
return rspHeaderDecoded.split('"')[1];
115164
}
116-
};
165+
return "download";
166+
}
117167

118-
return req;
119-
};
168+
async download({ url, chunkSize }: any) {
169+
const numberOfChunks = Math.ceil(this.fileSize / chunkSize);
170+
this.progressCallback(0);
171+
try {
172+
for (let i = 0; i < numberOfChunks; i++) {
173+
let start = i * chunkSize;
174+
let end =
175+
i + 1 === numberOfChunks
176+
? this.fileSize - 1
177+
: (i + 1) * chunkSize - 1;
178+
await this.getRangeContent(url, start, end);
179+
let percentComplete = Math.round(((i + 1) / numberOfChunks) * 100);
180+
this.progressCallback(percentComplete);
181+
}
182+
this.writer.close();
183+
this.completeCallback();
184+
removeTrace(this.id);
185+
} catch (e: any) {
186+
this.errorCallback(e.message);
187+
}
188+
}
189+
190+
downloadSafari() {
191+
const link = document.createElement("a");
192+
link.href = this.path;
193+
document.body.appendChild(link);
194+
link.click();
195+
document.body.removeChild(link);
196+
this.completeCallback();
197+
removeTrace(this.id);
198+
}
199+
}
120200

121201
// Review file extension by name & returns the type of preview browser that can be used
122202
export const extensionPreview = (

0 commit comments

Comments
 (0)