Skip to content

Add retry=3 to download() #7313

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Apr 6, 2022
Merged

Add retry=3 to download() #7313

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
62087ad
Add `retry=3` to `download()`
glenn-jocher Apr 6, 2022
924b036
Update general.py
glenn-jocher Apr 6, 2022
0dfb857
Update general.py
glenn-jocher Apr 6, 2022
d88f4bf
Update general.py
glenn-jocher Apr 6, 2022
d26f9ee
Update VOC.yaml
glenn-jocher Apr 6, 2022
d385cae
Update VisDrone.yaml
glenn-jocher Apr 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion data/VOC.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ download: |
urls = [url + 'VOCtrainval_06-Nov-2007.zip', # 446MB, 5012 images
url + 'VOCtest_06-Nov-2007.zip', # 438MB, 4953 images
url + 'VOCtrainval_11-May-2012.zip'] # 1.95GB, 17126 images
download(urls, dir=dir / 'images', delete=False, threads=3)
download(urls, dir=dir / 'images', delete=False, curl=True, threads=3)

# Convert
path = dir / f'images/VOCdevkit'
Expand Down
2 changes: 1 addition & 1 deletion data/VisDrone.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ download: |
'https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-val.zip',
'https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-test-dev.zip',
'https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-test-challenge.zip']
download(urls, dir=dir, threads=4)
download(urls, dir=dir, curl=True, threads=4)

# Convert
for d in 'VisDrone2019-DET-train', 'VisDrone2019-DET-val', 'VisDrone2019-DET-test-dev':
Expand Down
24 changes: 18 additions & 6 deletions utils/general.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,20 +497,32 @@ def url2file(url):
return file


def download(url, dir='.', unzip=True, delete=True, curl=False, threads=1):
def download(url, dir='.', unzip=True, delete=True, curl=False, threads=1, retry=3):
# Multi-threaded file download and unzip function, used in data.yaml for autodownload
def download_one(url, dir):
# Download 1 file
success = True
f = dir / Path(url).name # filename
if Path(url).is_file(): # exists in current path
Path(url).rename(f) # move to dir
elif not f.exists():
LOGGER.info(f'Downloading {url} to {f}...')
if curl:
os.system(f"curl -L '{url}' -o '{f}' --retry 9 -C -") # curl download, retry and resume on fail
else:
torch.hub.download_url_to_file(url, f, progress=threads == 1) # torch download
if unzip and f.suffix in ('.zip', '.gz'):
for i in range(retry + 1):
if curl:
s = 'sS' if threads > 1 else '' # silent
r = os.system(f"curl -{s}L '{url}' -o '{f}' --retry 9 -C -") # curl download
success = r == 0
else:
torch.hub.download_url_to_file(url, f, progress=threads == 1) # torch download
success = f.is_file()
if success:
break
elif i < retry:
LOGGER.warning(f'Download failure, retrying {i + 1}/{retry} {url}...')
else:
LOGGER.warning(f'Failed to download {url}...')

if unzip and success and f.suffix in ('.zip', '.gz'):
LOGGER.info(f'Unzipping {f}...')
if f.suffix == '.zip':
ZipFile(f).extractall(path=dir) # unzip
Expand Down