Skip to content

Download

loaders.download

Utility download function.

download_file 

download_file(
    url: str,
    fname: Union[str, Path],
    chunk_size: int = 1024,
    force_download: bool = True,
) -> None

Download a file with progress bar.

PARAMETER DESCRIPTION
url

URL to download.

TYPE: str

fname

File name.

TYPE: Union[str, Path]

chunk_size

Chunk size.

TYPE: int DEFAULT: 1024

force_download

Flag to force download even if file exists.

TYPE: bool DEFAULT: True

Source: https://gist.github.com/yanqd0/c13ed29e29432e3cf3e7c38467f42f51

Source code in srai/loaders/download.py 
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
def download_file(
    url: str, fname: Union[str, Path], chunk_size: int = 1024, force_download: bool = True
) -> None:
    """
    Download a file with progress bar.

    Args:
        url (str): URL to download.
        fname (Union[str, Path]): File name.
        chunk_size (int): Chunk size.
        force_download (bool): Flag to force download even if file exists.

    Source: https://gist.github.com/yanqd0/c13ed29e29432e3cf3e7c38467f42f51
    """
    if Path(fname).exists() and not force_download:
        warnings.warn("File exists. Skipping download.", stacklevel=1)
        return

    Path(fname).parent.mkdir(parents=True, exist_ok=True)
    resp = requests.get(
        url,
        headers={"User-Agent": "SRAI Python package (https://github.com/kraina-ai/srai)"},
        stream=True,
    )
    resp.raise_for_status()
    total = int(resp.headers.get("content-length", 0))
    with (
        open(fname, "wb") as file,
        tqdm(
            desc=Path(fname).name,
            total=total,
            unit="iB",
            unit_scale=True,
            unit_divisor=1024,
            disable=FORCE_TERMINAL,
        ) as bar,
    ):
        for data in resp.iter_content(chunk_size=chunk_size):
            size = file.write(data)
            bar.update(size)