-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathbulk-downloader.py
89 lines (81 loc) · 3.21 KB
/
bulk-downloader.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
import requests
import sys, getopt, os
import time
import datetime
CHUNK_SIZE = 1024
MB_SIZE = 1048576
links = None
outputdir = None
def main():
try:
opts, args = getopt.getopt(sys.argv[1:],"hf:o:",["file=","outdir="])
except getopt.GetoptError:
print('usage: bulk-downloader.py -f <link.txt> -o <output_dir>')
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('usage: bulk-downloader.py -f <link.txt> -o <output_dir>')
sys.exit()
elif opt in ("-f", "--file"):
links = arg
elif opt in ("-o", "--outdir"):
outputdir = arg
if links is None:
print('Missing links.txt parameter.')
sys.exit(2)
if outputdir is None:
print('Missing output_dir parameter.')
sys.exit(2)
print('Output dir: ' + outputdir)
if not os.path.exists(outputdir):
print(outputdir + " does not exists... creating...")
os.makedirs(outputdir)
print(outputdir + " created!")
print('Opening ' + links + "...")
with open(links) as links_file:
for url in links_file.readlines():
url = url.replace('\n', '')
last_slash_index = url.rindex('/')
file_name = url[last_slash_index+1 : len(url)]
res = requests.get(url, stream=True)
total_length = res.headers.get('content-length')
print("downloading " + file_name)
dl = 0
total_length = int(total_length)
loops = 0
speeds = 0
with open(outputdir + "/" + file_name, 'wb') as file:
total_length_mb = total_length / MB_SIZE
start_time = time.mktime(time.localtime())
for chunk in res.iter_content(CHUNK_SIZE):
file.write(chunk)
elapsed_time = time.mktime(time.localtime()) - start_time
if elapsed_time == 0:
elapsed_time = 1
dl = dl + len(chunk)
done = int(25 * dl / total_length)
total_mb_downloaded = float(dl / MB_SIZE)
remaining_size = total_length_mb - total_mb_downloaded
speed = float(total_mb_downloaded / elapsed_time)
speeds = speeds + speed;
loops = loops + 1
sys.stdout.write('\r[%s%s] %.2f Mb of %.2f Mb %.2f Mb/s ETA: %s' %
(
'=' * done, ' ' * (25-done),
total_mb_downloaded,
float(total_length_mb),
speed,
str(datetime.timedelta(seconds=int(remaining_size/speed)))
)
)
sys.stdout.flush()
sys.stdout.write("\n")
sys.stdout.write("\n")
sys.stdout.flush()
print("Elapsed time: %s, Avg Speed: %.2f Mb/s" %
(
str(datetime.timedelta(seconds= elapsed_time)), float(speeds/loops))
)
print(file_name + " saved to " + outputdir + " folder")
if __name__ == "__main__":
main()