Skip to content

Commit e3aebe9

Browse files
Added Progress bar
1 parent 4525d94 commit e3aebe9

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+28
-9
lines changed

.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
.vscode/
2+
__pycache__/
3+
images/

.vscode/settings.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"python.pythonPath": "/Library/Frameworks/Python.framework/Versions/3.8/bin/python3"
2+
"python.pythonPath": "C:\\Users\\Jelly\\AppData\\Local\\Programs\\Python\\Python38-32\\python.exe"
33
}

Downloader.py

+16-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import requests
22
import os
3+
from tqdm import tqdm
34

45
class Downloader:
56
urls = []
@@ -11,9 +12,20 @@ def download(self, folder):
1112
for link in self.urls:
1213
print("Downloading {}".format(link))
1314
r = requests.get(link, allow_redirects=True)
14-
if not os.path.exists(folder):
15-
os.mkdir(folder)
16-
15+
16+
if not os.path.exists("images"):
17+
os.mkdir("images")
18+
19+
if not os.path.exists("images/" + folder):
20+
os.mkdir("images/" + folder)
21+
1722
if link.find('/'):
1823
filename = (link.rsplit('/',1)[1])
19-
open("{}/{}".format(folder, filename), 'wb').write(r.content)
24+
total_size = int(r.headers.get('content_length', 0))
25+
block_size = 1024 # One 1kb
26+
t = tqdm(total_size, unit="iB", unit_scale=True)
27+
with open("images/{}/{}".format(folder, filename), 'wb') as f:
28+
for data in r.iter_content(block_size):
29+
t.update(len(data))
30+
f.write(data)
31+
t.close()

Scrape.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import requests
2-
import twitter
2+
from bs4 import BeautifulSoup
33

44

55
class Scrape:
@@ -11,7 +11,7 @@ def __init__(self, url):
1111
soup = BeautifulSoup(r.content, 'html.parser')
1212

1313
for images in soup.findAll("img"):
14-
if (link := images.get("src")) is not None: #make sure to install python3.8
14+
if (link := images.get("src")) is not None and "https://pbs.twimg.com/media" in link: #make sure to install python3.8 to use the walrus
1515
self.img_links.append(link)
1616

1717
def getImageLinks(self):

__pycache__/Downloader.cpython-38.pyc

319 Bytes
Binary file not shown.

__pycache__/Scrape.cpython-38.pyc

39 Bytes
Binary file not shown.

main.py

+6-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
from Scrape import Scrape
22
from Downloader import Downloader
33

4-
sc = Scrape("https://twitter.com/MasterDaye/media")
4+
5+
twitter_user = "MasterDaye"
6+
7+
sc = Scrape("https://twitter.com/{}/media".format(twitter_user))
58
dl = Downloader(sc.getImageLinks())
6-
dl.download("masterdaye")
9+
dl.download("masterdaye")
10+

masterdaye/1500x500

-125 KB
Binary file not shown.

masterdaye/1f324.png

-759 Bytes
Binary file not shown.

masterdaye/1f383.png

-1017 Bytes
Binary file not shown.

masterdaye/1f3b5.png

-408 Bytes
Binary file not shown.

masterdaye/1f426.png

-980 Bytes
Binary file not shown.

masterdaye/1f4e8.png

-837 Bytes
Binary file not shown.

masterdaye/1f603.png

-825 Bytes
Binary file not shown.

masterdaye/1f989.png

-1.12 KB
Binary file not shown.

masterdaye/2614.png

-1.03 KB
Binary file not shown.

masterdaye/270f.png

-647 Bytes
Binary file not shown.

masterdaye/ABLmj7Nm_400x400.jpg

-14.6 KB
Binary file not shown.

masterdaye/ABLmj7Nm_bigger.jpg

-2.9 KB
Binary file not shown.

masterdaye/ABLmj7Nm_normal.jpg

-2.06 KB
Binary file not shown.

masterdaye/D-GF7dvWkAE5akv.jpg

-96.7 KB
Binary file not shown.

masterdaye/D-GGCjTX4AEne0s.png

-112 KB
Binary file not shown.

masterdaye/D-GGFrjXoAALsfx.png

-218 KB
Binary file not shown.

masterdaye/D-GGJZYXUAAKI7H.jpg

-78.1 KB
Binary file not shown.

masterdaye/D-GY1tjXYAEiIWy.png

-26.3 KB
Binary file not shown.

masterdaye/D-QJWXeWkAAsZIF.jpg

-107 KB
Binary file not shown.

masterdaye/D-QJYXXXoAAkx4z.jpg

-91.3 KB
Binary file not shown.

masterdaye/D-QJalpXUAE3aSd.jpg

-101 KB
Binary file not shown.

masterdaye/D-QJgquXoAAUgG0.jpg

-120 KB
Binary file not shown.

masterdaye/D-fpSmGXsAg9vks.jpg

-88.9 KB
Binary file not shown.

masterdaye/D-fpUZWXsAojcRL.jpg

-88.7 KB
Binary file not shown.

masterdaye/D-rAfXNWkAAfGSI.jpg

-101 KB
Binary file not shown.

masterdaye/D-rAg2UXsAArP5L.jpg

-109 KB
Binary file not shown.

masterdaye/D-rAmLjWkAAi_QR.jpg

-147 KB
Binary file not shown.

masterdaye/D-rApL_WkAEOYF9.jpg

-65.6 KB
Binary file not shown.

masterdaye/EBATp2gWkAA7Ot_.jpg

-160 KB
Binary file not shown.

masterdaye/EBOkvZtXUAEl-EY.jpg

-139 KB
Binary file not shown.

masterdaye/EBTYZkRWkAAvPxC.jpg

-89.6 KB
Binary file not shown.

masterdaye/EBess81XUAESCZ3.jpg

-74.5 KB
Binary file not shown.

masterdaye/EBess82XUAc4VZ-.jpg

-100 KB
Binary file not shown.

masterdaye/EBess8zWwAEhrG9.jpg

-109 KB
Binary file not shown.

masterdaye/EBess8zXoAsj966.jpg

-139 KB
Binary file not shown.

masterdaye/EBohqyRXkAAjes3.jpg

-102 KB
Binary file not shown.

masterdaye/EBohqyTXsAQSdd7.jpg

-92.4 KB
Binary file not shown.

masterdaye/ECwVSAaX4AE4-Kx.png

-266 KB
Binary file not shown.

masterdaye/ECwVSAbXUAAXN0P.png

-208 KB
Binary file not shown.

masterdaye/ECwVSAhWwAEazrP.png

-197 KB
Binary file not shown.

masterdaye/EDk9dg1XYAA58QG.jpg

-102 KB
Binary file not shown.

masterdaye/EDk9dg2WwAEka4c.png

-74.2 KB
Binary file not shown.

masterdaye/EDk9dgzXkAAcLPG.jpg

-127 KB
Binary file not shown.

masterdaye/EDk9dgzXoAEvFGU.png

-49.1 KB
Binary file not shown.

masterdaye/EEISHXWUUAAkUVm.jpg

-101 KB
Binary file not shown.

masterdaye/EEISI2IU8AIi8Bc.jpg

-102 KB
Binary file not shown.

masterdaye/EEhx_vmXoAA2OYO.jpg

-29.6 KB
Binary file not shown.

masterdaye/EFfHBxiWkAAXA0d.jpg

-56.2 KB
Binary file not shown.

masterdaye/EGIZ8AKWwAAE3W_.jpg

-92 KB
Binary file not shown.

masterdaye/EGIZ8AKXkAIo9CD.jpg

-72.7 KB
Binary file not shown.

masterdaye/EGIZ8AOXoAIiOax.jpg

-135 KB
Binary file not shown.

masterdaye/EGKL10BWoAEWzK6.jpg

-91.9 KB
Binary file not shown.

masterdaye/EIPF4w0WwAIvckZ.png

-247 KB
Binary file not shown.

0 commit comments

Comments
 (0)