Skip to content

Commit

Permalink
-Created __init__.py
Browse files Browse the repository at this point in the history
-Plugins that crash now have fails reported and logged as info
-Added plugin geonode.py
-Removed proxy test method
  • Loading branch information
Themis3000 committed Aug 21, 2021
1 parent 29cf641 commit b5b58b0
Show file tree
Hide file tree
Showing 4 changed files with 39 additions and 7 deletions.
5 changes: 3 additions & 2 deletions src/FreeProxyScraper.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import traceback
import logging
from src.utils import Plugins, GenLimiter, Proxy
from typing import Iterator

Expand All @@ -20,7 +20,8 @@ def exec_iter_plugin(self, method_name: str, sort_asc_fails: bool = True, *args,
for value in return_iter:
yield value
except Exception:
traceback.print_exc()
logging.info(f"FreeProxyScraper plugin \"{plugin.plugin_name}\" has crashed")
plugin.report_fail()
continue

def find_proxies(self, limit: int = -1) -> Iterator[Proxy]:
Expand Down
Empty file added src/__init__.py
Empty file.
36 changes: 36 additions & 0 deletions src/plugins/geonode.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import requests
from src.utils import Proxy, Plugin
from typing import Iterator


anon_dict = {"transparent": 0, "anonymous": 1, "elite": 2}


class GeoNode(Plugin):
plugin_name = "Geonode"
plugin_url = "https://geonode.com/free-proxy-list"

def find(self) -> Iterator[Proxy]:
page = 1
while True:
resource = f"https://proxylist.geonode.com/api/proxy-list?limit=15&page={page}&sort_by=lastChecked&sort_type=desc"
response = requests.get(resource)

if response.status_code != 200:
self.report_fail()
return

response_json = response.json()

if len(response_json["data"]) == 0:
return

for proxy_json in response_json["data"]:
yield Proxy(
ip=proxy_json["ip"],
anon_level=anon_dict[proxy_json["anonymityLevel"]],
port=int(proxy_json["port"]),
protocol=proxy_json["protocols"][0]
)

page += 1
5 changes: 0 additions & 5 deletions src/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,11 +89,6 @@ def address(self) -> str:
"""Gets the full form address for the proxy used for connecting"""
return f"{self.protocol}://{self.ip}:{self.port}"

def test(self) -> bool:
"""Tests the proxy to see if it's operational"""
# todo implement proxy test method
return True


class Plugin(ABC):
"""Represents a plugin"""
Expand Down

0 comments on commit b5b58b0

Please sign in to comment.