The second time to find proxies with default judges, providers.

[fygul]

import asyncio
from proxybroker import Broker

async def find(proxies, loop):
    broker = Broker(queue=proxies,
                    # If I comment judges or/and providers, some errors occur at 2nd execution of get_proxies()
                    judges=['https://httpheader.net/', 'http://httpheader.net/'],
                    providers=['http://www.proxylists.net/', 'http://fineproxy.org/eng/'],
                    loop=loop)

    types = [('HTTP', ('High', ))]
    limit = 10

    await broker.find(types=types, limit=limit)


def get_proxies():
    loop = asyncio.get_event_loop()
    proxies = asyncio.Queue(loop=loop)
    loop.run_until_complete(find(proxies, loop))
    proxy_list = []
    while True:
        proxy = proxies.get_nowait()
        if proxy is None:
            break
        p = 'http://{}:{}'.format(proxy.host, proxy.port)
        if p not in proxy_list:
            proxy_list.append(p)
    print(proxy_list)
    return proxy_list

if __name__ == '__main__':
    plist = get_proxies()

    # Get some web pages with proxies. Some proxies might be blocked by web servers after mass connections.
    # Keep a record of blocked proxies here, avoid to use them for several moments.
    # However, if the number of available proxies was declined too much, retrieve some new proxies.

    plist = get_proxies()  # Some errors occur if I comment judges or/and providers.

    # Other tasks ...

If I comment judges/providers, something happens:

D:\Temp\proxybroker\venv\Scripts\python.exe D:/Temp/proxybroker/renew_proxy.py
['http://198.62.109.139:3128', 'http://117.135.251.132:80', 'http://212.224.70.124:8080', 'http://108.59.10.138:55555', 'http://120.26.213.55:9999', 'http://108.59.10.129:55555', 'http://108.59.10.135:55555', 'http://216.220.165.62:8080', 'http://51.254.103.206:3128', 'http://138.36.27.5:80']
Traceback (most recent call last):
File "D:/Temp/proxybroker/renew_proxy.py", line 90, in
plist = get_proxies() # Some errors occur if I comment judges or/and providers.
File "D:/Temp/proxybroker/renew_proxy.py", line 71, in get_proxies
loop.run_until_complete(find(proxies, loop))
File "D:\ProgramFiles\Python35\lib\asyncio\base_events.py", line 337, in run_until_complete
return future.result()
File "D:\ProgramFiles\Python35\lib\asyncio\futures.py", line 274, in result
raise self._exception
File "D:\ProgramFiles\Python35\lib\asyncio\tasks.py", line 241, in _step
result = coro.throw(exc)
File "D:/Temp/proxybroker/renew_proxy.py", line 65, in find
await broker.find(types=types, limit=limit)
File "D:\Temp\proxybroker\venv\lib\site-packages\proxybroker\api.py", line 108, in find
await self._run(self._checker.check_judges(), action)
File "D:\Temp\proxybroker\venv\lib\site-packages\proxybroker\api.py", line 114, in _run
await tasks
File "D:\ProgramFiles\Python35\lib\asyncio\futures.py", line 358, in iter
yield self # This tells Task to wait for completion.
File "D:\ProgramFiles\Python35\lib\asyncio\tasks.py", line 290, in _wakeup
future.result()
File "D:\ProgramFiles\Python35\lib\asyncio\futures.py", line 274, in result
raise self._exception
File "D:\ProgramFiles\Python35\lib\asyncio\tasks.py", line 241, in _step
result = coro.throw(exc)
File "D:\Temp\proxybroker\venv\lib\site-packages\proxybroker\checker.py", line 26, in check_judges
await asyncio.gather(*[j.check() for j in self._judges])
File "D:\ProgramFiles\Python35\lib\asyncio\futures.py", line 358, in iter
yield self # This tells Task to wait for completion.
File "D:\ProgramFiles\Python35\lib\asyncio\tasks.py", line 290, in _wakeup
future.result()
File "D:\ProgramFiles\Python35\lib\asyncio\futures.py", line 274, in result
raise self._exception
File "D:\ProgramFiles\Python35\lib\asyncio\tasks.py", line 239, in _step
result = coro.send(None)
File "D:\Temp\proxybroker\venv\lib\site-packages\proxybroker\judge.py", line 61, in check
allow_redirects=False) as resp:
File "D:\Temp\proxybroker\venv\lib\site-packages\aiohttp\client.py", line 538, in aenter
self._resp = yield from self._coro
File "D:\Temp\proxybroker\venv\lib\site-packages\aiohttp\client.py", line 150, in _request
raise RuntimeError('Session is closed')
RuntimeError: Session is closed

Process finished with exit code 1

[constverum]

Fixed another bug thanks to your report @fygul! =)
This fix is included in release 0.1.4. Please update your version.

[fygul]

Awesome! 👍
Thanks.