Skip to content

Commit

Permalink
linting
Browse files Browse the repository at this point in the history
  • Loading branch information
stripuramallu3 committed Oct 22, 2023
1 parent 65cf832 commit 7193942
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 8 deletions.
1 change: 0 additions & 1 deletion mirageml/commands/utils/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

from custom_inputs import input_or_timeout

__all__ = [
Expand Down
13 changes: 6 additions & 7 deletions mirageml/commands/utils/web_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,13 @@
import requests
import typer
from bs4 import BeautifulSoup
from custom_inputs import input_or_timeout
from langchain.document_loaders import AsyncChromiumLoader
from langchain.document_transformers import BeautifulSoupTransformer
from rich.console import Console
from rich.live import Live
from rich.panel import Panel

from custom_inputs import input_or_timeout

logger = logging.getLogger(__name__)
logger.setLevel(logging.CRITICAL)
logging.basicConfig(level=logging.CRITICAL)
Expand Down Expand Up @@ -92,12 +91,12 @@ def get_all_links(base_url, url, live=None):
if cleaned_link.startswith(base_url): # Only yield child URLs
if live:
live.update(
Panel(
f"Scraping: {cleaned_link}",
title="[bold green]Scraper[/bold green]",
border_style="green",
Panel(
f"Scraping: {cleaned_link}",
title="[bold green]Scraper[/bold green]",
border_style="green",
)
)
)
yield cleaned_link
else:
for link in crawl_with_playwright(live, url):
Expand Down

0 comments on commit 7193942

Please sign in to comment.