forked from jamesnicolas/yomichan-forvo-server
-
Notifications
You must be signed in to change notification settings - Fork 4
/
__init__.py
223 lines (192 loc) · 8.69 KB
/
__init__.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
import http.server
import socketserver
import requests
import re
import json
import base64
import threading
from http import HTTPStatus
from bs4 import BeautifulSoup
from urllib.parse import urlparse
from urllib.parse import parse_qs
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
class Forvo():
"""
Forvo web-scraper utility class that matches YomiChan's expected output for a custom audio source
"""
_SERVER_HOST = "https://forvo.com"
_AUDIO_HTTP_HOST = "https://audio00.forvo.com"
def __init__(self, language):
self.language = language
self._set_session()
def _set_session(self):
"""
Sets the session with basic backoff retries.
Put in a separate function so we can try resetting the session if something goes wrong
"""
retry_strategy = Retry(
total=3,
backoff_factor=1,
status_forcelist=[429, 500, 502, 503, 504],
method_whitelist=["HEAD", "GET", "OPTIONS"]
)
adapter = HTTPAdapter(max_retries=retry_strategy)
self.session = requests.Session()
self.session.mount("https://", adapter)
self.session.mount("http://", adapter)
# Use my personal user agent to try to avoid scraping detection
self.session.headers.update(
{
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.72 Safari/537.36 Edg/89.0.774.45",
"Accept-Language": "en-US,en;q=0.5",
}
)
def _get(self, path):
"""
Makes a GET request assuming base url. Creates a new session if something goes wrong
"""
url = self._SERVER_HOST + path
try:
return self.session.get(url, timeout=10).text
except Exception:
self._set_session()
return self.session.get(url, timeout=10).text
def word(self, w):
"""
Scrape forvo's word page for audio sources
"""
w = w.strip()
if len(w) == 0:
return []
path = f"/word/{w}/"
html = self._get(path)
soup = BeautifulSoup(html, features="html.parser")
# Forvo's word page returns multiple result sets grouped by langauge like:
# <div id="language-container-zh">
# <article>
# <ul class="show-all-pronunciations">
# <li>
# <span class="play" onclick"(some javascript to play the word audio)"></span>
# "Pronunciation by <span><a href="/username/link">skent</a></span>"
# <div class="more">...</div>
# </li>
# </ul>
# ...
# </article>
# <article id="extra-word-info-76">...</article>
# </ul>
# We also filter out ads
results = soup.select('#language-container-zh>article>ul.pronunciations-list>li:not(.li-ad)')
audio_sources = []
for i in results:
url = self._extract_url(i.div)
# Capture the username of the user
# Some users have deleted accounts which is why can't just parse it from the <a> tag
username = re.search(r"Pronunciation by([^(]+)\(",i.get_text(strip=True)).group(1).strip()
audio_sources.append({"name":f"Forvo ({username})","url":url})
return audio_sources
@classmethod
def _extract_url(cls, element):
play = element['onclick']
# We are interested in Forvo's javascript Play function which takes in some parameters to play the audio
# Example: Play(786514,'OTA3Mjk2Ny83Ni85MDcyOTY3Xzc2XzExNDk0NzNfMS5tcDM=',...);return false;
# Match anything that isn't commas, parentheses or quotes to capture the function arguments
# Regex will match something like ["Play","786514","OTA3Mjk2Ny83Ni85MDcyOTY3Xzc2XzExNDk0NzNfMS5tcDM=", ...]
play_args = re.findall(r"([^',\(\)]+)", play)
# It seems that forvo has two locations for mp3, /audios/mp3 and just /mp3. I don't know what the difference
# is so I'm just going to use the /mp3 version, which is the second argument in Play() base64 encoded
file = base64.b64decode(play_args[2]).decode("utf-8")
url = f"{cls._AUDIO_HTTP_HOST}/mp3/{file}"
return url
def search(self, s):
"""
Scrape Forvo's search page for audio sources. Note that the search page omits the username
"""
s = s.strip()
if len(s) == 0:
return []
path = f"/search/{s}/{self.language}/"
html = self._get(path)
soup = BeautifulSoup(html, features="html.parser")
# Forvo's search page returns two result sets like:
# <ul class="word-play-list-icon-size-l">
# <li><span class="play" onclick"(some javascript to play the word audio)"></li>
# </ul>
results = soup.select('ul.word-play-list-icon-size-l>li>div.play')
audio_sources = []
for i in results:
url = self._extract_url(i)
audio_sources.append({"name":"Forvo Search","url":url})
return audio_sources
class ForvoHandler(http.server.SimpleHTTPRequestHandler):
forvo = Forvo('zh')
# By default, SimpleHTTPRequestHandler logs to stderr
# This would cause Anki to show an error, even on successful requests
# log_error is still a useful function though, so replace it with the inherited log_message
# Make log_message do nothing
def log_error(self, *args, **kwargs):
super().log_message(*args, **kwargs)
def log_message(self, *args):
pass
def do_GET(self):
# Extract 'term' and 'reading' query parameters
query_components = parse_qs(urlparse(self.path).query)
term = query_components["term"][0] if "term" in query_components else ""
# Yomichan used to use "expression" but renamed to term. Still support "expression" for older versions
expression = query_components["expression"][0] if "expression" in query_components else ""
if term == "":
term = expression
reading = query_components["reading"][0] if "reading" in query_components else ""
debug = query_components["debug"][0] if "debug" in query_components else False
if debug:
debug_resp = {
"debug":True
}
debug_resp['reading'] = reading
debug_resp['term'] = term
debug_resp['word.term'] = self.forvo.word(term)
debug_resp['word.reading'] = self.forvo.word(reading)
debug_resp['search.term'] = self.forvo.search(term)
debug_resp['search.reading'] = self.forvo.search(reading)
self.wfile.write(bytes(json.dumps(debug_resp), "utf8"))
return
audio_sources = []
# Try looking for word sources for 'term' first
audio_sources = self.forvo.word(term)
# Try looking for word sources for 'reading'
if len(audio_sources) == 0:
audio_sources += self.forvo.word(reading)
# Finally use forvo search to look for similar words
if len(audio_sources) == 0:
audio_sources += self.forvo.search(term)
if len(audio_sources) == 0:
audio_sources += self.forvo.search(reading)
# Build JSON that yomichan requires
# Ref: https://github.com/FooSoft/yomichan/blob/master/ext/data/schemas/custom-audio-list-schema.json
resp = {
"type": "audioSourceList",
"audioSources": audio_sources
}
# Writing the JSON contents with UTF-8
payload = bytes(json.dumps(resp), "utf8")
self.send_response(HTTPStatus.OK)
self.send_header("Content-type", "application/json")
self.send_header("Content-length", str(len(payload)))
self.end_headers()
try:
self.wfile.write(payload)
except BrokenPipeError:
self.log_error("BrokenPipe when sending reply")
return
if __name__ == "__main__":
# If we're not in Anki, run the server directly and blocking for easier debugging
print("Running in debug mode...")
httpd = socketserver.TCPServer(('localhost', 8770), ForvoHandler)
httpd.serve_forever()
else:
# Else, run it in a separate thread so it doesn't block
httpd = http.server.ThreadingHTTPServer(('localhost', 8770), ForvoHandler)
server_thread = threading.Thread(target=httpd.serve_forever)
server_thread.daemon = True
server_thread.start()