Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

improvement of InformalNormalizer #219

Closed
wants to merge 2 commits into from
+93 −5
Closed

improvement of InformalNormalizer #219

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8734555
improvement of InformalNormalizer
riasati May 1, 2022
daf6c04
Merge branch 'development' of https://github.com/riasati/hazm into de…
riasati May 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 69 additions & 1 deletion hazm/InformalNormalizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ def analyzeVerbWord(word):

collectionOfVerbList = []

endVerbList = ["یم", "دم", "دیم", "ید", "دی", "دید", "ند", "دن", "دند", "ین", "دین", "ست", "ستم", "ستی", "ستیم", "ستید", "ستند", "م", "ی", "ه", "د", "ن"]
endVerbList = ["یم", "دم", "دیم", "ید", "دی", "دید", "ند", "دن", "دند", "ین", "دین", "ست", "ستم", "ستی", "ستیم", "ستید", "ستند", "ستن", "م", "ی", "ه", "د", "ن"]

for endVerb in endVerbList:
if word.endswith(endVerb):
Expand All @@ -247,6 +247,11 @@ def analyzeVerbWord(word):
"word": word[:-2],
"suffix": "ید"
})
elif endVerb == "ستن":
collectionOfVerbList.append({
"word": word[:-3],
"suffix": "ستند"
})
elif endVerb == "ن":
collectionOfVerbList.append({
"word": word[:-1],
Expand Down Expand Up @@ -351,13 +356,25 @@ def analyzeVerbWord(word):
if mainWord.startswith("‌") or mainWord.startswith("‎"):
mainWord = mainWord[1:]

mainWord2 = None
if mainWord.startswith("ا"):
mainWord2 = "آ" + mainWord[1:]

if mainWord in self.pastVerbs:
collectionOfVerbList[i]["word"] = self.pastVerbs[mainWord]
collectionOfRealVerbList.append(collectionOfVerbList[i])
if mainWord in self.presentVerbs:
collectionOfVerbList[i]["word"] = self.presentVerbs[mainWord]
collectionOfRealVerbList.append(collectionOfVerbList[i])

if mainWord2 != None and not (collectionOfVerbList[i]["preffix"] == "بربی" or collectionOfVerbList[i]["preffix"] == "بی"):
if mainWord2 in self.pastVerbs:
collectionOfVerbList[i]["word"] = self.pastVerbs[mainWord2]
collectionOfRealVerbList.append(collectionOfVerbList[i])
if mainWord2 in self.presentVerbs:
collectionOfVerbList[i]["word"] = self.presentVerbs[mainWord2]
collectionOfRealVerbList.append(collectionOfVerbList[i])

for i in range(len(collectionOfRealVerbList)):
preffix = collectionOfRealVerbList[i]["preffix"]
suffix = collectionOfRealVerbList[i]["suffix"]
Expand Down Expand Up @@ -472,10 +489,59 @@ def straightForwardResult(word):
straightForwardDic = {
"ب": ["به"],
"ک": ["که"],
"آن": ["آن"],
"می‌آید": ["می‌آید"],
"میاید": ["می‌آید"],
"می‌آیم": ["می‌آیم"],
"میایم": ["می‌آیم"],
"نمی‌آید": ["نمی‌آید"],
"نمیاید": ["نمی‌آید"],
"نمی‌آیم": ["نمی‌آیم"],
"نمیایم": ["نمی‌آیم"],
"برمی‌آید": ["برمی‌آید"],
"برمیاید": ["برمی‌آید"],
"برمی‌آیم": ["برمی‌آیم"],
"برمیایم": ["برمی‌آیم"],
"برنمی‌آید": ["برنمی‌آید"],
"برنمیاید": ["برنمی‌آید"],
"برنمی‌آیم": ["برنمی‌آیم"],
"برنمیایم": ["برنمی‌آیم"],
"منظوره": ["منظوره"],
"بدن": ["بدن"],
"میا": ["میا"],
"نیس": ["نیست"],
"فک": ["فکر"],
"برام": ["برایم"],
"آ": ["آ"],
"آی": ["آی"],
"این": ["این"],
"است": ["است"],
"ان": ["ان"],
"اند": ["اند"],
"میان": ["میان"],
"گردن": ["گردن"],
"اینهمه": ["اینهمه"],
"آنهمه": ["آنهمه"],
"الیه": ["الیه"],
"غرغره": ["غرغره"],
"لیله": ["لیله"],
"بزرگانه": ["بزرگانه"],
"پرستانه": ["پرستانه"],
"ام": ["ام"],
"بادی": ["بادی"],
"نان": ["نان"],
"باورم": ["باورم"],
"اوه": ["اوه"],
"چقد": ["چقدر"],
"چو": ["چون"],
"هس": ["هست"],
"اومدند": ["آمدند"],
"ش": ["اش"],
"بش": ["بهش"],
"ازت": ["از تو"],
"رو": ["را", "رو"],
"پایتون": ["پایتون"],
"اردن": ["اردن"],
"دست": ["دست"],
"دستی": ["دستی"],
"دستم": ["دستم"],
Expand Down Expand Up @@ -514,6 +580,8 @@ def straightForwardResult(word):
"بشین": ["بشین"],
"هارو": ["ها را"],
"مارو": ["ما را"],
"میاومد": ["می‌آمد"],
"می‌اومد": ["می‌آمد"],
"میخواسته": ["می‌خواسته"],
"می‌خواسته": ["می‌خواسته"],
"نمیخواسته": ["نمی‌خواسته"],
Expand Down
3 changes: 3 additions & 0 deletions hazm/WordTokenizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,9 @@ def join_verb_parts(self, tokens):
['خسته', 'شدید']
"""

if len(tokens) == 1:
return tokens

result = ['']
for token in reversed(tokens):
if token in self.before_verbs or (result[-1] in self.after_verbs and token in self.verbe):
Expand Down
6 changes: 5 additions & 1 deletion hazm/data/iverbs.dat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,8 @@
نشاند#نشان نشون 0
آورد#اور ار 0
خواست#خواه خاه 1
گذراند#گذران گذرون 0
خواست#خواه خا 1
گذراند#گذران گذرون 0
رساند#رسان رسون 0
فهماند#فهمان فهمون 0
آمد#آی آ 0
14 changes: 13 additions & 1 deletion hazm/data/iwords.dat
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
یه یک
دیگه دیگر
همدیگه همدیگر
اون آن
اگه اگر
اقای آقای
Expand Down Expand Up @@ -37,7 +38,7 @@
شمبه شنبه
شیش شش
صاحاب صاحب
صب صبر
صب صبح
کا کاه
کوجا کجا
کیلید کلید
Expand Down Expand Up @@ -71,6 +72,7 @@
دفه دفعه
سیریش سریش
کدوم کدام
هرکدوم هرکدام
کوچیک کوچک
مث مثل
مرتیکه مردکه
Expand Down Expand Up @@ -228,3 +230,13 @@
صریحن صریحاً
صراحتا صراحتاً
صراحتن صراحتاً
اومد آمد
جون جان
جونور جانور
اسلا اصلاً
اسلن اصلاً
پشیمون پشیمان
پردازی پردازی
چسپ چسب
وقط وقت
گرگینه گرگینه
5 changes: 3 additions & 2 deletions hazm/data/verbs.dat
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
#هست
آمد#آ
آموخت#آموز
آورد#آور
افتاد#افت
Expand Down Expand Up @@ -94,4 +93,6 @@
گشت#گرد
گشود#گشا
گفت#گو
یافت#یاب
یافت#یاب
چیند#چین
چسبید#چسب