Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Vincenzo new new #45

Open
wants to merge 12 commits into
base: dev7
Choose a base branch
from
Open

Vincenzo new new #45

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
4c865b7
asd
vncz14 Mar 21, 2023
ccb20f9
add_characters toggle for initialPopulation
vncz14 Mar 21, 2023
baf6902
aaaa
vncz14 Mar 21, 2023
47abbf3
aaaaaaaaaaaaa
vncz14 May 9, 2023
e2f07b4
final version that matches docs
vncz14 May 11, 2023
3d2d1fc
Merge branch 'voting-new' into vincenzo-new-new
vncz14 May 11, 2023
16329be
turns out you can give more requests to the api than i thought
vncz14 May 12, 2023
1e7aa6f
Merge branch 'vincenzo-new-new' of https://github.com/sitechtimes/ani…
vncz14 May 12, 2023
8608dda
improved school appropriate filtering
vncz14 May 22, 2023
e26d5f2
resolve conflict
vncz14 May 22, 2023
9bfb56d
bug fix for inappropriate genres
vncz14 May 25, 2023
6fc0847
Update db_pop.py
harveyj123 May 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 72 additions & 73 deletions anime/scripts/db_pop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ def __init__(self):
self.our_airing_anime = set()
self.their_airing_anime = set()
self.update_characters_anime = set()
self.inappropriate_ratings = ["Rx - Hentai", "R+ - Mild Nudity"]
self.inappropriate_genres = ["Ecchi", "Erotica", "Hentai",]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

:(

self.response = None

def requestAPI(self, api_url):
Expand All @@ -34,10 +36,18 @@ def addAnime(self, anime_instance: dict):

# if the rating is not school appropriate
# move on to the next anime, don't add it
if anime_instance["rating"] == "Rx - Hentai" or anime_instance["rating"] == "R+ - Mild Nudity":
if anime_instance["rating"] in self.inappropriate_ratings:
print(f"not school appropriate: {my_anime_name}")
return

# create a list of the names of the anime's genres
genre_list = []
for genre in anime_instance["genres"]:
if genre["name"] in self.inappropriate_genres:
print(f"not school appropriate: {my_anime_name}")
return
genre_list.append(genre["name"])

# if the anime is not popular enough
# move on to the next anime, don't add it
if anime_instance["members"] < 5000:
Expand All @@ -47,8 +57,7 @@ def addAnime(self, anime_instance: dict):
# get the date where the anime started airing, if any
date = anime_instance["aired"]["prop"]["from"]
if date["day"] is not None:
my_from_date = datetime.date(
date["year"], date["month"], date["day"])
my_from_date = datetime.date(date["year"], date["month"], date["day"])
else:
my_from_date = None

Expand All @@ -58,13 +67,13 @@ def addAnime(self, anime_instance: dict):
my_month = my_from_date.month

match my_month:
case 1 | 2 | 3:
case 1|2|3:
my_time_of_year = "Winter"
case 4 | 5 | 6:
case 4|5|6:
my_time_of_year = "Spring"
case 7 | 8 | 9:
case 7|8|9:
my_time_of_year = "Summer"
case 10 | 11 | 12:
case 10|11|12:
my_time_of_year = "Fall"
case None:
my_time_of_year = ""
Expand All @@ -75,21 +84,19 @@ def addAnime(self, anime_instance: dict):
else:
my_season = None


# get the date where the anime stopped airing, if any
date = anime_instance["aired"]["prop"]["to"]
if date["day"] is not None:
my_to_date = datetime.date(
date["year"], date["month"], date["day"])
my_to_date = datetime.date(date["year"], date["month"], date["day"])
else:
my_to_date = None

except Exception as err:
raise Exception(
f"something in the special anime attributes didnt work: {err}")
raise Exception(f"something in the special anime attributes didnt work: {err}")

try:
# try finding a django Anime with the MAL id of the anime
my_anime = Anime.objects.get(mal_id=anime_instance["mal_id"])
my_anime = Anime.objects.get(mal_id=anime_instance["mal_id"]) # try finding a django Anime with the MAL id of the anime

# if it does exist, update all of its attributes to what the api says
my_anime.mal_id = anime_instance["mal_id"]
Expand Down Expand Up @@ -129,19 +136,13 @@ def addAnime(self, anime_instance: dict):
)
my_anime.save()

# create a list of the names of the anime's genres
genre_list = []
for genre in anime_instance["genres"]:
genre_list.append(genre["name"])

# for every name in the genre list
for genre_name in genre_list:
try:
# try and find a Genre object that matches the genre name
Genre.objects.get(genre=genre_name)
# if it does exist just print that it already exists
print(f"genre already exists: {genre_name}")
except Genre.DoesNotExist: # if it does not exist
Genre.objects.get(genre=genre_name) # try and find a Genre object that matches the genre name
print(f"genre already exists: {genre_name}") # if it does exist just print that it already exists
except Genre.DoesNotExist: # if it does not exist
# create a Genre object with that name and print that it was created
my_genre = Genre(
genre=genre_name
Expand All @@ -161,11 +162,9 @@ def addAnime(self, anime_instance: dict):
# for every name in the studio list
for studio_name in studio_list:
try:
# try and find a Studio object that matches the studio name
Studio.objects.get(studio=studio_name)
# if it does exist, just print that it already exists
print(f"studio already exists: {studio_name}")
except Studio.DoesNotExist: # if it does not exist
Studio.objects.get(studio=studio_name) # try and find a Studio object that matches the studio name
print(f"studio already exists: {studio_name}") # if it does exist, just print that it already exists
except Studio.DoesNotExist: # if it does not exist
# create a Studio object with that name and print that it was created
my_studio = Studio(
studio=studio_name
Expand All @@ -177,22 +176,23 @@ def addAnime(self, anime_instance: dict):
my_anime.anime_studio.add(Studio.objects.get(studio=studio_name))
my_anime.save()

def initialPopulation(self, pages: int = 5, min_characters: int = 10, min_side_characters: int = 5, add_characters: bool = True):
def initialPopulation(self, pages: int = 5, min_characters: int = 10, min_side_characters: int = 5, add_characters: bool = False):

for page_num in range(1, pages+1):
for page_num in range(1,pages+1):

api_url = f"{self.base_top_api_url}&page={page_num}"
self.requestAPI(api_url)

for instance in self.response["data"]: # instance is the anime
for instance in self.response["data"]: # instance is the anime
self.addAnime(instance)
time.sleep(4)

time.sleep(1)

if add_characters is True:
self.noCharacterAnime(min_characters, min_side_characters)

def updateAiringAnime(self, min_characters: int = 10, min_side_characters: int = 5,):

def updateAiringAnime(self, min_characters: int = 10, min_side_characters: int = 5, add_characters: bool = False):

# get all the anime that we have that are airing
try:
Expand All @@ -201,69 +201,61 @@ def updateAiringAnime(self, min_characters: int = 10, min_side_characters: int =
self.our_airing_anime.add(anime.mal_id)
print(f"our_airing_anime: {self.our_airing_anime}")
except Anime.DoesNotExist:
# if there are none then just say so
print("our_airing_anime: none")
print("our_airing_anime: none") # if there are none then just say so

api_url = f"{self.base_airing_api_url}"
self.requestAPI(api_url)

page_count = self.response["pagination"]["last_visible_page"]
for page_num in range(1, (page_count+1)):
time.sleep(4)
time.sleep(1)

api_url = f"{self.base_airing_api_url}&page={page_num}"
self.requestAPI(api_url)

update_characters_anime = []

for instance in self.response["data"]:
# add the anime's id to their airing anime
self.their_airing_anime.add(instance["mal_id"])
self.addAnime(instance)
# add the newly created anime to the update characters set
self.update_characters_anime.add(
Anime.objects.get(mal_id=instance["mal_id"]))
self.their_airing_anime.add(instance["mal_id"]) #add the anime's id to their airing anime

if add_characters is True:
update_characters_anime.add(Anime.objects.get(mal_id=instance["mal_id"])) # add the newly created anime to the update characters set

if add_characters is True:
self.updateCharacters(update_characters_anime, min_characters, min_side_characters,)

self.updateCharacters(self.update_characters_anime,
min_characters, min_side_characters,)

print(f"our_airing_anime: {self.our_airing_anime}")
print(f"their_airing_anime: {self.their_airing_anime}")
# get the anime that are in our_airing_anime but not their_airing_anime # this means that they are not airing anymore and our info is outdated
wrong_anime = self.our_airing_anime.difference(self.their_airing_anime)
wrong_anime = self.our_airing_anime.difference(self.their_airing_anime) # get the anime that are in our_airing_anime but not their_airing_anime # this means that they are not airing anymore and our info is outdated
print(f"wrong_anime: {wrong_anime}")

for mal_id in wrong_anime:
my_anime = Anime.objects.get(mal_id=mal_id)
# so just set the status to "Finished Airing"
my_anime.status = "Finished Airing"
my_anime.status = "Finished Airing" # so just set the status to "Finished Airing"
my_anime.save()

self.our_airing_anime = set()
self.their_airing_anime = set()

def noCharacterAnime(self, min_characters: int = 10, min_side_characters: int = 5,):
no_character_anime_set = set(
Anime.objects.filter(anime_characters=None))
self.updateCharacters(no_character_anime_set,
min_characters, min_side_characters)

def noCharacterAnime(self, min_characters: int = 10, min_side_characters: int = 5,):
no_character_anime_set = set(Anime.objects.filter(anime_characters=None))
self.updateCharacters(no_character_anime_set, min_characters, min_side_characters)
def updateCharacters(self, anime_set: set, min_characters: int = 10, min_side_characters: int = 5,):

for anime in anime_set:
my_mal_id = anime.mal_id
self.requestAPI(
f"https://api.jikan.moe/v4/anime/{my_mal_id}/characters")
self.requestAPI(f"https://api.jikan.moe/v4/anime/{my_mal_id}/characters")
supporting_index_favorites = dict()

for character in self.response["data"]:
if character["role"] == "Main":
self.addCharacter(character, my_mal_id)
else:
# add a key value pair to the dictionary {(character mal id): (number of favorites)}
supporting_index_favorites[self.response["data"].index(
character)] = character["favorites"]

sorted_supporting_index_favorites = dict(sorted(supporting_index_favorites.items(
), key=lambda x: x[1], reverse=True)) # sort the dictionary by descending order of favorites
supporting_index_favorites[self.response["data"].index(character)] = character["favorites"] # add a key value pair to the dictionary {(character mal id): (number of favorites)}
sorted_supporting_index_favorites = dict(sorted(supporting_index_favorites.items(), key=lambda x:x[1], reverse=True)) # sort the dictionary by descending order of favorites
print(f"SORTED: {sorted_supporting_index_favorites}")

main_characters = anime.anime_characters.all().count()
Expand All @@ -273,24 +265,26 @@ def updateCharacters(self, anime_set: set, min_characters: int = 10, min_side_ch
else:
side_characters = min_characters - main_characters

side_characters_to_add = list(sorted_supporting_index_favorites.keys())[
:side_characters] # get the first (side_characters) mal ids of the set
side_characters_to_add = list(sorted_supporting_index_favorites.keys())[:side_characters] # get the first (side_characters) mal ids of the set
for character_index in side_characters_to_add:
# find the character that corresponds to the given index
side_character = self.response["data"][character_index]
# add that character
self.addCharacter(side_character, my_mal_id)
side_character = self.response["data"][character_index] # find the character that corresponds to the given index
self.addCharacter(side_character, my_mal_id) # add that character

time.sleep(1)




time.sleep(4)

def addCharacter(self, character: dict, anime_mal_id: int):

my_character_mal_id = character["character"]["mal_id"]
my_character_name = character["character"]["name"]

try:
my_character_mal_id = character["character"]["mal_id"]
my_character_name = character["character"]["name"]
my_anime = Anime.objects.get(mal_id=anime_mal_id)
# if it exists it will move on fine
my_character = Character.objects.get(mal_id=my_character_mal_id)
# if not then it will move onto except
my_character = Character.objects.get(mal_id=my_character_mal_id) #if it exists it will move on fine
#if not then it will move onto except
print(f"character already exists: {my_character_name}")
my_anime.anime_characters.add(my_character)
except Character.DoesNotExist:
Expand All @@ -302,8 +296,13 @@ def addCharacter(self, character: dict, anime_mal_id: int):
)
my_character.save()
my_anime.anime_characters.add(my_character)
except Anime.DoesNotExist:
raise Exception("no anime with this mal id exists in our database lol")





DBPopulate = DBPopulate()

DBPopulate.initialPopulation(pages=10)
DBPopulate.initialPopulation(pages=100)
2 changes: 1 addition & 1 deletion poetry.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,4 @@ autopep8 = "^2.0.1"

[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
build-backend = "poetry.core.masonry.api"