Skip to content

Commit

Permalink
Add absent to sample that have not been uploaded on iNaturalist for g…
Browse files Browse the repository at this point in the history
…ood reasons (soil for example)
  • Loading branch information
edouardbruelhart committed May 15, 2024
1 parent db8bb63 commit f90930b
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 18 deletions.
2 changes: 1 addition & 1 deletion inat_fetcher/src/db_updater.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@

# Update the observation dictionary with values from the current row
for col_name, value in obs.items():
if col_name == "geojson.coordinates":
if col_name == "geojson.coordinates" and value:
observation[col_name.replace(".", "_")] = "{" + geo_prefix + value + "}"
else:
observation[col_name.replace(".", "_")] = value
Expand Down
31 changes: 23 additions & 8 deletions inat_fetcher/src/directus_link_maker.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,26 @@
item_id = [item["id"] for item in data]
emi_id = [item["emi_external_id"] for item in data]
for i in range(len(item_id)):
directus_patch = f"{directus_instance}/items/Field_Samples/" + emi_id[i]
inaturalist_link = "https://www.inaturalist.org/observations/" + item_id[i]
observation = {"inat_observation_id": item_id[i], "inaturalist_link": inaturalist_link}
response = session.patch(url=directus_patch, headers=headers, json=observation)
if response.status_code != 200:
print(f"error, couldn't make the link between {item_id[i]} and {emi_id[i]}")
print(response.status_code)
print(response.text)
if emi_id[i]:
directus_patch = f"{directus_instance}/items/Field_Samples/" + emi_id[i]
inaturalist_link = "https://www.inaturalist.org/observations/" + item_id[i]
observation = {"inat_observation_id": item_id[i], "inaturalist_link": inaturalist_link}
response = session.patch(url=directus_patch, headers=headers, json=observation)
if response.status_code != 200:
print(f"error, couldn't make the link between {item_id[i]} and {emi_id[i]}")
print(response.status_code)
print(response.text)

# Send get request to check if data has been added to directus. If not, set inat_observation_id and inaturalist_link to absent
directus_api2 = f"{directus_instance}/items/Qfield_Data/"
response2 = session.get(url=f"{directus_api2}?limit=-1", headers=headers)
data2 = response2.json()["data"]
item_id2 = [item["field_sample_name"] for item in data2]
emi_id2 = [item["field_sample_id_pk"] for item in data2]
for i in range(len(item_id2)):
if item_id2[i] == "Mixte":
print(item_id2[i])
print(emi_id2[i])
directus_patch2 = f"{directus_instance}/items/Field_Samples/" + emi_id2[i]
observation2 = {"inat_observation_id": "absent", "inaturalist_link": "absent"}
response2 = session.patch(url=directus_patch2, headers=headers, json=observation2)
46 changes: 37 additions & 9 deletions inat_fetcher/src/fetcher.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
import os
from pathlib import Path

# import db_updater
import pandas as pd
from pyinaturalist import get_observations
from pyinaturalist_convert import to_dataframe

# import format_module


# To obtain actual path to inat_fetcher dir
p = Path(__file__).parents[1]

Expand All @@ -20,19 +17,50 @@
# import env variable
access_token = os.getenv("INATURALIST_ACCESS_TOKEN")

response = get_observations(project_id=130644, page="all", per_page=200, access_token=access_token)
df = to_dataframe(response)
# Fetch values from iNaturalist for the different users
response_project = get_observations(project_id=130644, page="all", per_page=200, access_token=access_token)
df_project = to_dataframe(response_project)

response_dbgi = get_observations(user_id="dbgi", page="all", per_page=200, access_token=access_token)
df_dbgi = to_dataframe(response_dbgi)

response_edouardbruelhart = get_observations(
user_id="edouardbruelhart", page="all", per_page=200, access_token=access_token
)
df_edouardbruelhart = to_dataframe(response_edouardbruelhart)

response_edouardbrulhart = get_observations(
user_id="edouard-brulhart", page="all", per_page=200, access_token=access_token
)
df_edouardbrulhart = to_dataframe(response_edouardbrulhart)

# Before exporting we move the id column to the beginning since it is needed to be at this position to be detected as a PK in airtbale or siomnilar dbs
response_manu_dfz = get_observations(user_id="manu_dfz", page="all", per_page=200, access_token=access_token)
df_manu_dfz = to_dataframe(response_manu_dfz)

response_lenditaschwegler = get_observations(
user_id="lenditaschwegler", page="all", per_page=200, access_token=access_token
)
df_lenditaschwegler = to_dataframe(response_lenditaschwegler)

response_guetchuengst = get_observations(user_id="guetchuengst", page="all", per_page=200, access_token=access_token)
df_guetchuengst = to_dataframe(response_guetchuengst)

# Merge iNaturalist data
df = pd.concat(
[df_project, df_dbgi, df_edouardbruelhart, df_edouardbrulhart, df_manu_dfz, df_lenditaschwegler, df_guetchuengst],
ignore_index=True,
)

# shift column 'id' to first position
first_column = df.pop("id")

# insert column using insert(position,column_name,
# first_column) function
df.insert(0, "id", first_column)

# We keep the table
# Drop duplicates
df = df.drop_duplicates(subset=["id"])

# Write the table as CSV
df.to_csv(path_to_output_file, index=False)

print("csv correctly written")

0 comments on commit f90930b

Please sign in to comment.