Skip to content

Commit

Permalink
chore: write output files in temp directories (#1478)
Browse files Browse the repository at this point in the history
  • Loading branch information
@PierreLeveau
PierreLeveau authored Sep 22, 2023
1 parent 6aa45f8 commit 7c70818
Show file tree
Hide file tree
Showing 3 changed files with 122 additions and 98 deletions.
38 changes: 22 additions & 16 deletions tests/unit/services/export/test_geojson.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@


def test_kili_export_labels_geojson(mocker: pytest_mock.MockerFixture):
# Given
get_project_return_val = {
"jsonInterface": {"jobs": {"JOB": {"tools": ["rectangle"], "mlTask": "OBJECT_DETECTION"}}},
"inputType": "IMAGE",
Expand Down Expand Up @@ -43,25 +44,30 @@ def test_kili_export_labels_geojson(mocker: pytest_mock.MockerFixture):
kili.kili_api_gateway = mocker.MagicMock()
kili.kili_api_gateway.get_project.return_value = {"inputType": "IMAGE"}

kili.export_labels(
"fake_proj_id",
filename="export_geojson.zip",
fmt="geojson",
with_assets=False,
layout="merged",
)
with TemporaryDirectory() as export_folder:
export_filename = str(Path(export_folder) / "export_geojson.zip")

# When
kili.export_labels(
"fake_proj_id",
filename=export_filename,
fmt="geojson",
with_assets=False,
layout="merged",
)

with TemporaryDirectory() as extract_folder:
with ZipFile("export_geojson.zip", "r") as z_f:
# extract in a temp dir
z_f.extractall(extract_folder)
with TemporaryDirectory() as extract_folder:
with ZipFile(export_filename, "r") as z_f:
# extract in a temp dir
z_f.extractall(extract_folder)

assert Path(f"{extract_folder}/README.kili.txt").is_file()
assert Path(f"{extract_folder}/labels").is_dir()
assert Path(f"{extract_folder}/labels/sample.geojson").is_file()
# Then
assert Path(f"{extract_folder}/README.kili.txt").is_file()
assert Path(f"{extract_folder}/labels").is_dir()
assert Path(f"{extract_folder}/labels/sample.geojson").is_file()

with Path(f"{extract_folder}/labels/sample.geojson").open() as f:
output = json.load(f)
with Path(f"{extract_folder}/labels/sample.geojson").open() as f:
output = json.load(f)

assert output["type"] == "FeatureCollection"
assert len(output["features"]) == 5 # 5 annotations in geotiff_image_project_assets.json
102 changes: 57 additions & 45 deletions tests/unit/services/export/test_kili.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,21 +247,24 @@ def test_kili_export_labels_non_normalized_pdf(mocker: pytest_mock.MockerFixture
kili.kili_api_gateway = mocker.MagicMock()
kili.kili_api_gateway.get_project.return_value = {"inputType": "PDF"}

kili.export_labels(
"fake_proj_id", "export_pixel_coords_kili_pdf.zip", fmt="kili", normalized_coordinates=False
)
with TemporaryDirectory() as export_folder:
export_filename = str(Path(export_folder) / "export_pixel_coords_kili_pdf.zip")

kili.export_labels(
"fake_proj_id", export_filename, fmt="kili", normalized_coordinates=False
)

with TemporaryDirectory() as extract_folder:
with ZipFile("export_pixel_coords_kili_pdf.zip", "r") as z_f:
# extract in a temp dir
z_f.extractall(extract_folder)
with TemporaryDirectory() as extract_folder:
with ZipFile(export_filename, "r") as z_f:
# extract in a temp dir
z_f.extractall(extract_folder)

assert Path(f"{extract_folder}/README.kili.txt").is_file()
assert Path(f"{extract_folder}/labels").is_dir()
assert Path(f"{extract_folder}/labels/Cas_technique_n9.pdf.json").is_file()
assert Path(f"{extract_folder}/README.kili.txt").is_file()
assert Path(f"{extract_folder}/labels").is_dir()
assert Path(f"{extract_folder}/labels/Cas_technique_n9.pdf.json").is_file()

with Path(f"{extract_folder}/labels/Cas_technique_n9.pdf.json").open() as f:
output = json.load(f)
with Path(f"{extract_folder}/labels/Cas_technique_n9.pdf.json").open() as f:
output = json.load(f)

assert output == pdf_project_asset_unnormalized

Expand Down Expand Up @@ -391,24 +394,30 @@ def test_kili_export_labels_non_normalized_image(mocker: pytest_mock.MockerFixtu
kili.kili_api_gateway = mocker.MagicMock()
kili.kili_api_gateway.get_project.return_value = {"inputType": "IMAGE"}

kili.export_labels(
"fake_proj_id",
"export_pixel_coords_kili_image.zip",
fmt="kili",
normalized_coordinates=False,
)

with TemporaryDirectory() as extract_folder:
with ZipFile("export_pixel_coords_kili_image.zip", "r") as z_f:
# extract in a temp dir
z_f.extractall(extract_folder)

assert Path(f"{extract_folder}/README.kili.txt").is_file()
assert Path(f"{extract_folder}/labels").is_dir()
assert Path(f"{extract_folder}/labels/42015077eed072c50d59232dcc0ad0b1.jpg.json").is_file()

with Path(f"{extract_folder}/labels/42015077eed072c50d59232dcc0ad0b1.jpg.json").open() as f:
output = json.load(f)
with TemporaryDirectory() as export_folder:
export_filename = str(Path(export_folder) / "export_pixel_coords_kili_image.zip")
kili.export_labels(
"fake_proj_id",
export_filename,
fmt="kili",
normalized_coordinates=False,
)

with TemporaryDirectory() as extract_folder:
with ZipFile(export_filename, "r") as z_f:
# extract in a temp dir
z_f.extractall(extract_folder)

assert Path(f"{extract_folder}/README.kili.txt").is_file()
assert Path(f"{extract_folder}/labels").is_dir()
assert Path(
f"{extract_folder}/labels/42015077eed072c50d59232dcc0ad0b1.jpg.json"
).is_file()

with Path(
f"{extract_folder}/labels/42015077eed072c50d59232dcc0ad0b1.jpg.json"
).open() as f:
output = json.load(f)

assert output == image_project_asset_unnormalized

Expand Down Expand Up @@ -460,23 +469,26 @@ def test_kili_export_labels_non_normalized_video(mocker: pytest_mock.MockerFixtu
kili.kili_api_gateway = mocker.MagicMock()
kili.kili_api_gateway.get_project.return_value = {"inputType": "VIDEO"}

kili.export_labels(
"fake_proj_id",
"export_pixel_coords_kili_video.zip",
fmt="kili",
normalized_coordinates=False,
)
with TemporaryDirectory() as export_folder:
export_filename = str(Path(export_folder) / "export_pixel_coords_kili_video.zip")

kili.export_labels(
"fake_proj_id",
export_filename,
fmt="kili",
normalized_coordinates=False,
)

with TemporaryDirectory() as extract_folder:
with ZipFile("export_pixel_coords_kili_video.zip", "r") as z_f:
# extract in a temp dir
z_f.extractall(extract_folder)
with TemporaryDirectory() as extract_folder:
with ZipFile(export_filename, "r") as z_f:
# extract in a temp dir
z_f.extractall(extract_folder)

assert Path(f"{extract_folder}/README.kili.txt").is_file()
assert Path(f"{extract_folder}/labels").is_dir()
assert Path(f"{extract_folder}/labels/Click_here_to_start.json").is_file()
assert Path(f"{extract_folder}/README.kili.txt").is_file()
assert Path(f"{extract_folder}/labels").is_dir()
assert Path(f"{extract_folder}/labels/Click_here_to_start.json").is_file()

with Path(f"{extract_folder}/labels/Click_here_to_start.json").open() as f:
output = json.load(f)
with Path(f"{extract_folder}/labels/Click_here_to_start.json").open() as f:
output = json.load(f)

assert output == video_project_asset_unnormalized
80 changes: 43 additions & 37 deletions tests/unit/services/export/test_yolo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,28 +292,31 @@ def test_yolo_v8_merged(mocker: pytest_mock.MockerFixture):
kili.kili_api_gateway = mocker.MagicMock()
kili.kili_api_gateway.get_project.return_value = {"inputType": "IMAGE"}

kili.export_labels(
"clktm4vzz001a0j324elr5dsy",
filename="export_yolo_v8.zip",
fmt="yolo_v8",
layout="merged",
with_assets=False,
)
with TemporaryDirectory() as export_folder:
export_filename = str(Path(export_folder) / "export_yolo_v8.zip")

kili.export_labels(
"clktm4vzz001a0j324elr5dsy",
filename=export_filename,
fmt="yolo_v8",
layout="merged",
with_assets=False,
)

with TemporaryDirectory() as extract_folder:
with ZipFile("export_yolo_v8.zip", "r") as z_f:
# extract in a temp dir
z_f.extractall(extract_folder)
with TemporaryDirectory() as extract_folder:
with ZipFile(export_filename, "r") as z_f:
# extract in a temp dir
z_f.extractall(extract_folder)

assert Path(f"{extract_folder}/README.kili.txt").is_file()
assert Path(f"{extract_folder}/README.kili.txt").is_file()

assert Path(f"{extract_folder}/data.yaml").read_text() == """nc: 4
assert Path(f"{extract_folder}/data.yaml").read_text() == """nc: 4
names: ['OBJECT_DETECTION_JOB/A', 'OBJECT_DETECTION_JOB/B', 'POLYGON_JOB/F', 'POLYGON_JOB/G']
"""

assert Path(f"{extract_folder}/labels").is_dir()
assert Path(f"{extract_folder}/labels").is_dir()

label = Path(f"{extract_folder}/labels/trees.txt").read_text()
label = Path(f"{extract_folder}/labels/trees.txt").read_text()

# bbox annotation: class bbox_center_x bbox_center_y bbox_w bbox_h
assert "0 0.65 0.1 0.5 0.09999999999999999" in label
Expand All @@ -340,32 +343,35 @@ def test_yolo_v8_split_jobs(mocker: pytest_mock.MockerFixture):
kili.kili_api_gateway = mocker.MagicMock()
kili.kili_api_gateway.get_project.return_value = {"inputType": "IMAGE"}

kili.export_labels(
"clktm4vzz001a0j324elr5dsy",
filename="export_yolo_v8.zip",
fmt="yolo_v8",
layout="split",
with_assets=False,
)

with TemporaryDirectory() as extract_folder:
with ZipFile("export_yolo_v8.zip", "r") as z_f:
# extract in a temp dir
z_f.extractall(extract_folder)
export_filename = str(Path(extract_folder) / "export_yolo_v8.zip")

kili.export_labels(
"clktm4vzz001a0j324elr5dsy",
filename=export_filename,
fmt="yolo_v8",
layout="split",
with_assets=False,
)

assert Path(f"{extract_folder}/README.kili.txt").is_file()
assert Path(f"{extract_folder}/OBJECT_DETECTION_JOB/data.yaml").read_text() == """nc: 2
with TemporaryDirectory() as extract_folder:
with ZipFile(export_filename, "r") as z_f:
# extract in a temp dir
z_f.extractall(extract_folder)

assert Path(f"{extract_folder}/README.kili.txt").is_file()
assert Path(f"{extract_folder}/OBJECT_DETECTION_JOB/data.yaml").read_text() == """nc: 2
names: ['A', 'B']
"""
assert Path(f"{extract_folder}/POLYGON_JOB/data.yaml").read_text() == """nc: 2
assert Path(f"{extract_folder}/POLYGON_JOB/data.yaml").read_text() == """nc: 2
names: ['F', 'G']
"""

assert (
Path(f"{extract_folder}/OBJECT_DETECTION_JOB/labels/trees.txt").read_text()
== "0 0.65 0.1 0.5 0.09999999999999999\n"
)
assert (
Path(f"{extract_folder}/POLYGON_JOB/labels/trees.txt").read_text()
== "0 0.75 0.23 0.35 0.22 0.07 0.35\n"
)
assert (
Path(f"{extract_folder}/OBJECT_DETECTION_JOB/labels/trees.txt").read_text()
== "0 0.65 0.1 0.5 0.09999999999999999\n"
)
assert (
Path(f"{extract_folder}/POLYGON_JOB/labels/trees.txt").read_text()
== "0 0.75 0.23 0.35 0.22 0.07 0.35\n"
)

0 comments on commit 7c70818

Please sign in to comment.