Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove pandas from gsaid api #3393

Open
wants to merge 42 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
84f208f
feat(rm_pandas): WIP
henrikstranneheim Jul 2, 2024
7d8d619
feat(rm_pandas): WIP
henrikstranneheim Jul 3, 2024
b056ad7
feat(rm_pandas): WIP
henrikstranneheim Jul 3, 2024
029cf08
feat(rm_pandas):
henrikstranneheim Jul 4, 2024
095574e
feat(rm_pandas):
henrikstranneheim Jul 4, 2024
5dbbeff
feat(rm_pandas):
henrikstranneheim Jul 4, 2024
c2cd1c4
feat(update): Linters
henrikstranneheim Aug 6, 2024
2150c07
Merge remote-tracking branch 'origin/master'
henrikstranneheim Aug 8, 2024
1735c03
Merge remote-tracking branch 'origin/master'
henrikstranneheim Aug 8, 2024
0da7f66
Merge remote-tracking branch 'origin/master'
henrikstranneheim Aug 9, 2024
5926546
Merge
henrikstranneheim Aug 9, 2024
69ce6f2
Merge remote-tracking branch 'origin/master'
henrikstranneheim Aug 9, 2024
3a20a70
Merge branch 'master' into remove-pandas-from-gsaid-api
henrikstranneheim Aug 9, 2024
2904d9e
feat(remove): pandas
henrikstranneheim Aug 9, 2024
bceafdc
Merge remote-tracking branch 'origin/master'
henrikstranneheim Aug 9, 2024
a5d21d6
Merge branch 'master' into remove-pandas-from-gsaid-api
henrikstranneheim Aug 9, 2024
b795184
Merge remote-tracking branch 'origin/master'
henrikstranneheim Aug 11, 2024
90c06ef
Merge remote-tracking branch 'origin/master'
henrikstranneheim Aug 14, 2024
b1e548b
Merge branch 'master' into remove-pandas-from-gsaid-api
henrikstranneheim Aug 16, 2024
ca58e05
feat(header): Use complementary
henrikstranneheim Aug 16, 2024
82644c3
Merge remote-tracking branch 'origin/master'
henrikstranneheim Aug 19, 2024
13c0352
Merge branch 'master' into remove-pandas-from-gsaid-api
henrikstranneheim Aug 19, 2024
26e627f
feat(rm_pandas): WIP
henrikstranneheim Aug 19, 2024
cab4d11
feat(rm_pandas): WIP
henrikstranneheim Aug 19, 2024
57e4ddd
feat(rm_pandas): WIP
henrikstranneheim Aug 20, 2024
dd6c9f1
feat(rm_pandas): WIP
henrikstranneheim Aug 20, 2024
676f8a4
Merge branch 'master' into remove-pandas-from-gsaid-api
henrikstranneheim Aug 20, 2024
40a3c1f
feat(rm_pandas): WIP
henrikstranneheim Aug 21, 2024
97db74f
feat(rm_pandas): WIP
henrikstranneheim Aug 21, 2024
6869e0d
feat(rm_pandas): WIP
henrikstranneheim Aug 21, 2024
6ebdfbd
feat(rm_pandas): WIP
henrikstranneheim Aug 21, 2024
7a22141
feat(rm_pandas): WIP
henrikstranneheim Aug 21, 2024
d331d12
Merge branch 'master' into remove-pandas-from-gsaid-api
henrikstranneheim Aug 22, 2024
6dea4dd
feat(rm_pandas): WIP
henrikstranneheim Aug 22, 2024
64f17b4
Merge remote-tracking branch 'origin/master'
henrikstranneheim Sep 1, 2024
3a5a550
Merge branch 'master' into remove-pandas-from-gsaid-api
henrikstranneheim Sep 1, 2024
a1073ed
feat(gisaid: Add test
henrikstranneheim Sep 1, 2024
259688e
feat(gisaid: Add test
henrikstranneheim Sep 1, 2024
f1c8fb6
feat(gisaid): WIP
henrikstranneheim Sep 1, 2024
12e817f
feat(gisaid): WIP
henrikstranneheim Sep 1, 2024
6d927b4
feat(gisaid): WIP
henrikstranneheim Sep 1, 2024
3c7ca9c
Merge branch 'master' into remove-pandas-from-gsaid-api
henrikstranneheim Sep 16, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cg/cli/upload/fohm.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ def preprocess_all(
upload_cases = []
for case_id in cases:
try:
gisaid_api.upload(case_id=case_id)
gisaid_api.upload_to_gisaid(case_id)
fohm_api.update_upload_started_at(case_id=case_id)
LOG.info(f"Upload of case {case_id} to GISAID was successful")
upload_cases.append(case_id)
Expand Down
6 changes: 3 additions & 3 deletions cg/cli/upload/gisaid.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Code for uploading genotype data via CLI"""
"""Code for uploading genotype data via CLI."""

import logging

Expand All @@ -18,7 +18,7 @@ def upload_to_gisaid(context: CGConfig, case_id: str):

LOG.info("----------------- GISAID UPLOAD -------------------")

gisaid_api = GisaidAPI(config=context)
gisaid_api = GisaidAPI(context)

gisaid_api.upload(case_id=case_id)
gisaid_api.upload_to_gisaid(case_id)
LOG.info("Upload to GISAID successful")
1 change: 1 addition & 0 deletions cg/constants/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,7 @@ class FileExtensions(StrEnum):
CONFIG: str = ".config"
CRAM: str = ".cram"
CSV: str = ".csv"
FASTA: str = ".fasta"
FASTQ: str = ".fastq"
FASTQ_GZ: str = ".fastq.gz"
GPG: str = ".gpg"
Expand Down
7 changes: 7 additions & 0 deletions cg/constants/housekeeper_tags.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,3 +250,10 @@ class JanusTags:
class FohmTag(StrEnum):
COMPLEMENTARY = "komplettering"
PANGOLIN_TYPING = "pangolin-typing-fohm"


class GisaidTag(StrEnum):
FASTA = "gisaid-fasta"
CONSENSUS_SAMPLE = "consensus-sample"
CSV = "gisaid-csv"
LOG = "gisaid-log"
376 changes: 229 additions & 147 deletions cg/meta/upload/gisaid/gisaid.py

Large diffs are not rendered by default.

24 changes: 12 additions & 12 deletions cg/meta/upload/gisaid/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,26 +45,26 @@ class GisaidSample(BaseModel):
model_config = ConfigDict(coerce_numbers_to_str=True)
case_id: str
cg_lims_id: str
submitter: str
region: str
region_code: str
fn: str
covv_authors: str | None = " ,".join(AUTHORS)
covv_collection_date: str
covv_subm_sample_id: str
covv_virus_name: str | None = None
covv_gender: str | None = "unknown"
covv_host: str | None = "Human"
covv_location: str | None = None
covv_orig_lab: str | None = None
covv_type: str | None = "betacoronavirus"
covv_orig_lab_addr: str | None = None
covv_passage: str | None = "Original"
covv_location: str | None = None
covv_host: str | None = "Human"
covv_gender: str | None = "unknown"
covv_patient_age: str | None = "unknown"
covv_patient_status: str | None = "unknown"
covv_seq_technology: str | None = "Illumina NovaSeq"
covv_orig_lab_addr: str | None = None
covv_subm_lab: str | None = "Karolinska University Hospital"
covv_subm_lab_addr: str | None = "171 76 Stockholm, Sweden"
covv_authors: str | None = " ,".join(AUTHORS)
covv_subm_sample_id: str
covv_type: str | None = "betacoronavirus"
covv_virus_name: str | None = None
fn: str
region: str
region_code: str
submitter: str

@model_validator(mode="before")
@classmethod
Expand Down
Empty file added cg/models/gisaid/__init__.py
Empty file.
11 changes: 11 additions & 0 deletions cg/models/gisaid/reports.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
"""GISAID report models."""

from pydantic import BaseModel, Field


class GisaidComplementaryReport(BaseModel):
"""Model for validating a GSAID complementary reports."""

gisaid_accession: str = Field(None, alias="GISAID_accession")
sample_number: str = Field(str, alias="provnummer")
selection_criteria: str = Field(str, alias="urvalskriterium")
Comment on lines +10 to +11
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the str in the default position might cause a bug

133 changes: 1 addition & 132 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ marshmallow = "*"
MarkupSafe = "*"
openpyxl = "*"
packaging = "*"
pandas = "*"
paramiko = "*"
petname = "*"
psutil = "*"
Expand Down
31 changes: 24 additions & 7 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,13 @@
from cg.apps.slurm.slurm_api import SlurmAPI
from cg.apps.tb.dto.summary_response import AnalysisSummary, StatusSummary
from cg.constants import FileExtensions, SequencingFileTag, Workflow
from cg.constants.constants import CaseActions, CustomerId, FileFormat, GenomeVersion, Strandedness
from cg.constants.constants import (
CaseActions,
CustomerId,
FileFormat,
GenomeVersion,
Strandedness,
)
from cg.constants.gene_panel import GenePanelMasterList
from cg.constants.housekeeper_tags import HK_DELIVERY_REPORT_TAG
from cg.constants.priority import SlurmQos
Expand All @@ -49,16 +55,24 @@
from cg.models import CompressionData
from cg.models.cg_config import CGConfig, PDCArchivingDirectory
from cg.models.downsample.downsample_data import DownsampleData
from cg.models.raredisease.raredisease import RarediseaseParameters, RarediseaseSampleSheetHeaders
from cg.models.raredisease.raredisease import (
RarediseaseParameters,
RarediseaseSampleSheetHeaders,
)
from cg.models.rnafusion.rnafusion import RnafusionParameters, RnafusionSampleSheetEntry
from cg.models.run_devices.illumina_run_directory_data import IlluminaRunDirectoryData
from cg.models.taxprofiler.taxprofiler import TaxprofilerParameters, TaxprofilerSampleSheetEntry
from cg.models.taxprofiler.taxprofiler import (
TaxprofilerParameters,
TaxprofilerSampleSheetEntry,
)
from cg.models.tomte.tomte import TomteParameters, TomteSampleSheetHeaders
from cg.services.deliver_files.delivery_rsync_service.delivery_rsync_service import (
DeliveryRsyncService,
)
from cg.services.illumina.backup.encrypt_service import IlluminaRunEncryptionService
from cg.services.illumina.data_transfer.data_transfer_service import IlluminaDataTransferService
from cg.services.illumina.data_transfer.data_transfer_service import (
IlluminaDataTransferService,
)
from cg.store.database import create_all_tables, drop_all_tables, initialize_database
from cg.store.models import (
Application,
Expand Down Expand Up @@ -110,22 +124,25 @@
"tests.fixture_plugins.device_fixtures",
"tests.fixture_plugins.encryption_fixtures.encryption_fixtures",
"tests.fixture_plugins.fohm.fohm_fixtures",
"tests.fixture_plugins.io.csv_fixtures",
"tests.fixture_plugins.gisaid_fixtures.gisaid_fixtures",
"tests.fixture_plugins.illumina_clean_fixtures.clean_fixtures",
"tests.fixture_plugins.io.csv_fixtures",
"tests.fixture_plugins.io.csv_fixtures",
"tests.fixture_plugins.loqusdb_fixtures.loqusdb_api_fixtures",
"tests.fixture_plugins.loqusdb_fixtures.loqusdb_output_fixtures",
"tests.fixture_plugins.observations_fixtures.observations_api_fixtures",
"tests.fixture_plugins.observations_fixtures.observations_input_files_fixtures",
"tests.fixture_plugins.orders_fixtures.order_form_fixtures",
"tests.fixture_plugins.orders_fixtures.order_store_service_fixtures",
"tests.fixture_plugins.pacbio_fixtures.context_fixtures",
"tests.fixture_plugins.pacbio_fixtures.metrics_fixtures",
"tests.fixture_plugins.pacbio_fixtures.name_fixtures",
"tests.fixture_plugins.pacbio_fixtures.path_fixtures",
"tests.fixture_plugins.pacbio_fixtures.service_fixtures",
"tests.fixture_plugins.quality_controller_fixtures.sequencing_qc_check_scenario",
"tests.fixture_plugins.quality_controller_fixtures.sequencing_qc_fixtures",
"tests.fixture_plugins.sample.sample_fixtures",
"tests.fixture_plugins.timestamp_fixtures",
"tests.fixture_plugins.orders_fixtures.order_form_fixtures",
"tests.fixture_plugins.orders_fixtures.order_store_service_fixtures",
]


Expand Down
Loading
Loading