Merge pull request #489 from pepkit/minor_fixes

Release 0.40.3

peppy/_version.py

@@ -1 +1 @@
-__version__ = "0.40.2"
+__version__ = "0.40.3"

peppy/const.py

@@ -13,11 +13,12 @@
 SAMPLE_TABLE_INDEX_KEY = "sample_table_index"
 SUBSAMPLE_TABLE_INDEX_KEY = "subsample_table_index"
 CONFIG_KEY = "_config"
+ORIGINAL_CONFIG_KEY = "_original_config"
 PROJECT_TYPENAME = "Project"
 SAMPLE_MODS_KEY = "sample_modifiers"
 PROJ_MODS_KEY = "project_modifiers"
 NAME_TABLE_ATTR = "sample_table"
-CONSTANT_KEY = "append"
+APPEND_KEY = "append"
 REMOVE_KEY = "remove"
 DUPLICATED_KEY = "duplicate"
 DERIVED_KEY = "derive"
@@ -33,11 +34,11 @@
 AMENDMENTS_KEY = "amend"
 ACTIVE_AMENDMENTS_KEY = "_" + AMENDMENTS_KEY
 SAMPLE_EDIT_FLAG_KEY = "_samples_touched"
-SAMPLE_MODIFIERS = [CONSTANT_KEY, IMPLIED_KEY, DERIVED_KEY, DUPLICATED_KEY, REMOVE_KEY]
+SAMPLE_MODIFIERS = [APPEND_KEY, IMPLIED_KEY, DERIVED_KEY, DUPLICATED_KEY, REMOVE_KEY]
 PROJECT_MODIFIERS = [CFG_IMPORTS_KEY, AMENDMENTS_KEY]
 PROJECT_CONSTANTS = [
  "REQUIRED_VERSION",
- "CONSTANT_KEY",
+ "APPEND_KEY",
  "DERIVED_SOURCES_KEY",
  "DERIVED_KEY",
  "SAMPLE_MODS_KEY",

peppy/project.py

@@ -16,6 +16,7 @@
 from rich.console import Console
 from rich.progress import track
 from ubiquerg import is_url
+from copy import deepcopy
 
 from .const import (
  ACTIVE_AMENDMENTS_KEY,
@@ -27,7 +28,7 @@
  CONFIG_FILE_KEY,
  CONFIG_KEY,
  CONFIG_VERSION_KEY,
- CONSTANT_KEY,
+ APPEND_KEY,
  DERIVED_ATTRS_KEY,
  DERIVED_KEY,
  DERIVED_SOURCES_KEY,
@@ -59,6 +60,7 @@
  SUBSAMPLE_RAW_LIST_KEY,
  SUBSAMPLE_TABLE_INDEX_KEY,
  SUBSAMPLE_TABLES_FILE_KEY,
+ ORIGINAL_CONFIG_KEY,
 )
 from .exceptions import (
  InvalidSampleTableFileException,
@@ -137,7 +139,6 @@ def __init__(
 
  self._samples = []
  self[SAMPLE_EDIT_FLAG_KEY] = False
- self.is_private = False
  self.progressbar = False
 
  # table indexes can be specified in config or passed to the object constructor
@@ -200,6 +201,19 @@ def from_pandas(
  )
  return tmp_obj
 
+ @classmethod
+ def from_pephub(cls, registry_path: str) -> "Project":
+ """
+ Init project from pephubclient.
+
+ :param registry_path: PEPhub registry path
+ :return: peppy Project
+ """
+ from pephubclient import PEPHubClient
+
+ phc = PEPHubClient()
+ return phc.load_project(project_registry_path=registry_path)
+
  @classmethod
  def from_dict(cls, pep_dictionary: dict):
  """
@@ -317,13 +331,13 @@ def to_dict(
  else:
  sub_df = None
  try:
- self[CONFIG_KEY][NAME_KEY] = self.name
+ self[ORIGINAL_CONFIG_KEY][NAME_KEY] = self.name
  except NotImplementedError:
- self[CONFIG_KEY][NAME_KEY] = "unnamed"
- self[CONFIG_KEY][DESC_KEY] = self.description
+ self[ORIGINAL_CONFIG_KEY][NAME_KEY] = "unnamed"
+ self[ORIGINAL_CONFIG_KEY][DESC_KEY] = self.description
  p_dict = {
  SAMPLE_RAW_DICT_KEY: self[SAMPLE_DF_KEY].to_dict(orient=orient),
- CONFIG_KEY: dict(self[CONFIG_KEY]),
+ CONFIG_KEY: dict(self[ORIGINAL_CONFIG_KEY]),
  SUBSAMPLE_RAW_LIST_KEY: sub_df,
  }
  else:
@@ -438,6 +452,7 @@ def parse_config_file(
  )
 
  self[CONFIG_KEY].update(**config)
+ self[ORIGINAL_CONFIG_KEY] = deepcopy(self[CONFIG_KEY])
  # Parse yaml into the project.config attributes
  _LOGGER.debug("Adding attributes: {}".format(", ".join(config)))
  # Overwrite any config entries with entries in the amendments
@@ -587,8 +602,8 @@ def attr_constants(self):
  Update each Sample with constants declared by a Project.
  If Project does not declare constants, no update occurs.
  """
- if self._modifier_exists(CONSTANT_KEY):
- to_append = self[CONFIG_KEY][SAMPLE_MODS_KEY][CONSTANT_KEY]
+ if self._modifier_exists(APPEND_KEY):
+ to_append = self[CONFIG_KEY][SAMPLE_MODS_KEY][APPEND_KEY]
  _LOGGER.debug("Applying constant attributes: {}".format(to_append))
 
  for s in track(
@@ -1309,14 +1324,17 @@ def _read_sample_data(self):
  _LOGGER.info("No config key in Project, or reading project from dict")
  return
  if CFG_SAMPLE_TABLE_KEY not in self[CONFIG_KEY]:
- _LOGGER.debug("no {} found".format(CFG_SAMPLE_TABLE_KEY))
+ _LOGGER.debug(f"No {CFG_SAMPLE_TABLE_KEY} found in config file")
  return
  st = self[CONFIG_KEY][CFG_SAMPLE_TABLE_KEY]
 
  if self[SUBSAMPLE_TABLES_FILE_KEY] is not None:
  sst = self[SUBSAMPLE_TABLES_FILE_KEY]
  else:
- if CONFIG_KEY in self and CFG_SUBSAMPLE_TABLE_KEY in self[CONFIG_KEY]:
+ if (
+ CONFIG_KEY in self
+ and self[CONFIG_KEY].get(CFG_SUBSAMPLE_TABLE_KEY) is not None
+ ):
  sst = make_list(self[CONFIG_KEY][CFG_SUBSAMPLE_TABLE_KEY], str)
  else:
  sst = None

peppy/sample.py

@@ -4,6 +4,7 @@
 from copy import copy as cp
 from logging import getLogger
 from string import Formatter
+from typing import Optional, Union
 
 import pandas as pd
 import yaml
@@ -134,31 +135,39 @@ def _obj2dict(obj, name=None):
  serial.update({"prj": grab_project_data(self[PRJ_REF])})
  return serial
 
- def to_yaml(self, path, add_prj_ref=False):
+ def to_yaml(
+ self, path: Optional[str] = None, add_prj_ref=False
+ ) -> Union[str, None]:
  """
- Serializes itself in YAML format.
+ Serializes itself in YAML format. Writes to file if path is provided, else returns string representation.
 
  :param str path: A file path to write yaml to; provide this or
- the subs_folder_path
+ the subs_folder_path, defaults to None
  :param bool add_prj_ref: whether the project reference bound do the
  Sample object should be included in the YAML representation
+ :return str | None: returns string representation of sample yaml or None
  """
  serial = self.to_dict(add_prj_ref=add_prj_ref)
- path = os.path.expandvars(path)
- if not os.path.exists(os.path.dirname(path)):
- _LOGGER.warning(
- "Could not write sample data to: {}. "
- "Directory does not exist".format(path)
- )
- return
- with open(path, "w") as outfile:
- try:
- yaml_data = yaml.safe_dump(serial, default_flow_style=False)
- except yaml.representer.RepresenterError:
- _LOGGER.error("Serialized sample data: {}".format(serial))
- raise
- outfile.write(yaml_data)
- _LOGGER.debug("Sample data written to: {}".format(path))
+ if path:
+ path = os.path.expandvars(path)
+ if os.path.exists(os.path.dirname(path)):
+ with open(path, "w") as outfile:
+ try:
+ yaml_data = yaml.safe_dump(serial, default_flow_style=False)
+ except yaml.representer.RepresenterError:
+ _LOGGER.error("Serialized sample data: {}".format(serial))
+ raise
+ outfile.write(yaml_data)
+ _LOGGER.debug("Sample data written to: {}".format(path))
+ else:
+ _LOGGER.warning(
+ "Could not write sample data to: {}. "
+ "Directory does not exist".format(path)
+ )
+ return
+ else:
+ yaml_data = yaml.safe_dump(serial, stream=None, default_flow_style=False)
+ return yaml_data
 
  def derive_attribute(self, data_sources, attr_name):
  """

peppy/simple_attr_map.py

@@ -46,6 +46,9 @@ def __getattr__(self, item):
  except KeyError:
  raise AttributeError(f"Attribute not found: {item}")
 
+ def __eq__(self, other: "SimpleAttMap"):
+ return self._mapped_attr == other._mapped_attr
+
  @property
  def attributes(self):
  return self._mapped_attr

peppy/utils.py

@@ -2,7 +2,7 @@
 
 import logging
 import os
-from typing import Dict, Mapping
+from typing import Dict, Mapping, Type, Union
 from urllib.request import urlopen
 
 import yaml
@@ -76,14 +76,16 @@ def grab_project_data(prj):
  raise KeyError("Project lacks section '{}'".format(CONFIG_KEY))
 
 
-def make_list(arg, obj_class):
+def make_list(arg: Union[list, str], obj_class: Type) -> list:
  """
  Convert an object of predefined class to a list of objects of that class or
  ensure a list is a list of objects of that class
 
  :param list[obj] | obj arg: string or a list of strings to listify
  :param str obj_class: name of the class of intrest
+
  :return list: list of objects of the predefined class
+
  :raise TypeError: if a faulty argument was provided
  """
 

requirements/requirements-all.txt

@@ -3,3 +3,4 @@ pyyaml
 rich>=10.3.0
 ubiquerg>=0.6.2
 numpy
+pephubclient>=0.4.2

tests/conftest.py

@@ -51,11 +51,6 @@ def example_pep_cfg_noname_path(request):
  return get_path_to_example_file(EPB, "noname", request.param)
 
 
-@pytest.fixture
-def example_pep_cfg_custom_index(request):
- return get_path_to_example_file(EPB, "custom_index", request.param)
-
-
 @pytest.fixture
 def example_peps_cfg_paths(request):
  """

tests/data/example_peps-master/example_nextflow_subsamples/project_config.yaml

@@ -0,0 +1,5 @@
+pep_version: "2.0.0"
+sample_table: "samplesheet.csv"
+subsample_table: "subsamplesheet.csv"
+sample_table_index: "sample"
+subsample_table_index: "sample"

tests/data/example_peps-master/example_nextflow_subsamples/samplesheet.csv

@@ -0,0 +1,6 @@
+sample,strandedness
+WT_REP1,reverse
+WT_REP2,reverse
+RAP1_UNINDUCED_REP1,reverse
+RAP1_UNINDUCED_REP2,reverse
+RAP1_IAA_30M_REP1,reverse

tests/data/example_peps-master/example_nextflow_subsamples/subsamplesheet.csv

@@ -0,0 +1,8 @@
+sample,instrument_platform,run_accession,fastq_1,fastq_2
+WT_REP1,ABI_SOLID,runaccession1,https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/testdata/GSE110004/SRR6357070_1.fastq.gz,https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/testdata/GSE110004/SRR6357070_2.fastq.gz
+WT_REP1,BGISEQ,runaccession2,https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/testdata/GSE110004/SRR6357071_1.fastq.gz,https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/testdata/GSE110004/SRR6357071_2.fastq.gz
+WT_REP2,CAPILLARY,123123123,https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/testdata/GSE110004/SRR6357072_1.fastq.gz,https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/testdata/GSE110004/SRR6357072_2.fastq.gz
+RAP1_UNINDUCED_REP1,COMPLETE_GENOMICS,somerunaccesion,https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/testdata/GSE110004/SRR6357073_1.fastq.gz,
+RAP1_UNINDUCED_REP2,DNBSEQ,ERR2412421,https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/testdata/GSE110004/SRR6357074_1.fastq.gz,
+RAP1_UNINDUCED_REP2,HELICOS,xxxxxxxxxx,https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/testdata/GSE110004/SRR6357075_1.fastq.gz,
+RAP1_IAA_30M_REP1,ILLUMINA,None,https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/testdata/GSE110004/SRR6357076_1.fastq.gz,https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/testdata/GSE110004/SRR6357076_2.fastq.gz

tests/data/example_peps-master/example_subsamples_none/project_config.yaml

@@ -0,0 +1,3 @@
+pep_version: "2.0.0"
+sample_table: sample_table.csv
+subsample_table: null

tests/data/example_peps-master/example_subsamples_none/sample_table.csv

@@ -0,0 +1,4 @@
+sample_name,protocol,file
+frog_1,anySampleType,multi
+frog_2,anySampleType,multi
+frog_3,anySampleType,multi

tests/smoketests/test_Sample.py

@@ -58,6 +58,16 @@ def test_str_repr_correctness(self, example_pep_cfg_path):
  assert example_pep_cfg_path in str_repr
  assert "Sample '{}'".format(sample["sample_name"]) in str_repr
 
+ @pytest.mark.parametrize("example_pep_cfg_path", EXAMPLE_TYPES, indirect=True)
+ def test_sample_to_yaml_no_path(self, example_pep_cfg_path):
+ """
+ Verify that to_yaml returns representation without requiring a path.
+ """
+ p = Project(cfg=example_pep_cfg_path)
+ for sample in p.samples:
+ yaml_repr = sample.to_yaml()
+ assert "sample_name" in yaml_repr
+
  @pytest.mark.parametrize("example_pep_cfg_path", ["basic"], indirect=True)
  def test_sheet_dict_excludes_private_attrs(self, example_pep_cfg_path):
  """
@@ -77,3 +87,24 @@ def test_pickle_in_samples(self, example_pep_cfg_path):
  unpickled_sample = pickle.loads(pickled_data)
 
  assert sample.to_dict() == unpickled_sample.to_dict()
+
+ @pytest.mark.parametrize("example_pep_cfg_path", ["basic"], indirect=True)
+ def test_equals_samples(self, example_pep_cfg_path):
+ p1 = Project(cfg=example_pep_cfg_path)
+ p2 = Project(cfg=example_pep_cfg_path)
+ s1 = p1.samples[0]
+ s2 = p2.samples[0]
+
+ assert s1 == s2
+
+ @pytest.mark.parametrize("example_pep_cfg_path", ["basic"], indirect=True)
+ def test_not_equals_samples(self, example_pep_cfg_path):
+ p1 = Project(cfg=example_pep_cfg_path)
+ p2 = Project(cfg=example_pep_cfg_path)
+ s1 = p1.samples[0]
+ s2 = p2.samples[0]
+ s3 = p2.samples[1]
+
+ s2.new = "something"
+ assert not s1 == s2
+ assert not s1 == s3

tests/test_Project.py

@@ -330,14 +330,12 @@ def test_missing_sample_name_custom_index(self, example_pep_cfg_noname_path):
  p = Project(cfg=example_pep_cfg_noname_path, sample_table_index="id")
  assert p.sample_name_colname == "id"
 
- @pytest.mark.parametrize(
- "example_pep_cfg_custom_index", ["project_config.yaml"], indirect=True
- )
- def test_sample_name_custom_index(self, example_pep_cfg_custom_index):
+ @pytest.mark.parametrize("example_pep_cfg_path", ["custom_index"], indirect=True)
+ def test_sample_name_custom_index(self, example_pep_cfg_path):
  """
  Verify that sample_name attribute becomes st_index from cfg
  """
- p = Project(cfg=example_pep_cfg_custom_index)
+ p = Project(cfg=example_pep_cfg_path)
  assert p.sample_name_colname == "NOT_SAMPLE_NAME"
  assert p.samples[0].sample_name == "frog_1"
 
@@ -726,3 +724,21 @@ def test_sample_len(self, example_pep_cfg_path):
  """
  p = Project(cfg=example_pep_cfg_path)
  assert len(p.samples[0]) == 4
+
+ @pytest.mark.parametrize("example_pep_cfg_path", ["subsamples_none"], indirect=True)
+ def test_config_with_subsample_null(self, example_pep_cfg_path):
+ """
+ Tests if config can have value with subsample=null
+ """
+ p = Project(cfg=example_pep_cfg_path)
+ assert p.subsample_table is None
+
+ @pytest.mark.parametrize(
+ "example_pep_cfg_path", ["nextflow_subsamples"], indirect=True
+ )
+ def test_nextflow_subsamples(self, example_pep_cfg_path):
+ """
+ Tests if config can have value with subsample=null
+ """
+ p = Project(cfg=example_pep_cfg_path)
+ assert isinstance(p, Project)