fix merge conflicts

DEV-GUIDE.md

@@ -13,6 +13,8 @@ This supersedes [Fragalysis-API](https://github.com/xchem/fragalysis-api).
 Project dependencies are defined in the `pyproject.toml` file.
 
 You will need to use Python 3.10 or 3.11 (a requirement of the `pyproject.toml` file).
+Python 3.12 cannot currently be used.
+
 If you prefer to use [conda] you can create a Python 3.10 environment using the
 `environment.yaml` in this project, otherwise, if you have Python 3.10 or 3.11,
 you can create an environment using the built-in `venv` module: -
@@ -21,7 +23,8 @@ you can create an environment using the built-in `venv` module: -
     source venv/bin/activate
     pip install --upgrade pip
 
-Make sure you create the venv using Python 3.10 or 3.11.
+Make sure you create the venv using Python 3.10 or 3.11 (e.g. change the first command to `python3.11 -m venv venv`
+if needed).
 
 From your clean virtual environment you can now install the run-time and development
 dependencies like this: -
@@ -50,6 +53,26 @@ The main tools are implemented as the following Python modules:
 - Collator: xchemalign/collator.py
 - Aligner: xchemalign/aligner.py
 
+## Rollout
+
+There is an environment at Diamond where users user the XChem align code on their data.
+This can be found on the Diamond file system at `/dls/science/groups/i04-1/software/xchem-align`.
+To roll out a new version of this:
+
+1. Check that the repos are up to date on the `master` (XCA) and `main` (LNA) branches.
+2. Test locally
+3. Tag the XCA repo and push the tag: `git tag 1.2.3` and `git push origin 1.2.3` (using the appropriate tag number)
+4. ssh to Diamond and move into the `/dls/science/groups/i04-1/software/xchem-align` dir
+5. `git pull` - update the repo
+6. `git tag` - check you have the expected tag
+7. `rm -rf env_xchem_align` - remove the old conda environment
+8. `conda deactivate` - deactivate the current conda env (if necessary)
+9. `conda env create -f environment.yaml -p env_xchem_align` - create the new conda environment
+
+NOTE: the repo MUST be tagged before rolling out to users. Step 3 does this and is assumed to be done by the
+developers. If you want to roll out a new environment and the repo is not tagged (step 3) then the commands in step 3
+must instead be performed after step 5 and before step 6.
+
 ---
 
 [conda]: https://docs.conda.io/en/latest/

USER-GUIDE.md

@@ -35,21 +35,19 @@ If you won't run this at Diamond, you will first have to set up your environment
 
 Uploading data from Diamond Light Source is as simple as running a few commands.
 
-**EVERY TIME** you log in to run any XChemAlign tools:
+**ONLY THE VERY FIRST TIME** you must run this (the scripts below will fail and tell you).  _(It enables your linux account to read version info from the code's Git repository.)_:
 
 ```commandline
-source /dls/science/groups/i04-1/software/xchem-align/act
-conda activate /dls/science/groups/i04-1/software/xchem-align/env_xchem_align
+git config --global --add safe.directory /dls/science/groups/i04-1/software/xchem-align
 ```
 
-**THE VERY FIRST TIME** you must run this (the scripts above will fail and tell you).  _(It enables your linux account to read version info from the code's Git repository.)_:
+**EVERY TIME** you log in to run any XChemAlign tools:
 
 ```commandline
-git config --global --add safe.directory /dls/science/groups/i04-1/software/xchem-align
+source /dls/science/groups/i04-1/software/xchem-align/act
+conda activate /dls/science/groups/i04-1/software/xchem-align/env_xchem_align
 ```
 
-
-
 ## 2. Declaring things
 
 In order to run XChemAlign you will need to create two files:

src/xchemalign/aligner.py

@@ -19,6 +19,8 @@
 import yaml
 import gemmi
 
+from rich.traceback import install
+
 # Local alignment imports
 from ligand_neighbourhood_alignment import constants as lna_constants
 from ligand_neighbourhood_alignment.align_xmaps import _align_xmaps
@@ -63,6 +65,8 @@
 from xchemalign.utils import Constants
 from xchemalign.pdb_xtal import PDBXtal
 
+install(show_locals=True)
+
 
 def try_make(path):
     if not Path(path).exists():
@@ -212,6 +216,14 @@ def validate(self):
         elif not self.version_dir.is_dir():
             self._log_error("version dir {} is not a directory".format(self.version_dir))
         else:
+            output_meta_path = self.version_dir / Constants.METADATA_ALIGN_FILENAME
+            if output_meta_path.exists():
+                self._log_error(
+                    "aligner output {} already exists. You must run aligner on clean output from collator.".format(
+                        str(output_meta_path)
+                    )
+                )
+
             p = self.metadata_file
             if not p.exists():
                 self._log_error("metadata file {} does not exist. Did the collator step run successfully?".format(p))
@@ -444,23 +456,16 @@ def _perform_alignments(self, meta):
             working_fs_model = fs_model
 
         if working_fs_model.assembly_landmarks.exists():
-            assembly_landmarks = ah.load_yaml(
-                working_fs_model.assembly_landmarks,
-                ah.dict_to_assembly_landmarks
-            )
+            assembly_landmarks = ah.load_yaml(working_fs_model.assembly_landmarks, ah.dict_to_assembly_landmarks)
         else:
             assembly_landmarks = {}
 
         # Get the assembly transforms
-        if working_fs_model.assembly_landmarks.exists():
-            assembly_transforms = ah.load_yaml(
-                working_fs_model.assembly_landmarks,
-                lambda x: x
-            )
+        if working_fs_model.assembly_transforms.exists():
+            assembly_transforms = ah.load_yaml(working_fs_model.assembly_transforms, lambda x: x)
         else:
             assembly_transforms = {}
 
-
         # Run the update
         updated_fs_model = _update(
             fs_model,
@@ -482,7 +487,6 @@ def _perform_alignments(self, meta):
             assembly_landmarks,
             assembly_transforms,
             self.version_dir.name[7:],
-
         )
 
         # Update the metadata_file with aligned file locations and site information
@@ -646,18 +650,32 @@ def _perform_alignments(self, meta):
                             aligned_event_map_path = version_output.aligned_event_maps[site_id]
                             aligned_xmap_path = version_output.aligned_xmaps[site_id]
                             aligned_diff_map_path = version_output.aligned_diff_maps[site_id]
+
+                            aligned_crystallographic_event_map_path = (
+                                version_output.aligned_event_maps_crystallographic[site_id]
+                            )
+                            aligned_crystallographic_xmap_path = version_output.aligned_xmaps_crystallographic[site_id]
+                            aligned_crystallographic_diff_map_path = version_output.aligned_diff_maps_crystallographic[
+                                site_id
+                            ]
+
                             aligned_version_output[site_id] = {
                                 Constants.META_AIGNED_STRUCTURE: aligned_structure_path,
                                 # Constants.META_AIGNED_ARTEFACTS: aligned_artefacts_path,
                                 # Constants.META_AIGNED_EVENT_MAP: aligned_event_map_path,
                                 Constants.META_AIGNED_X_MAP: aligned_xmap_path,
                                 Constants.META_AIGNED_DIFF_MAP: aligned_diff_map_path,
+                                Constants.META_AIGNED_CRYSTALLOGRAPHIC_X_MAP: aligned_crystallographic_xmap_path,
+                                Constants.META_AIGNED_CRYSTALLOGRAPHIC_DIFF_MAP: aligned_crystallographic_diff_map_path,
                             }
                             # if the event map is present then include it in the output
                             if event_map_present:
                                 aligned_version_output[site_id][
                                     Constants.META_AIGNED_EVENT_MAP
                                 ] = aligned_event_map_path
+                                aligned_version_output[site_id][
+                                    Constants.META_AIGNED_CRYSTALLOGRAPHIC_EVENT_MAP
+                                ] = aligned_crystallographic_event_map_path
                     i += 1
 
         ## Add the reference alignments

src/xchemalign/collator.py

@@ -640,7 +640,9 @@ def read_versions(self):
             else:
                 break
         if version == 1:
-            self.logger.error("No version directory found. Please create one named upload_1")
+            self.logger.error(
+                "No version directory found. Please create one named", str(self.output_path / 'upload_1')
+            )
             return None
 
         # the working version dir is one less than the current value
@@ -873,6 +875,7 @@ def _copy_files(self, meta):
                     data_to_add[Constants.META_XTAL_PDB] = {
                         Constants.META_FILE: str(fdata[1]),
                         Constants.META_SHA256: fdata[2],
+                        Constants.META_SOURCE_FILE: str(fdata[0]),
                     }
                     # copy MTZ file
                     fdata = files_to_copy.get(Constants.META_XTAL_MTZ)
@@ -886,6 +889,7 @@ def _copy_files(self, meta):
                             data_to_add[Constants.META_XTAL_MTZ] = {
                                 Constants.META_FILE: str(fdata[1]),
                                 Constants.META_SHA256: fdata[2],
+                                Constants.META_SOURCE_FILE: str(fdata[0]),
                             }
                     fdata = files_to_copy.get(Constants.META_XTAL_CIF)
 
@@ -900,6 +904,7 @@ def _copy_files(self, meta):
                             data_to_add[Constants.META_XTAL_CIF] = {
                                 Constants.META_FILE: str(fdata[1]),
                                 Constants.META_SHA256: fdata[2],
+                                Constants.META_SOURCE_FILE: str(fdata[0]),
                             }
                             try:
                                 mols = utils.gen_mols_from_cif(str(self.output_path / fdata[1]))
@@ -946,6 +951,7 @@ def _copy_files(self, meta):
                                 data = {
                                     Constants.META_FILE: str(attested_ligand_event_data[1]),
                                     Constants.META_SHA256: attested_ligand_event_data[2],
+                                    Constants.META_SOURCE_FILE: str(attested_ligand_event_data[0]),
                                     Constants.META_PROT_MODEL: ligand_key[0],
                                     Constants.META_PROT_CHAIN: ligand_key[1],
                                     Constants.META_PROT_RES: ligand_key[2],

src/xchemalign/utils.py

@@ -77,6 +77,7 @@ class Constants:
     META_REFERENCE = "reference"
     META_FILE = "file"
     META_SHA256 = "sha256"
+    META_SOURCE_FILE = "source_file"
     META_XTAL_FILES = "crystallographic_files"
     META_ALIGNED_FILES = "aligned_files"
     META_REFERENCE_ALIGNMENTS = "reference_aligned_files"
@@ -97,6 +98,9 @@ class Constants:
     META_AIGNED_EVENT_MAP = "event_map"
     META_AIGNED_X_MAP = "sigmaa_map"
     META_AIGNED_DIFF_MAP = "diff_map"
+    META_AIGNED_CRYSTALLOGRAPHIC_EVENT_MAP = "event_map_crystallographic"
+    META_AIGNED_CRYSTALLOGRAPHIC_X_MAP = "sigmaa_map_crystallographic"
+    META_AIGNED_CRYSTALLOGRAPHIC_DIFF_MAP = "diff_map_crystallographic"
     META_CONFORMER_SITES = "conformer_sites"
     META_CONFORMER_SITE_NAME = "name"
     META_CONFORMER_SITE_REFERENCE_LIG = "lig_ref"