Provenance2 into paper

.github/workflows/unit-testing.Listeria.Kraken1.yml

@@ -2,6 +2,7 @@
 on: 
   push:
     branches: [master, dev, validate-taxonomy]
+  pull_request:
 name: Listeria-with-Kraken1
 
 env:
@@ -41,18 +42,10 @@ jobs:
           tree $(realpath .)
       - name: install-edirect
         run:  |
-          sudo apt-get install ncbi-entrez-direct
-          echo "installed edirect the apt way"
-          exit
-          cd $HOME
-          perl -MNet::FTP -e '$ftp = new Net::FTP("ftp.ncbi.nlm.nih.gov", Passive => 1); $ftp->login; $ftp->binary; $ftp->get("/entrez/entrezdirect/edirect.tar.gz");'
-          gunzip -cv edirect.tar.gz | tar xf -
-          rm -v edirect.tar.gz
-          echo $GITHUB_WORKSPACE/edirect >> $GITHUB_PATH
+          sh -c "$(curl -fsSL https://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/install-edirect.sh)"
+          echo $HOME/edirect >> $GITHUB_PATH
           echo $GITHUB_WORKSPACE/Kalamari/bin >> $GITHUB_PATH
-          #export PATH=${PATH}:$HOME/edirect >& /dev/null || setenv PATH "${PATH}:$HOME/edirect"
-          yes Y | ./edirect/setup.sh
-          tree edirect
+          tree $HOME/edirect
       - name: check-env
         run:  echo "$PATH"
       - name: select for only Listeria

.github/workflows/unit-testing.Yersinia.Kraken2.yml

@@ -2,6 +2,7 @@
 on: 
   push:
     branches: [master, dev, validate-taxonomy]
+  pull_request:
 name: Genera-with-Kraken2
 
 env:
@@ -34,6 +35,12 @@ jobs:
       - name: env check
         run:  |
           echo $PATH | tr ':' '\n' | sort
+      - name: install-edirect
+        run:  |
+          sh -c "$(curl -fsSL https://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/install-edirect.sh)"
+          echo $HOME/edirect >> $GITHUB_PATH
+          echo $GITHUB_WORKSPACE/Kalamari/bin >> $GITHUB_PATH
+          tree $HOME/edirect 
       - name: apt-get install
         run:  sudo apt-get install ca-certificates tree jellyfish ncbi-entrez-direct
       - name: select for only for this genus

.github/workflows/unit-testing.yml

@@ -1,6 +1,7 @@
 on: 
   push:
     branches: [master, dev, validate-taxonomy]
+  pull_request:
 name: Pull-down-all-accessions
 
 jobs:
@@ -27,16 +28,10 @@ jobs:
         run:  sudo apt-get install ca-certificates tree
       - name: install-edirect
         run:  |
-          sudo apt-get install ncbi-entrez-direct
-          echo "installed edirect the apt way"
-          exit
-          cd $HOME
-          perl -MNet::FTP -e '$ftp = new Net::FTP("ftp.ncbi.nlm.nih.gov", Passive => 1); $ftp->login; $ftp->binary; $ftp->get("/entrez/entrezdirect/edirect.tar.gz");'
-          gunzip -cv edirect.tar.gz | tar xf -
-          rm -v edirect.tar.gz
-          export PATH=${PATH}:$HOME/edirect >& /dev/null || setenv PATH "${PATH}:$HOME/edirect"
-          yes Y | ./edirect/setup.sh
-          tree edirect
+          sh -c "$(curl -fsSL https://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/install-edirect.sh)"
+          echo $HOME/edirect >> $GITHUB_PATH
+          echo $GITHUB_WORKSPACE/Kalamari/bin >> $GITHUB_PATH
+          tree $HOME/edirect
       - name: check-env
         run:  echo "$PATH"
       - name: download

.github/workflows/validateTaxonomy.yml

@@ -1,6 +1,7 @@
 on: 
   push:
-    branches: [master, dev, validate-taxonomy]
+    branches: [master, dev, esearch-input]
+  pull_request:
 name: Validate taxonomy
 
 jobs:
@@ -27,11 +28,18 @@ jobs:
           echo $PATH
           echo ""
           cat $GITHUB_PATH
+      - name: install taxonkit
+        run:  |
+          wget https://github.com/shenwei356/taxonkit/releases/download/v0.16.0/taxonkit_linux_amd64.tar.gz
+          tar -xvf taxonkit_linux_amd64.tar.gz
+          rm -v taxonkit_linux_amd64.tar.gz
+          chmod +x taxonkit
+          echo $(realpath .) >> $GITHUB_PATH
       - name: build taxonomy
         run:  |
           echo $PATH
-          bash Kalamari/bin/buildTaxonomy.sh
-          bash Kalamari/bin/filterTaxonomy.sh
+          bash -x Kalamari/bin/buildTaxonomy.sh
+          bash -x Kalamari/bin/filterTaxonomy.sh
           ls -lhR Kalamari/share/kalamari-*/taxonomy
       - name: validate taxonomy
         run:  |

.gitignore

@@ -2,3 +2,6 @@ edirect
 share
 paper/paper.html
 paper/paper.doc
+# pixi environments
+.pixi
+*.egg-info

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Lee Katz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,7 +1,6 @@
 # Kalamari
-A database of completed assemblies for metagenomics-related tasks
 
-[![Creative Commons License v4](https://licensebuttons.net/l/by-sa/4.0/88x31.png)](LICENSE.md)
+A database of completed assemblies for metagenomics-related tasks
 
 ## Synopsis
 

bin/buildKraken1.sh

@@ -22,11 +22,16 @@ cp -rv $TAXDIR $DB/taxonomy
 
 # Make --add-to-library more efficient with
 # concatenated fasta files
+export nl=$'\n'
 find $SRC -name '*.fasta.gz' | \
   xargs -n 100 -P 1 bash -c '
     for i in "$@"; do
       gzip -cd $i
     done > $tmpfile
+    echo -ne "ADDING to library:\n  "
+    zgrep "^>" $tmpfile | sed "s/^>//" | tr "$nl" " "
+    echo
+    echo "^^ contents of $tmpfile ^^"
     kraken-build --db $DB --add-to-library $tmpfile
   '
 
@@ -35,3 +40,6 @@ kraken-build --db $DB --build --threads 1
 # Reduce the size of the database
 kraken-build --db $DB --clean
 
+if [ ! -e "$sharedir/kalamari-kraken1" ]; then
+  ln -sv kalamari-kraken "$sharedir/kalamari-kraken1"
+fi

bin/downloadKalamari.pl

@@ -11,7 +11,7 @@
 use IO::Compress::Gzip;
 use version 0.77;
 
-our $VERSION = version->parse("5.6.0");
+our $VERSION = version->parse("5.7.2");
 
 use threads;
 
@@ -167,27 +167,25 @@ sub downloadEntries{
   my $numEntries = scalar(@$entries);
   my @acc = map{$$_{nuccoreAcc}} @$entries;
   logmsg "Downloading ".scalar(@acc)." accessions";
-  my $queryArg = join("[accession] OR ", sort(@acc))."[accession]";
   my $dir = tempdir("download.XXXXXX", DIR=>$$settings{tempdir});
 
+  # Make the input file for efetch
+  my $inputAcc = "$dir/input.acc";
+  open(my $fh, ">", $inputAcc) or die "ERROR: could not write to $inputAcc: $!";
+  print $fh join("\n", @acc)."\n";
+  close $fh;
+
   # Accessions that had errors
   my @err;
 
-  # Get the esearch xml in place for at least one downstream query
-  my $esearchXml = "$dir/esearch.xml";
-  my $esearchCmd = "esearch -db nuccore -query '$queryArg' > $esearchXml";
-  command($esearchCmd);
+  # Get started on the comprehensive assembly file
+  my $outfile = "$dir/all.fasta";
+  logmsg "Downloading all accessions to $outfile using input accessions in $inputAcc";
+  command("efetch -db nuccore -input $inputAcc -format fasta > $dir/all.fasta");
   if($?){
-    die "ERROR running: $esearchCmd: $!";
+    die "ERROR: could not download all accessions";
   }
 
-  # Get started on the assembly file
-  my $outfile = "$dir/all.fasta";
-
-  # Main query: efetch
-  my $efetchCmd = "cat $esearchXml | efetch -format fasta > $outfile";
-  system($efetchCmd);
-
   my $seqsWithVersion = readSeqs($outfile);
   my $seqs = {};
   while(my($acc, $seq) = each(%$seqsWithVersion)){

bin/downloadKalamari.sh

@@ -23,8 +23,8 @@ echo "TEMPDIR is $tempdir" >&2
 echo "OUTDIR  is $outdir_prefix" >&2
 
 TSV="$tempdir/in.tsv"
-cat $thisdir/../src/chromosomes.tsv > $TSV
-cat $thisdir/../src/plasmids.tsv   >> $TSV
+cat $thisdir/../src/chromosomes.tsv       > $TSV
+tail -n +2 $thisdir/../src/plasmids.tsv  >> $TSV
 
 cp -rv $thisdir/../src/taxonomy $tempdir/taxonomy
 

bin/filterTaxonomy.sh

@@ -2,6 +2,11 @@
 
 set -eu
 
+# Check for dependencies
+echo "Check for dependencies"
+which taxonkit
+echo
+
 thisdir=$(dirname $0)
 thisfile=$(basename $0)
 KALAMARI_VER=$(downloadKalamari.pl --version)

paper/mra.md

@@ -47,7 +47,9 @@ Kalamari also contains a custom taxonomy and software for downloading and format
 
 ## Announcement
 
-Public Health laboratories sequence microbial pathogens daily for genomic epidemiology, i.e., to track pathogen spread [@armstrong2019pathogen].
+Public Health laboratories sequence microbial pathogens daily for many applications including genomic epidemiology [@armstrong2019pathogen],
+species identification [@lindsey2023rapid],
+and metagenomic analysis [@huang2017metagenomics].
 Relevant databases exist such as RefSeq [@o2016reference] or The Genome Taxonomy Database (GTDB) [@parks2022gtdb].
 However, due to their so comprehensive nature,
 they are disadvantageous for our specific purposes:
@@ -64,7 +66,7 @@ All chromosomes and plasmids are complete, i.e., no contig breaks,
 and obtained from trusted sources, e.g., FDA-ARGOS [@sichtig2019fda] or the NCTC 3000 collection [@dicks2023nctc3000], or provided and reviewed by a CDC subject matter expert.
 
 We obtained the list of plasmids from the Mob-Suite project [@robertsonMobsuite]
-and clustered them at 97% average nucleotide identity (ANI) [@lindsey2023rapid].
+and clustered them at 97% average nucleotide identity using edlb_ani_mummer v1 with default options [@lindsey2023rapid].
 For each cluster, the taxonomy identifier was raised to the lowest common tier of taxonomy.
 For example, if a cluster of plasmids were identified in both _Escherichia coli_ and _Salmonella enterica_, then taxonomy identifiers for all the plasmids in the cluster were changed to their common family, _Enterobacteriaceae_.
 As a result, any taxonomic signature from these plasmids