Features/fix mvp 114

.travis.yml

@@ -1,69 +1,81 @@
 language: perl
-perl:
-  - "5.14"
-  - "5.26.2"
-services:
-  - mysql
-env:
-  - COVERALLS=true  DB=mysql
-addons:
-  apt:
-    update: true      
-    packages:
-      - unzip
-      - sendmail
-      - graphviz
-      - emboss
-before_install:
-  - git clone --depth 1 https://github.com/Ensembl/ensembl-git-tools.git
-  - export PATH=$PATH:$PWD/ensembl-git-tools/bin
-  - export ENSEMBL_BRANCH=master
-  - export SECONDARY_BRANCH=main
-  - echo "TRAVIS_BRANCH=$TRAVIS_BRANCH"
-  - if [[ $TRAVIS_BRANCH =~ ^release\/[0-9]+$ ]]; then export ENSEMBL_BRANCH=$TRAVIS_BRANCH; export SECONDARY_BRANCH=$TRAVIS_BRANCH; fi
-  - echo "ENSEMBL_BRANCH=$ENSEMBL_BRANCH"
-  - echo "SECONDARY_BRANCH=$SECONDARY_BRANCH"
-  - git-ensembl --clone --branch $ENSEMBL_BRANCH --secondary_branch $SECONDARY_BRANCH --depth 1 ensembl-test
-  - git-ensembl --clone --branch $ENSEMBL_BRANCH --secondary_branch $SECONDARY_BRANCH --depth 1 ensembl
-  - git-ensembl --clone --branch $ENSEMBL_BRANCH --secondary_branch $SECONDARY_BRANCH --depth 1 ensembl-compara
-  - git-ensembl --clone --branch $ENSEMBL_BRANCH --secondary_branch $SECONDARY_BRANCH --depth 1 ensembl-datacheck
-  - git-ensembl --clone --branch $ENSEMBL_BRANCH --secondary_branch $SECONDARY_BRANCH --depth 1 ensembl-variation
-  - git-ensembl --clone --branch $ENSEMBL_BRANCH --secondary_branch $SECONDARY_BRANCH --depth 1 ensembl-metadata
-  - git-ensembl --clone --branch $ENSEMBL_BRANCH --secondary_branch $SECONDARY_BRANCH --depth 1 ensembl-funcgen
-  - git-ensembl --clone --branch master --secondary_branch main --depth 1 ensembl-hive
-  - git-ensembl --clone --branch master --secondary_branch main --depth 1 ensembl-orm
-  - git-ensembl --clone --branch master --secondary_branch main --depth 1 ensembl-taxonomy
-  - git clone --branch 1.9 --depth 1 https://github.com/samtools/htslib.git
-  - git clone --branch release-1-6-924 --depth 1 https://github.com/bioperl/bioperl-live.git
-  - cd htslib
-  - make
-  - export HTSLIB_DIR=$(pwd -P)
-  - cd ..
-install:
-  - cpanm --sudo -v --installdeps --with-recommends --notest --cpanfile ensembl/cpanfile .
-  - cpanm --sudo -v --installdeps --notest --cpanfile ensembl-hive/cpanfile .
-  - cpanm --sudo -v --installdeps --notest --cpanfile ensembl-datacheck/cpanfile .
-  - export PERL5LIB=$PERL5LIB:$PWD/bioperl-live
-  - cpanm --sudo -v --installdeps --notest .
-  - cpanm --sudo -n Devel::Cover::Report::Coveralls
-  - cp travisci/MultiTestDB.conf.travisci  modules/t/MultiTestDB.conf
-  - mysql -u root -h localhost -e 'GRANT ALL PRIVILEGES ON *.* TO "travis"@"%"'
-script:
-  - ./travisci/harness.sh
+os: linux
 jobs:
   include:
-    - language: python
-      python: 3.8
+    - name: "Perl Job"
+      perl: "5.26.2"
+      services:
+        - mysql
+      env:
+        - COVERALLS=true  DB=mysql
+      addons:
+        apt:
+          update: true
+          packages:
+            - unzip
+            - sendmail
+            - graphviz
+            - emboss
+            - libkyotocabinet-dev
+      before_install:
+        - git clone --depth 1 https://github.com/Ensembl/ensembl-git-tools.git
+        - export PATH=$PATH:$PWD/ensembl-git-tools/bin
+        - export ENSEMBL_BRANCH=master
+        - export SECONDARY_BRANCH=main
+        - echo "TRAVIS_BRANCH=$TRAVIS_BRANCH"
+        - if [[ $TRAVIS_BRANCH =~ ^release\/[0-9]+$ ]]; then export ENSEMBL_BRANCH=$TRAVIS_BRANCH; export SECONDARY_BRANCH=$TRAVIS_BRANCH; fi
+        - echo "ENSEMBL_BRANCH=$ENSEMBL_BRANCH"
+        - echo "SECONDARY_BRANCH=$SECONDARY_BRANCH"
+        - git-ensembl --clone --branch $ENSEMBL_BRANCH --secondary_branch $SECONDARY_BRANCH --depth 1 ensembl-test
+        - git-ensembl --clone --branch $ENSEMBL_BRANCH --secondary_branch $SECONDARY_BRANCH --depth 1 ensembl
+        - git-ensembl --clone --branch $ENSEMBL_BRANCH --secondary_branch $SECONDARY_BRANCH --depth 1 ensembl-compara
+        - git-ensembl --clone --branch $ENSEMBL_BRANCH --secondary_branch $SECONDARY_BRANCH --depth 1 ensembl-datacheck
+        - git-ensembl --clone --branch $ENSEMBL_BRANCH --secondary_branch $SECONDARY_BRANCH --depth 1 ensembl-variation
+        - git-ensembl --clone --branch $ENSEMBL_BRANCH --secondary_branch $SECONDARY_BRANCH --depth 1 ensembl-metadata
+        - git-ensembl --clone --branch $ENSEMBL_BRANCH --secondary_branch $SECONDARY_BRANCH --depth 1 ensembl-funcgen
+        - git-ensembl --clone --branch master --secondary_branch main --depth 1 ensembl-hive
+        - git-ensembl --clone --branch master --secondary_branch main --depth 1 ensembl-orm
+        - git-ensembl --clone --branch master --secondary_branch main --depth 1 ensembl-taxonomy
+        - git clone --branch 1.9 --depth 1 https://github.com/samtools/htslib.git
+        - git clone --branch release-1-6-924 --depth 1 https://github.com/bioperl/bioperl-live.git
+        - cd htslib
+        - make
+        - export HTSLIB_DIR=$(pwd -P)
+        - mysql -e "SET GLOBAL local_infile=1;"
+        - cd ..
+      install:
+        - cpanm --sudo -v --installdeps --with-recommends --notest --cpanfile ensembl/cpanfile .
+        - cpanm --sudo -v --installdeps --notest --cpanfile ensembl-hive/cpanfile .
+        - cpanm --sudo -v --installdeps --notest --cpanfile ensembl-datacheck/cpanfile .
+        - export PERL5LIB=$PERL5LIB:$PWD/bioperl-live
+        - cpanm travisci/kyotocabinet-perl-1.20.tar.gz
+        - cpanm --sudo -v --installdeps --notest .
+        - cpanm --sudo -n Devel::Cover::Report::Coveralls
+        - cp travisci/MultiTestDB.conf.travisci  modules/t/MultiTestDB.conf
+        - mysql -u root -h localhost -e 'GRANT ALL PRIVILEGES ON *.* TO "travis"@"%"'
+      script:
+        - ./travisci/harness.sh
+
+    - name: "Python Job"
+      language: python
+      python:
+        - "3.10"
+        - "3.11"
+      services:
+        - mysql
+      env:
+        - COVERALLS=true  DB=mysql
       install:
-        - pip install -e .
         - pip install -r requirements-test.txt
+        - pip install -e .
+      before_script:
+        - mysql -e "SET GLOBAL local_infile=1;"
       script:
         - pytest src/python/test
 
-
 notifications:
   email:
     on_success: always
     on_failure: always
   slack:
-    secure: BkrSPAkOM5aTOpeyO9vZnHdZ0LF1PLk0r2HtcXN2eTMyiHoGXkl6VUjdAL8EkzI4gunW2GProdSIjHpf60WdiEmKAulMdJRI+xyUbuxnY31mwiikS9HYwqmPBbMTf0Mh2pMBngZRFs+gaFZDUMTfLfp+8MQfU1R54yb6hPuVt5I=
+    secure: BkrSPAkOM5aTOpeyO9vZnHdZ0LF1PLk0r2HtcXN2eTMyiHoGXkl6VUjdAL8EkzI4gunW2GProdSIjHpf60WdiEmKAulMdJRI+xyUbuxnY31mwiikS9HYwqmPBbMTf0Mh2pMBngZRFs+gaFZDUMTfLfp+8MQfU1R54yb6hPuVt5I=

cpanfile

@@ -14,7 +14,7 @@ requires 'File::Slurp';
 requires 'Log::Log4perl';
 requires 'XML::Simple';
 requires 'Time::Duration';
-requires 'Tie::LevelDB';
 requires 'IO::Zlib';
 requires 'File::Temp';
 requires 'Fcntl';
+requires 'KyotoCabinet';

modules/Bio/EnsEMBL/Production/Pipeline/AlphaFold/CreateAlphaDB.pm

@@ -33,12 +33,12 @@
 
  This module prepares a DB with a mapping from Uniprot accession to related
  Alphafold data (Alphafold accession, protein start, end). The DB is created on
- disk in LevelDB format.
+ disk in KyotoCabinet format.
 
 =head1 DESCRIPTION
 
  - We expect the file accession_ids.csv to be available
- - We go through the file and build a LevelDB mapping the Uniprot accession to the Alphafold data
+ - We go through the file and build a DB mapping the Uniprot accession to the Alphafold data
 
 =cut
 
@@ -49,7 +49,7 @@ use strict;
 
 use parent 'Bio::EnsEMBL::Production::Pipeline::Common::Base';
 use Bio::EnsEMBL::Utils::Exception qw(throw info);
-use Tie::LevelDB;
+use KyotoCabinet;
 use File::Temp 'tempdir';
 
 
@@ -66,7 +66,7 @@ sub run {
 
     throw ("Data file not found: '$map_file' on host " . `hostname`) unless -f $map_file;
 
-    my $idx_dir = $self->param_required('alphafold_db_dir') . '/uniprot-to-alpha.leveldb';
+    my $idx_dir = $self->param_required('alphafold_db_dir') . '/uniprot-to-alphafold';
     if (-d $idx_dir) {
         system(qw(rm -rf), $idx_dir);
     }
@@ -78,33 +78,42 @@ sub run {
         $copy_to = $idx_dir;
         $idx_dir = tempdir(DIR => '/dev/shm/');
     }
-
-    tie(my %idx, 'Tie::LevelDB', $idx_dir)
-        or die "Error trying to tie Tie::LevelDB $idx_dir: $!";
+
+    my $db = new KyotoCabinet::DB;
+
+    # Set 4 GB mmap size
+    my $mapsize_gb = 4 << 30;
+
+    # Open the DB
+    # Open as the exclusive writer, truncate if it exists, otherwise create the DB
+    # Open the database as a file hash DB, 600M buckets, 4GB mmap, linear option for
+    # hash collision handling. These are tuned for write speed and for approx. 300M entries.
+    # As with a regular Perl hash, a duplicate entry will overwrite the previous
+    # value.
+    $db->open("$idx_dir/uniprot-to-alphafold.kch#bnum=600000000#msiz=$mapsize_gb#opts=l",
+        $db->OWRITER | $db->OCREATE | $db->OTRUNCATE
+    ) or die "Error opening DB: " . $db->error();
 
     my $map;
     open($map, '<', $map_file) or die "Opening map file $map_file failed: $!";
 
-    # A line from accession_ids.csv looks like this:
-    # Uniprot accession, hit start, hit end, Alphafold accession, Alphafold version
-    # A0A2I1PIX0,1,200,AF-A0A2I1PIX0-F1,4
-    # Currently, all entries in this file have a unique uniprot accession and
-    # have a hit starting at 1
-
     while (my $line = <$map>) {
+        chomp $line;
+        # A line from accession_ids.csv looks like this:
+        # Uniprot accession, hit start, hit end, Alphafold accession, Alphafold version
+        # A0A2I1PIX0,1,200,AF-A0A2I1PIX0-F1,4
+        # Currently, all entries in this file have a unique uniprot accession and
+        # have a hit starting at 1
         unless ($line =~ /^\w+,\d+,\d+,[\w_-]+,\d+$/) {
-            chomp $line;
-            warn "Data error. Line is not what we expect: '$line'";
-            next;
+            die "Data error. Line is not what we expect: '$line'";
         }
         my @x = split(",", $line, 2);
 
-        # This is the DB write operation. Tie::LevelDB will croak on errors (e.g. disk full)
-        $idx{$x[0]} = $x[1];
+        # This is the DB write operation.
+        $db->set($x[0], $x[1]) or die "Error inserting data: " . $db->error();
     }
 
-    close($map);
-    untie %idx;
+    $db->close() or die "Error closing DB: " . $db->error();
 
     if ($copy_back) {
         system (qw(cp -r), $idx_dir, $copy_to);

modules/Bio/EnsEMBL/Production/Pipeline/AlphaFold/CreateUniparcDB.pm

@@ -32,12 +32,12 @@
 =head1 SYNOPSIS
 
  This module prepares a DB with a mapping from Uniparc accession to Uniprot
- accession. The DB is created on disk in LevelDB format.
+ accession. The DB is created on disk in KyotoCabinet format.
 
 =head1 DESCRIPTION
 
  - We expect the file idmapping_selected.tab.gz to be available
- - We go through the file and build a LevelDB mapping the Uniparc accessions to Uniprot accessions
+ - We go through the file and build a DB mapping the Uniparc accessions to Uniprot accessions
 
 =cut
 
@@ -49,7 +49,7 @@ use strict;
 use parent 'Bio::EnsEMBL::Production::Pipeline::Common::Base';
 
 use Bio::EnsEMBL::Utils::Exception qw(throw info);
-use Tie::LevelDB;
+use KyotoCabinet;
 use IO::Zlib;
 use File::Temp 'tempdir';
 
@@ -66,7 +66,7 @@ sub run {
 
     throw ("Data file not found: '$map_file' on host " . `hostname`) unless -f $map_file;
 
-    my $idx_dir = $self->param_required('uniparc_db_dir') . '/uniparc-to-uniprot.leveldb';
+    my $idx_dir = $self->param_required('uniparc_db_dir') . '/uniparc-to-uniprot';
     if (-d $idx_dir) {
         system(qw(rm -rf), $idx_dir);
     }
@@ -79,8 +79,21 @@ sub run {
         $idx_dir = tempdir(DIR => '/dev/shm/');
     }
 
-    tie(my %idx, 'Tie::LevelDB', $idx_dir)
-        or die "Error trying to tie Tie::LevelDB $idx_dir: $!";
+    my $db = new KyotoCabinet::DB;
+
+    # Set 4 GB mmap size
+    my $mapsize_gb = 4 << 30;
+
+    # Open the DB
+    # Open as the exclusive writer, truncate if it exists, otherwise create the DB
+    # Open the database as a file hash DB, 600M buckets, 4GB mmap, linear option for
+    # hash collision handling. These are tuned for write speed and for approx. 300M entries.
+    # Uniparc has 251M entries at the moment.
+    # As with a regular Perl hash, a duplicate entry will overwrite the previous
+    # value.
+    $db->open("$idx_dir/uniparc-to-uniprot.kch#bnum=600000000#msiz=$mapsize_gb#opts=l",
+        $db->OWRITER | $db->OCREATE | $db->OTRUNCATE
+    ) or die "Error opening DB: " . $db->error();
 
     my $map = new IO::Zlib;
     $map->open($map_file, 'rb') or die "Opening map file $map_file with IO::Zlib failed: $!";
@@ -90,22 +103,27 @@ sub run {
     # We pick out the Uniparc accession and Uniprot accession
     # index[10] (Uniparc): UPI00003B0FD4; index[0] (Uniprot): Q6GZX4
     my $line;
+
     while ($line = <$map>) {
+        chomp $line;
         unless ($line =~ /^\w+\t[[:print:]\t]+$/) {
-                warn "Data error: Line is not what we expect: '$line'";
-                next;
+            die "Data error: Uniparc accession is not what we expect: '$line'";
         }
         my @x = split("\t", $line, 12);
         unless ($x[10] and $x[10] =~ /^UPI\w+$/) {
-            warn "Data error: Uniparc accession is not what we expect: '$line'";
-            next;
+            die "Data error: Uniparc accession is not what we expect: '$line'";
+        }
+        # This is the DB write operation.
+        my $oldval;
+        if ($oldval = $db->get($x[10])) {
+            $db->set($x[10], "$oldval\t" . $x[0]) or die "Error inserting data: " . $db->error();
+        } else {
+            $db->set($x[10], $x[0]) or die "Error inserting data: " . $db->error();
         }
-        # This is the DB write operation. Tie::LevelDB will croak on errors (e.g. disk full)
-        $idx{$x[10]} = $x[0];
     }
 
     $map->close;
-    untie %idx;
+    $db->close() or die "Error closing DB: " . $db->error();
 
     if ($copy_back) {
         system (qw(cp -r), $idx_dir, $copy_to);

modules/Bio/EnsEMBL/Production/Pipeline/AlphaFold/InsertProteinFeatures.pm

@@ -164,7 +164,7 @@ sub run {
             -db            => 'alphafold',
             -db_version    => $alpha_version,
             -db_file       => $self->param('db_dir') . '/accession_ids.csv',
-            -display_label => 'AlphaFold DB import',
+            -display_label => 'AFDB-ENSP mapping',
             -displayable   => '1',
             -description   => 'Protein features based on AlphaFold predictions, mapped with GIFTS or UniParc'
     );

modules/Bio/EnsEMBL/Production/Pipeline/FileDump/Geneset_GFF3.pm

@@ -266,7 +266,8 @@ sub Bio::EnsEMBL::Transcript::summary_as_hash {
   $summary{'transcript_support_level'} = $self->tsl if $self->tsl;
 
   my @tags;
-  push(@tags, 'basic') if $self->gencode_basic();
+  push(@tags, 'gencode_basic') if $self->gencode_basic();
+  push(@tags, 'gencode_primary') if $self->gencode_primary();
   push(@tags, 'Ensembl_canonical') if $self->is_canonical();
 
   # A transcript can have different types of MANE-related attributes (MANE_Select, MANE_Plus_Clinical)

modules/Bio/EnsEMBL/Production/Pipeline/GFF3/DumpFile.pm

@@ -279,7 +279,8 @@ sub Bio::EnsEMBL::Transcript::summary_as_hash {
   $summary{'transcript_support_level'} = $self->tsl if $self->tsl;
 
   my @tags;
-  push(@tags, 'basic') if $self->gencode_basic();
+  push(@tags, 'gencode_basic') if $self->gencode_basic();
+  push(@tags, 'gencode_primary') if $self->gencode_primary();
   push(@tags, 'Ensembl_canonical') if $self->is_canonical();
 
   # A transcript can have different types of MANE-related attributes (MANE_Select, MANE_Plus_Clinical)

modules/Bio/EnsEMBL/Production/Pipeline/GTF/DumpFile.pm

@@ -383,7 +383,8 @@ feature for the position of this on the genome
 - cds_start_NF: the coding region start could not be confirmed
 - mRNA_end_NF: the mRNA end could not be confirmed
 - mRNA_start_NF: the mRNA start could not be confirmed.
-- basic: the transcript is part of the gencode basic geneset
+- gencode_basic: the transcript is part of the gencode basic geneset
+- gencode_primary: the transcript is part of the gencode primary geneset
 
 Comments
 

modules/Bio/EnsEMBL/Production/Pipeline/Ga4ghChecksum/ChecksumGenerator.pm

@@ -218,7 +218,7 @@ sub all_hashes {
     } ## end foreach my $slice (@slices)
 
     for my $seq_type (keys %$batch) {
-        for my $attrib_table (keys $batch->{$seq_type}) {
+        for my $attrib_table (keys %{$batch->{$seq_type}}) {
             $attribute_adaptor->store_batch_on_Object($attrib_table, $batch->{$seq_type}->{$attrib_table}, 1000);
         }
     }

modules/Bio/EnsEMBL/Production/Pipeline/JSON/JsonRemodeller.pm

@@ -292,7 +292,10 @@ sub merge_xrefs {
         $obj->{$dbname} = [];
       }
       for my $ann ( @{ $subobj->{$dbname} } ) {
-        push $obj->{$dbname}, $self->copy_hash($ann);
+        if (ref($obj->{$dbname}) ne 'ARRAY') {
+          $obj->{$dbname} = [];
+        }
+        push @{ $obj->{$dbname} }, $self->copy_hash($ann);
       }
     }
   }