move greyhound-core into sourmash

add getset, wip parallel feature flag

wip colors

simpler impl first

parallel hash to color construction

wip

Revert "wip"

This reverts commit d65da76.

must insert small_color into large_color before setting it

trying out a small set impl

try compressing colors inside reduce

size test

use new released vec-collections

update cbindgen

make parallel/sequential more maintainable

some notes on partial serde

start revindex in py

start ffi

first test passing

second test passing

modify colors.update to accept an iter instead of slices

color count tracker

update sourmash.h

blanket implementation for counter_gather

start working on memstorage

niv update

avoid a mut ref in save by using lots of mutexes

fix codecov path fixes

expose InnerStorage

basic test passing in-memory sigs working

revert counter_gather to gather in search.py

lint cleanup

cbindgen fixes

moved MemStorage to #1463

implement signatures()

fix initialization

Makefile

@@ -30,6 +30,8 @@ include/sourmash.h: src/core/src/lib.rs \
                     src/core/src/ffi/minhash.rs \
                     src/core/src/ffi/signature.rs \
                     src/core/src/ffi/nodegraph.rs \
+                    src/core/src/ffi/index/mod.rs \
+                    src/core/src/ffi/index/revindex.rs \
                     src/core/src/errors.rs
 	cd src/core && \
 	RUSTUP_TOOLCHAIN=nightly cbindgen -c cbindgen.toml . -o ../../$@

doc/developer.md

@@ -273,6 +273,13 @@ Some installation issues can be solved by simply removing the intermediate build
 make clean
 ```
 
+## Notes on implementing a new `Index`
+
+- required method: `signatures`
+- after that (if there are more efficient impls):
+  * `find`
+  * `search`/`gather`
+
 ## Contents
 
 ```{toctree}

include/sourmash.h

@@ -52,6 +52,10 @@ typedef struct SourmashKmerMinHash SourmashKmerMinHash;
 
 typedef struct SourmashNodegraph SourmashNodegraph;
 
+typedef struct SourmashRevIndex SourmashRevIndex;
+
+typedef struct SourmashSearchResult SourmashSearchResult;
+
 typedef struct SourmashSignature SourmashSignature;
 
 /**
@@ -302,6 +306,49 @@ SourmashNodegraph *nodegraph_with_tables(uintptr_t ksize,
                                          uintptr_t starting_size,
                                          uintptr_t n_tables);
 
+void revindex_free(SourmashRevIndex *ptr);
+
+const SourmashSearchResult *const *revindex_gather(const SourmashRevIndex *ptr,
+                                                   const SourmashSignature *sig_ptr,
+                                                   double threshold,
+                                                   bool _do_containment,
+                                                   bool _ignore_abundance,
+                                                   uintptr_t *size);
+
+SourmashRevIndex *revindex_new_with_paths(const SourmashStr *const *search_sigs_ptr,
+                                          uintptr_t insigs,
+                                          const SourmashKmerMinHash *template_ptr,
+                                          uintptr_t threshold,
+                                          const SourmashKmerMinHash *const *queries_ptr,
+                                          uintptr_t inqueries,
+                                          bool keep_sigs);
+
+SourmashRevIndex *revindex_new_with_sigs(const SourmashSignature *const *search_sigs_ptr,
+                                         uintptr_t insigs,
+                                         const SourmashKmerMinHash *template_ptr,
+                                         uintptr_t threshold,
+                                         const SourmashKmerMinHash *const *queries_ptr,
+                                         uintptr_t inqueries);
+
+uint64_t revindex_scaled(const SourmashRevIndex *ptr);
+
+const SourmashSearchResult *const *revindex_search(const SourmashRevIndex *ptr,
+                                                   const SourmashSignature *sig_ptr,
+                                                   double threshold,
+                                                   bool do_containment,
+                                                   bool _ignore_abundance,
+                                                   uintptr_t *size);
+
+SourmashSignature **revindex_signatures(const SourmashRevIndex *ptr, uintptr_t *size);
+
+SourmashStr searchresult_filename(const SourmashSearchResult *ptr);
+
+void searchresult_free(SourmashSearchResult *ptr);
+
+double searchresult_score(const SourmashSearchResult *ptr);
+
+SourmashSignature *searchresult_signature(const SourmashSearchResult *ptr);
+
 void signature_add_protein(SourmashSignature *ptr, const char *sequence);
 
 void signature_add_sequence(SourmashSignature *ptr, const char *sequence, bool force);

setup.py

@@ -8,7 +8,10 @@
 
 
 def build_native(spec):
-    cmd = ["cargo", "build", "--manifest-path", "src/core/Cargo.toml", "--lib"]
+    cmd = ["cargo", "build",
+           "--manifest-path", "src/core/Cargo.toml",
+           "--features", "experimental,parallel",
+           "--lib"]
 
     target = "debug"
     if not DEBUG_BUILD:

shell.nix

@@ -19,6 +19,9 @@ in
       cargo-limit
       wasmtime
       wasm-pack
+      cargo-watch
+      cargo-limit
+      rust-cbindgen
     ];
 
     shellHook = ''

src/core/Cargo.toml

@@ -20,13 +20,15 @@ bench = false
 [features]
 from-finch = ["finch"]
 parallel = ["rayon"]
+experimental = []
 
 [dependencies]
 az = "1.0.0"
 backtrace = "0.3.56"
 bytecount = "0.6.0"
 byteorder = "1.4.3"
 cfg-if = "1.0"
+counter = "0.5.2"
 finch = { version = "0.4.1", optional = true }
 fixedbitset = "0.4.0"
 getset = "0.1.1"
@@ -43,6 +45,8 @@ serde_json = "1.0.53"
 primal-check = "0.3.1"
 thiserror = "1.0"
 typed-builder = "0.9.0"
+twox-hash = "1.6.0"
+vec-collections = "0.3.4"
 
 [dev-dependencies]
 assert_matches = "1.3.0"

src/core/cbindgen.toml

@@ -8,6 +8,7 @@ clean = true
 
 [parse.expand]
 crates = ["sourmash"]
+features = ["experimental"]
 
 [enum]
 rename_variants = "QualifiedScreamingSnakeCase"

src/core/src/encodings.rs

@@ -1,12 +1,20 @@
+use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::convert::TryFrom;
+use std::hash::{BuildHasher, BuildHasherDefault, Hash, Hasher};
 use std::iter::Iterator;
 use std::str;
 
+use nohash_hasher::BuildNoHashHasher;
 use once_cell::sync::Lazy;
 
 use crate::Error;
 
+pub type Color = u64;
+pub type Idx = u64;
+type IdxTracker = (vec_collections::VecSet<[Idx; 4]>, u64);
+type ColorToIdx = HashMap<Color, IdxTracker, BuildNoHashHasher<Color>>;
+
 #[allow(non_camel_case_types)]
 #[derive(Debug, Clone, Copy, PartialEq)]
 #[repr(u32)]
@@ -357,3 +365,202 @@ pub const VALID: [bool; 256] = {
     lookup[b'T' as usize] = true;
     lookup
 };
+
+#[derive(Serialize, Deserialize, Default)]
+pub struct Colors {
+    colors: ColorToIdx,
+}
+
+impl Colors {
+    pub fn new() -> Colors {
+        Default::default()
+    }
+
+    /// Given a color and a new idx, return an updated color
+    ///
+    /// This might create a new one, or find an already existing color
+    /// that contains the new_idx
+    ///
+    /// Future optimization: store a count for each color, so we can track
+    /// if there are extra colors that can be removed at the end.
+    /// (the count is decreased whenever a new color has to be created)
+    pub fn update<'a, I: IntoIterator<Item = &'a Idx>>(
+        &mut self,
+        current_color: Option<Color>,
+        new_idxs: I,
+    ) -> Result<Color, Error> {
+        if let Some(color) = current_color {
+            if let Some(idxs) = self.colors.get_mut(&color) {
+                let idx_to_add: Vec<_> = new_idxs
+                    .into_iter()
+                    .filter(|new_idx| !idxs.0.contains(&new_idx))
+                    .collect();
+
+                if idx_to_add.is_empty() {
+                    // Easy case, it already has all the new_idxs, so just return this color
+                    idxs.1 += 1;
+                    Ok(color)
+                } else {
+                    // We need to either create a new color,
+                    // or find an existing color that have the same idxs
+
+                    let mut idxs = idxs.clone();
+                    idxs.0.extend(idx_to_add.into_iter().cloned());
+                    let new_color = Colors::compute_color(&idxs);
+
+                    if new_color != color {
+                        self.colors.get_mut(&color).unwrap().1 -= 1;
+                        if self.colors[&color].1 == 0 {
+                            self.colors.remove(&color);
+                        };
+                    };
+
+                    self.colors
+                        .entry(new_color)
+                        .and_modify(|old_idxs| {
+                            assert_eq!(old_idxs.0, idxs.0);
+                            old_idxs.1 += 1;
+                        })
+                        .or_insert_with(|| (idxs.0, 1));
+                    Ok(new_color)
+                }
+            } else {
+                unimplemented!("throw error, current_color must exist in order to be updated. current_color: {:?}, colors: {:#?}", current_color, &self.colors);
+            }
+        } else {
+            let mut idxs = IdxTracker::default();
+            idxs.0.extend(new_idxs.into_iter().cloned());
+            idxs.1 = 1;
+            let new_color = Colors::compute_color(&idxs);
+            self.colors
+                .entry(new_color)
+                .and_modify(|old_idxs| {
+                    assert_eq!(old_idxs.0, idxs.0);
+                    old_idxs.1 += 1;
+                })
+                .or_insert_with(|| (idxs.0, 1));
+            Ok(new_color)
+        }
+    }
+
+    fn compute_color(idxs: &IdxTracker) -> Color {
+        let s = BuildHasherDefault::<twox_hash::Xxh3Hash128>::default();
+        let mut hasher = s.build_hasher();
+        idxs.0.hash(&mut hasher);
+        hasher.finish()
+    }
+
+    pub fn len(&self) -> usize {
+        self.colors.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.colors.is_empty()
+    }
+
+    pub fn contains(&self, color: Color, idx: Idx) -> bool {
+        if let Some(idxs) = self.colors.get(&color) {
+            idxs.0.contains(&idx)
+        } else {
+            false
+        }
+    }
+
+    pub fn indices(&self, color: &Color) -> Indices {
+        // TODO: what if color is not present?
+        Indices {
+            iter: self.colors.get(&color).unwrap().0.iter(),
+        }
+    }
+
+    pub fn retain<F>(&mut self, f: F)
+    where
+        F: FnMut(&Color, &mut IdxTracker) -> bool,
+    {
+        self.colors.retain(f)
+    }
+}
+
+pub struct Indices<'a> {
+    iter: vec_collections::VecSetIter<core::slice::Iter<'a, Idx>>,
+}
+
+impl<'a> Iterator for Indices<'a> {
+    type Item = &'a Idx;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.iter.next()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn colors_update() {
+        let mut colors = Colors::new();
+
+        let color = colors.update(None, &[1_u64]).unwrap();
+        assert_eq!(colors.len(), 1);
+
+        dbg!("update");
+        let new_color = colors.update(Some(color), &[1_u64]).unwrap();
+        assert_eq!(colors.len(), 1);
+        assert_eq!(color, new_color);
+
+        dbg!("upgrade");
+        let new_color = colors.update(Some(color), &[2_u64]).unwrap();
+        assert_eq!(colors.len(), 2);
+        assert_ne!(color, new_color);
+    }
+
+    #[test]
+    fn colors_retain() {
+        let mut colors = Colors::new();
+
+        let color1 = colors.update(None, &[1_u64]).unwrap();
+        assert_eq!(colors.len(), 1);
+        // used_colors:
+        //   color1: 1
+
+        dbg!("update");
+        let same_color = colors.update(Some(color1), &[1_u64]).unwrap();
+        assert_eq!(colors.len(), 1);
+        assert_eq!(color1, same_color);
+        // used_colors:
+        //   color1: 2
+
+        dbg!("upgrade");
+        let color2 = colors.update(Some(color1), &[2_u64]).unwrap();
+        assert_eq!(colors.len(), 2);
+        assert_ne!(color1, color2);
+        // used_colors:
+        //   color1: 1
+        //   color2: 1
+
+        dbg!("update");
+        let same_color = colors.update(Some(color2), &[2_u64]).unwrap();
+        assert_eq!(colors.len(), 2);
+        assert_eq!(color2, same_color);
+        // used_colors:
+        //   color1: 1
+        //   color1: 2
+
+        dbg!("upgrade");
+        let color3 = colors.update(Some(color1), &[3_u64]).unwrap();
+        assert_ne!(color1, color3);
+        assert_ne!(color2, color3);
+        // used_colors:
+        //   color1: 0
+        //   color2: 2
+        //   color3: 1
+
+        // This is the pre color-count tracker, where it is needed
+        // to call retain to maintain colors
+        //assert_eq!(colors.len(), 3);
+        //colors.retain(|c, _| [color2, color3].contains(c));
+
+        assert_eq!(colors.len(), 2);
+    }
+}

src/core/src/errors.rs

@@ -43,10 +43,10 @@ pub enum SourmashError {
     HLLPrecisionBounds,
 
     #[error(transparent)]
-    ReadDataError(#[from] crate::index::storage::ReadDataError),
+    ReadDataError(#[from] crate::storage::ReadDataError),
 
     #[error(transparent)]
-    StorageError(#[from] crate::index::storage::StorageError),
+    StorageError(#[from] crate::storage::StorageError),
 
     #[error(transparent)]
     SerdeError(#[from] serde_json::error::Error),

src/core/src/ffi/hyperloglog.rs

@@ -2,9 +2,9 @@ use std::ffi::CStr;
 use std::os::raw::c_char;
 use std::slice;
 
-use crate::index::sbt::Update;
 use crate::signature::SigsTrait;
 use crate::sketch::hyperloglog::HyperLogLog;
+use crate::traits::Update;
 
 use crate::ffi::minhash::SourmashKmerMinHash;
 use crate::ffi::utils::ForeignObject;