updated KGs: robokop, rtx-kg2, and text-mining; added AT/KL to edges;…

… solve dict-get failed error; added mvp1 templates; filter edges with same subject and object

medikanren2/neo/neo-low-level/make-query-low-level.rkt

@@ -21,6 +21,7 @@
 
 (define (make-query-low-level
          db-path-under-parent ;; for example, "rtx-kg2/pre_2.8.0/rtx-kg2pre_2.8.0.db"
+         kg-infores
          )
 
   (pretty-log `(In make-query-low-level for)
@@ -94,17 +95,18 @@
   (define (curie->properties curie)
     (enumerator->list
      (lambda (yield)
-       ((dict-enumerator (dict-get curie=>ckey=>cvalue=>1 (string->id curie)))
+       ((dict-enumerator (dict-get-safe curie=>ckey=>cvalue=>1 (string->id curie) dict.empty))
         (lambda (ckey cvalue=>1)
           (yield (map id->string (cons ckey (enumerator->list
                                              (dict-key-enumerator cvalue=>1))))))))))
   (define (edge-id->properties eid)
-    (enumerator->list
-     (lambda (yield)
-       ((dict-enumerator (dict-get eid=>ekey=>evalue=>1 eid))
-        (lambda (ekey evalue=>1)
-          (yield (map id->string (cons ekey (enumerator->list
-                                             (dict-key-enumerator evalue=>1))))))))))
+    (cons (list "upstream_resource_ids" (list kg-infores))
+          (enumerator->list
+           (lambda (yield)
+             ((dict-enumerator (dict-get eid=>ekey=>evalue=>1 eid))
+              (lambda (ekey evalue=>1)
+                (yield (map id->string (cons ekey (enumerator->list
+                                                   (dict-key-enumerator evalue=>1)))))))))))
 
   (define (query:Known->X curie*.K predicate*.K->X category*.X)
     (define (query. yield)
@@ -381,46 +383,36 @@
   (define (query:dict.Known->dict.Known curie=>1.S predicate*.S->O curie=>1.O direction-tag)
     (define (query.sub->obj yield)
       (let* ((ekey.predicate    (string->id str.predicate))
-             (ckey.name         (string->id "name"))
              (predicate=>1      (string*->id=>1 predicate*.S->O))
              (predicate=>eid=>1 (dict-get ekey=>evalue=>eid=>1 ekey.predicate)))
         ((merge-join fx< curie=>1.S subject=>eid=>object=>1)
          (lambda (id.S __ eid=>O=>1)
-           (let* ((name.S (get-name-from-dict-safe (dict-get curie=>ckey=>cvalue=>1 id.S) ckey.name))
-                  (S      (id->string id.S)))
+           (let* ((S      (id->string id.S)))
              ((merge-join fx< predicate=>1 predicate=>eid=>1)
               (lambda (id.predicate.S->O __ eid=>1)
                 (let ((predicate.S->O (id->string id.predicate.S->O)))
                   ((merge-join fx< eid=>1 eid=>O=>1)
                    (lambda (eid __ O=>1)
                      ((merge-join fx< curie=>1.O O=>1)
                       (lambda (id.O __ ___)
-                        (let* ((name.O (get-name-from-dict-safe (dict-get curie=>ckey=>cvalue=>1 id.O)
-                                                                ckey.name))
-                               (O      (id->string id.O)))
-                          (yield (list* S name.S predicate.S->O O name.O
-                                        (edge-id->properties eid))))))))))))))))
+                        (let* ((O      (id->string id.O)))
+                          (yield (list* S predicate.S->O O (edge-id->properties eid))))))))))))))))
     (define (query.obj->sub yield)
       (let* ((ekey.predicate    (string->id str.predicate))
-             (ckey.name         (string->id "name"))
              (predicate=>1      (string*->id=>1 predicate*.S->O))
              (predicate=>eid=>1 (dict-get ekey=>evalue=>eid=>1 ekey.predicate)))
         ((merge-join fx< curie=>1.O object=>eid=>subject=>1)
          (lambda (id.O __ eid=>S=>1)
-           (let* ((name.O (get-name-from-dict-safe (dict-get curie=>ckey=>cvalue=>1 id.O) ckey.name))
-                  (O      (id->string id.O)))
+           (let* ((O      (id->string id.O)))
              ((merge-join fx< predicate=>1 predicate=>eid=>1)
               (lambda (id.predicate.O->S __ eid=>1)
                 (let ((predicate.O->S (id->string id.predicate.O->S)))
                   ((merge-join fx< eid=>1 eid=>S=>1)
                    (lambda (eid __ S=>1)
                      ((merge-join fx< curie=>1.S S=>1)
                       (lambda (id.S __ ___)
-                        (let* ((name.S (get-name-from-dict-safe (dict-get curie=>ckey=>cvalue=>1 id.S)
-                                                                ckey.name))
-                               (S      (id->string id.S)))
-                          (yield (list* S name.S predicate.O->S O name.O
-                                        (edge-id->properties eid))))))))))))))))
+                        (let* ((S      (id->string id.S)))
+                          (yield (list* S predicate.O->S O (edge-id->properties eid))))))))))))))))
     (cond
       [(eq? direction-tag 'obj->sub) (maybe-time (enumerator->rlist query.obj->sub))]
       [(eq? direction-tag 'sub->obj) maybe-time (enumerator->rlist query.sub->obj)]

medikanren2/neo/neo-low-level/query-low-level-multi-db.rkt

@@ -513,10 +513,11 @@
    (curie-in-db?-rtx-kg2 curie)))
 
 (define (curies-in-db curie*)
+  (remove-duplicates
    (append
     (curies-in-db-robokop curie*)
     (curies-in-db-text-mining curie*)
-    (curies-in-db-rtx-kg2 curie*)))
+    (curies-in-db-rtx-kg2 curie*))))
 
 (define (curie->properties curie)
   (append

medikanren2/neo/neo-low-level/query-low-level-robokop.rkt

@@ -23,7 +23,8 @@
  "make-query-low-level.rkt"
  racket/match)
 
-(define db-path-under-parent "robokop-may-9-2024/march-7-2024/robokop.db")
+#;(define db-path-under-parent "robokop-may-9-2024/march-7-2024/robokop.db")
+(define db-path-under-parent "robokop-aug-6-2024/july-31-2024/robokop.db")
 
 (match-define
   (list
@@ -46,4 +47,4 @@
    edge-id->properties
    get-highest-bucket-number
    )
-  (make-query-low-level db-path-under-parent))
+  (make-query-low-level db-path-under-parent "infores:robokop-kg"))

medikanren2/neo/neo-low-level/query-low-level-rtx-kg2.rkt

@@ -23,7 +23,8 @@
  "make-query-low-level.rkt"
  racket/match)
 
-(define db-path-under-parent "rtx-kg2-may-9-2024/rtx-kg2-2.9.0pre/rtx-kg2.db")
+#;(define db-path-under-parent "rtx-kg2-may-9-2024/rtx-kg2-2.9.0pre/rtx-kg2.db")
+(define db-path-under-parent "rtx-kg2-Aug-6-2024/rtx-kg2-2.10.0pre/rtx-kg2.db")
 
 (match-define
   (list
@@ -46,4 +47,4 @@
    edge-id->properties
    get-highest-bucket-number
    )
-  (make-query-low-level db-path-under-parent))
+  (make-query-low-level db-path-under-parent "infores:rtx-kg2"))

medikanren2/neo/neo-low-level/query-low-level-text-mining.rkt

@@ -23,7 +23,8 @@
  "make-query-low-level.rkt"
  racket/match)
 
-(define db-path-under-parent "text-mining-apr-5-2024/apr_2_2024/text_mining.db")
+(define db-path-under-parent "text-mining-aug-5-2024/aug_5_2024/text_mining.db")
+#;(define db-path-under-parent "text-mining-apr-5-2024/apr_2_2024/text_mining.db")
 
 (match-define
   (list
@@ -46,4 +47,4 @@
    edge-id->properties
    get-highest-bucket-number
    )
-  (make-query-low-level db-path-under-parent))
+  (make-query-low-level db-path-under-parent  "infores:text-mining-provider-targeted"))

medikanren2/neo/neo-server/neo-server-utils.rkt

@@ -10,19 +10,23 @@
          minus-one-before-zero
          find-max-number
          get-source
+         UNSECRET-SOURCE
          num-pubs
          get-score-from-result
          set-score-in-result
          normalize-scores
          edge-has-source?
-         data-attributes
+         publication-attributes
          auxiliary-graph-attribute
+         qualifier-attribute*
          merge-trapi-responses
          semantic-exclude*
          domain-exclude*
          range-exclude*
          get-object
          get-and-print-qualifiers
+         Unsecret-agent-type-attribute
+         Unsecret-knowledge-level-attribute
          agent-type-attribute
          knowledge-level-attribute
          )
@@ -59,19 +63,18 @@
         '())))
 
 (define mvp2-filter
-  (lambda (target-eprop direction)
+  (lambda (target-eprop direction care-aspect?)
     (let* ((aspect (get-assoc "object_aspect_qualifier" target-eprop))
            (direction^ (get-assoc "object_direction_qualifier" target-eprop)))
-      (and
-       aspect
-       direction^
-       (or
-        (equal? "activity" aspect)
-        (equal? "abundance" aspect)
-        (equal? "activity_or_abundance" aspect)
-        (equal? "expression" aspect)
-        (equal? "synthesis" aspect))
-       (equal? direction direction^)))))
+      (if care-aspect?
+          (and
+           aspect
+           direction^
+           (member aspect '("activity" "abundance" "activity_or_abundance" "expression" "synthesis"))
+           (equal? direction direction^))
+          (and
+           direction^
+           (equal? direction direction^))))))
 
 #|
 A increases B increases C = A increases C
@@ -92,17 +95,21 @@ A decreases B increases C = A decreases C
             ,(? string? curie_z)
             ,props_xy
             ,props_yz)
-          (if (equal? direction "increased")
-              (or
-               (and (mvp2-filter props_xy "increased")
-                    (mvp2-filter props_yz "increased"))
-               (and (mvp2-filter props_xy "decreased")
-                    (mvp2-filter props_yz "decreased")))
-              (or
-               (and (mvp2-filter props_xy "increased")
-                    (mvp2-filter props_yz "decreased"))
-               (and (mvp2-filter props_xy "decreased")
-                    (mvp2-filter props_yz "increased"))))]
+          (if (equal? pred_xy "biolink:affects")
+              (if (equal? direction "increased")
+                  (or
+                   (and (mvp2-filter props_xy "increased" #f)
+                        (mvp2-filter props_yz "increased" #t))
+                   (and (mvp2-filter props_xy "decreased" #f)
+                        (mvp2-filter props_yz "decreased" #t)))
+                  (or
+                   (and (mvp2-filter props_xy "increased" #f)
+                        (mvp2-filter props_yz "decreased" #t))
+                   (and (mvp2-filter props_xy "decreased" #f)
+                        (mvp2-filter props_yz "increased" #t))))
+              (if (equal? direction "increased")
+                  (mvp2-filter props_yz "increased" #t)
+                  (mvp2-filter props_yz "decreased" #t)))]
          [else #f]))
      e*)))
 
@@ -111,7 +118,7 @@ A decreases B increases C = A decreases C
     (filter
      (lambda (e)
        (let-values ([(_ eprop) (split-at e 3)])
-         (mvp2-filter eprop direction)))
+         (mvp2-filter eprop direction #t)))
      q)))
 
 (define find-max-number
@@ -125,14 +132,13 @@ A decreases B increases C = A decreases C
              (loop (cdr n*) greatest)))))))
 
 (define (get-source-helper props)
-  (or (get-assoc "primary_knowledge_source" props) 
-      (and (get-assoc "json_attributes" props)
-           "infores:text-mining-provider-targeted")))
+  (get-assoc "primary_knowledge_source" props))
 
 (define (get-source props)
-    (hash
-      'resource_id (get-source-helper props)
-      'resource_role "primary_knowledge_source"))
+  (hash
+   'type "biolink:RetrievalSource"
+   'resource_id (get-source-helper props)
+   'resource_role "primary_knowledge_source"))
 
 (define (num-pubs props)
   (let ((score (string->number (get-assoc "mediKanren-score" props)))
@@ -142,6 +148,13 @@ A decreases B increases C = A decreases C
       ((equal? source "infores:text-mining-provider-targeted") (* 10 score))
       (else score))))
 
+(define (UNSECRET-SOURCE prop)
+  (hash
+   'type "biolink:RetrievalSource"
+   'resource_id "infores:unsecret-agent"
+   'resource_role "aggregator_knowledge_source"
+   'upstream_resource_ids (get-assoc "upstream_resource_ids" prop)))
+
 (define (get-score-from-result result)
   (let ((analyses (hash-ref result 'analyses #f)))
     (if analyses
@@ -165,21 +178,10 @@ A decreases B increases C = A decreases C
             results
             (map (lambda (x) (set-score-in-result x (/ (get-score-from-result x) (* 1.0 max-score)))) results)))))
 
-(define edge-has-source?
-  (lambda (props)
-    (or (get-assoc "primary_knowledge_source" props)
-        (and (get-assoc "json_attributes" props)
-             (let ((attr-hl (string->jsexpr (get-assoc "json_attributes" props))))
-               (let loop ((hl attr-hl))
-                 (cond
-                   ((null? hl) #f)
-                   ((equal?
-                     (hash-ref (car hl) 'attribute_type_id #f)
-                     "biolink:primary_knowledge_source")
-                    #t)
-                   (else (loop (cdr hl))))))))))
-
-(define (data-attributes props has-pub?)
+(define (edge-has-source? props)
+  (get-source-helper props))
+
+(define (publication-attributes props has-pub?)
   (if has-pub?
     (list (get-publications props))
     (list)))
@@ -190,8 +192,7 @@ A decreases B increases C = A decreases C
       (cond
         [(null? props) pubs]
         [else
-         (let ((publication (or (get-assoc "publications" (car props))
-                                (get-assoc "supporting_publications" (car props)))))
+         (let ((publication (get-assoc "publications" (car props))))
            (helper (cdr props)
                    (append 
                     (cond
@@ -215,18 +216,40 @@ A decreases B increases C = A decreases C
      'attribute_type_id "biolink:support_graphs"
      'value (list id))))
 
-(define agent-type-attribute
+(define (qualifier-attribute* qualifier-name props)
+  (let ((qualifer (get-assoc qualifier-name props)))
+    (if qualifer
+        (list (hash 'qualifier_type_id (string-append "biolink:" qualifier-name)
+              'qualifier_value qualifer))
+        '())))
+
+
+(define Unsecret-agent-type-attribute
   (hash
    'attribute_type_id "biolink:agent_type"
    'value "computational_model"
    'attribute_source "infores:unsecret-agent"))
 
-(define knowledge-level-attribute
+(define Unsecret-knowledge-level-attribute
   (hash
    'attribute_type_id "biolink:knowledge_level"
    'value "prediction"
    'attribute_source "infores:unsecret-agent"))
 
+(define (agent-type-attribute prop)
+  (let* ((edge-from-tmkg? (get-assoc "json_attributes" prop))
+         (agent-type (if edge-from-tmkg? "text_mining_agent" (get-assoc "agent_type" prop))))
+        (hash
+         'attribute_type_id "biolink:agent_type"
+         'value agent-type)))
+
+(define (knowledge-level-attribute prop)
+  (let* ((edge-from-tmkg? (get-assoc "json_attributes" prop))
+         (knowledge-level (if edge-from-tmkg? "not_provided" (get-assoc "knowledge_level" prop))))
+        (hash
+         'attribute_type_id "biolink:knowledge_level"
+         'value knowledge-level)))
+
 ;; TODO: test it with calling out Genetics KP
 (define (merge-trapi-responses r1 r2 original-query_graph)
   (let* ((message1 (hash-ref r1 'message))