First commit of the natLib branch.

The goal is to split the NeuroCurator in two project, a back-end (an API
called nat for Neuro-Annotation Toolbox) and a graphical user interface
called NeuroCurator. This is to allow interacting with corpora using
only the nat package, so that users can avoid installing QT.

Author: christian-oreilly

LICENSE.txt

@@ -0,0 +1,4 @@
+__all__ = ["TreeData", "Annotation", "TextLocalizer"]
+
+from .treeData import TreeData
+from .annotation import Annotation, TextLocalizer

__init__.py

@@ -3,15 +3,32 @@
 __author__ = "Christian O'Reilly"
 
 from uuid import uuid1
-from PySide import QtGui, QtCore
-#from parse import parse
-import json
+from os.path import join
 from abc import abstractmethod
-from modelingParameter import ParameterInstance, ParamRef# ExperimentProperty
-from tag import Tag
+import json
+
+from .modelingParameter import ParameterInstance, ParamRef
+from .tag import Tag
+from . import utils
+
+
+
+
+def getParametersForPub(dbPath, pubId):
+    fileName = join(dbPath, pubId + ".pcr") 
+    with open(fileName, "r", encoding="utf-8", errors='ignore') as f:
+        try:
+            annotations = Annotation.readIn(f)
+        except ValueError:
+            raise ValueError("Problem reading file " + fileName + ". The JSON coding of this file seems corrupted.")
+
+    parameters = []
+    for annot in annotations:
+        parameters.extend(annot.parameters)
+    return parameters
+
+
 
-import utils
-from os.path import join
 
 class Localizer:
     @staticmethod    
@@ -357,81 +374,3 @@ def __repr__(self):
     def __str__(self):
         return '"{}";"{}";"{}";"{}";{}'.format(self.ID, self.comment, type(TextLocalizer), self.users, self.tags)
 
-
-
-class AnnotationListModel(QtCore.QAbstractTableModel):
-
-    def __init__(self, parent, annotationList = [], header = ['ID', 'type', 'localizer', 'comment'], *args):
-        QtCore.QAbstractTableModel.__init__(self, parent, *args)
-        self.annotationList = annotationList
-        self.header = header
-        self.nbCol = len(header)
-        self.sortCol   = 0 
-        self.sortOrder = QtCore.Qt.AscendingOrder        
-
-    def rowCount(self, parent=None):
-        return len(self.annotationList)
-
-    def columnCount(self, parent=None):
-        return self.nbCol 
-
-
-    def getSelectedAnnotation(self, selection):
-
-        if isinstance(selection, list):
-            if selection == []:
-                return None
-            elif isinstance(selection[0], QtCore.QModelIndex):
-                index = selection[0]
-        else:
-            if selection.at(0) is None:
-                return None
-            index = selection.at(0).indexes()[0]
-        return self.annotationList[index.row()]
-
-
-
-    def getByIndex(self, annot, ind):
-        if ind == 0:
-            return annot.ID
-        elif ind == 1:
-            return str(annot.type)
-        elif ind == 2:
-            return str(annot.localizer)
-        elif ind == 3 :
-            return annot.comment
-        else:
-            raise ValueError
-
-    def data(self, index, role):
-        if not index.isValid():
-            return None
-
-        if role != QtCore.Qt.DisplayRole:
-            return None
-
-        return self.getByIndex(self.annotationList[index.row()], index.column())
-
-    def headerData(self, col, orientation, role):
-        if orientation == QtCore.Qt.Horizontal and role == QtCore.Qt.DisplayRole:
-            return self.header[col]
-        return None
-
-    def sort(self, col=None, order=None):
-        if col is None:
-            col = self.sortCol 
-        if order is None:
-            order = self.sortOrder
-
-        """sort table by given column number col"""
-        self.emit(QtCore.SIGNAL("layoutAboutToBeChanged()"))
-        reverse = (order == QtCore.Qt.DescendingOrder)
-        self.annotationList = sorted(self.annotationList, key=lambda x: self.getByIndex(x, col), reverse = reverse) #operator.itemgetter(col))
-        #if order == QtCore.Qt.DescendingOrder:
-        #    self.mylist.reverse()
-        self.emit(QtCore.SIGNAL("layoutChanged()"))
-
-    def refresh(self):
-        self.emit(QtCore.SIGNAL("layoutChanged()"))
-
-

additionsToNeurolex.csv additionsToOntologies.csv

@@ -4,14 +4,16 @@
 __email__  = '[email protected]'
 
 
-import sys, os
+import os
 from glob import glob
-from annotation import Annotation
-from modelingParameter import NumericalVariable, getParameterTypeNameFromID, Variable
-from qtNeurolexTree import flatten_list, loadTreeData, TreeData
 import pandas as pd
 import numpy as np
 
+from .annotation import Annotation
+from .modelingParameter import NumericalVariable, getParameterTypeNameFromID, Variable
+from .treeData import flatten_list, OntoManager
+
+
 annotationKeys         = ["Annotation type", "Publication ID", "Has parameter", "Tag name", "Author"]
 annotationResultFields = ["Annotation type", "Publication ID", "Nb. parameters", "Tag name", "Comment", "Authors", "Localizer"]
 
@@ -209,7 +211,10 @@ def __init__(self, pathDB=None):
         if pathDB is None:
             pathDB = os.path.join(os.path.dirname(__file__), './curator_DB')
 
-        self.treeData, self.dicData    = loadTreeData()
+        ontoMng = OntoManager()
+        self.treeData                  = ontoMng.trees 
+        self.dicData                   = ontoMng.dics
+        
         self.conditions = Condition()    
 
         self.pathDB     = pathDB 

annotation.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jun 10 17:16:54 2016
+
+@author: oreilly
+"""
+
+import os, json
+
+from neurocurator.scigraph_client import Vocabulary, Graph
+from neurocurator.treeData import TreeData
+
+
+def rebuildTreeFromKS(maxDepth=100, verbose=False, fileName="onto", recompute=True, start=0):
+    vocab = Vocabulary()
+    graph = Graph()            
+
+    def addSubtree(root, depth=0): 
+        if depth+1 < maxDepth:
+            if root is None:
+                return
+            neighbors = graph.getNeighbors(root.id, relationshipType="subClassOf", direction="INCOMING")
+            if neighbors is None:
+                nodes = []
+            else:
+                nodes = neighbors["nodes"]
+            for node in nodes:
+                nodeLabel = node["lbl"]
+                nodeId    = node["id"]
+                
+                if root.id == nodeId:
+                    continue
+
+                if nodeLabel is None:
+                    continue
+                
+                child = TreeData(nodeLabel, nodeId, root)            
+                termDic[nodeId] = nodeLabel
+                if verbose:
+                    print(" "*(depth+1) + nodeLabel)
+                addSubtree(child, depth+1)
+                root.children.append(child)
+
+
+    categories = vocab.getCategories()
+    
+    root_ids = []
+    for cat in categories:
+        catTerms = vocab.findByTerm(cat)
+        if catTerms is None:
+            continue
+        for catTerm in catTerms :
+            if catTerm["curie"] is None:
+                continue
+            root_ids.append(catTerm["curie"])
+
+    # Supplementary tree roots not included in categories
+    root_ids.extend(["NIFMOL:nifext_8054", "BFO:0000023", "BIRNOBI:birnlex_11009", "NIFMOL:birnlex_15"])
+    sorted(root_ids, key=str.lower)
+
+    for root_no, root_id in enumerate(root_ids):
+        
+        if root_no >= start:
+            termDic = {}            
+            root = vocab.findById(root_id)
+            if root is None:
+                print("Skipping " + str(root_no))
+                continue
+            if root["labels"] is None:
+                print("Skipping " + str(root_no))
+                continue
+            neighbors = graph.getNeighbors(root_id, relationshipType="subClassOf", direction="INCOMING")    
+            if neighbors is None:
+                print("Skipping " + str(root_no))
+                continue
+            if len(neighbors["nodes"]) == 1:
+                print("Skipping " + str(root_no))
+                continue
+            
+            root_name = root["labels"][0]
+            if not recompute:
+                if (os.path.isfile(fileName + "_" + root_name + "_" + str(root_no) + ".tree") and
+                    os.path.isfile(fileName + "_" + root_name + "_" + str(root_no) + ".dic")) :
+                    print("Skipping " + fileName + "_" + root_name + "_" + str(root_no))
+                    continue
+                
+            if verbose:
+                print(root_name)
+            root = TreeData(root_name, root_id, root_no=root_no)            
+            termDic[root_id] = root_name
+            addSubtree(root)
+
+            root.saveTree(fileName)
+            with open(fileName + "_" + root_name + "_" + str(root_no) + ".dic", 'w') as f:
+                json.dump(termDic, f, sort_keys=True, indent=4, separators=(',', ': '))
+
+
+
+import sys
+if __name__ == "__main__":
+    import codecs
+    sys.stdout = codecs.getwriter('utf8')(sys.stdout.detach())
+    sys.stderr = codecs.getwriter('utf8')(sys.stderr.detach())
+    if len(sys.argv) == 1:
+        rebuildTreeFromKS(maxDepth=100, verbose=True, fileName="onto/onto", recompute=False)
+    elif len(sys.argv) == 2:
+        rebuildTreeFromKS(maxDepth=100, verbose=True, fileName="onto/onto", recompute=False, start=float(sys.argv[1]))
+        
+

annotationSearch.py

@@ -1,5 +1,6 @@
 #!/usr/bin/python3
 
+"""
 __author__ = "Christian O'Reilly"
 
 import urllib
@@ -16,3 +17,4 @@ def checkDOI(DOI):
 	soup = BeautifulSoup(html, "lxml")
 	return not "DOI Not Found" in soup.title.string
 
+"""

buildOntoTree.py

@@ -2,24 +2,23 @@
 
 
 from git import Repo, exc
-import os
 
-from PySide import QtGui, QtCore
+from PySide import QtGui
 
 
 class GitManager:
 
-    def __init__(self, settings):
+    def __init__(self, gitSettings):
 
-        self.localRepoDir = settings.config["GIT"]["local"]
+        self.localRepoDir = gitSettings["local"]
         self.offline = False        
 
         try:
             self.repo = Repo(self.localRepoDir)
             assert not self.repo.bare
 
         except (exc.InvalidGitRepositoryError,exc.NoSuchPathError):
-            self.repo = Repo.clone_from("ssh://" + settings.config["GIT"]["user"] + "@" + settings.config["GIT"]["remote"], self.localRepoDir)
+            self.repo = Repo.clone_from("ssh://" + gitSettings["user"] + "@" + gitSettings["remote"], self.localRepoDir)
 
 
         self.tryToFetch()
@@ -156,8 +155,7 @@ def commit(self, msg = "..."):
         # we use this "..." default message.
 
         try:
-            commitObj = self.repo.index.commit(msg)
-            #print(commitObj)
+            self.repo.index.commit(msg)
         except exc.UnmergedEntriesError as e:
             print(e)
             raise

doi.py

@@ -6,6 +6,27 @@
 from bs4 import BeautifulSoup
 
 
+
+def getIDfromPMID(PMID):
+    url = "http://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?tool=neurocurator&[email protected]&ids=" + PMID + "&format=json&versions=no"
+    try:
+        with urllib.request.urlopen(url) as response:
+           html = response.read()
+    except urllib.error.HTTPError:
+        return None
+
+    soup = BeautifulSoup(html, "lxml")
+    jsonStr = eval(str(soup)[15:-19])
+    if jsonStr["status"] == "ok":
+        if "doi" in jsonStr["records"][0]:
+            return jsonStr["records"][0]["doi"]
+        else:
+            return "PMID_" + PMID
+    else:
+        return None
+
+    
+
 def checkID(ID): 
 	if "PMID" in ID:
 		return checkPMID(ID)
@@ -15,14 +36,15 @@ def checkID(ID):
 
 def checkPMID(ID):
 	idKind, PMID = ID.split("_")
-	url = "http://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?tool=curator&[email protected]&ids=" + PMID + "&format=json&versions=no"
+	url = "http://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?tool=neurocurator&[email protected]&ids=" + PMID + "&format=json&versions=no"
 	try:
 		with urllib.request.urlopen(url) as response:
 		   html = response.read()
 	except urllib.error.HTTPError:
 		return False
 
 	soup = BeautifulSoup(html, "lxml")
+     
 	return eval(str(soup)[15:-19])["status"] == "ok"
 
 

gitManager.py

@@ -5,7 +5,6 @@
 
 import sys
 from PySide import QtGui
-from qtNeurolexTree import TreeData
 from mainWin import Window
 
 if __name__ == '__main__':