Dev

Scipyen.desktop

@@ -1,17 +1,17 @@
 [Desktop Entry]
-Comment[en_GB]=
-Comment=
-Exec=/home/cezar/bin/scipyen_desktop.sh
-GenericName[en_GB]=Scipyen
-GenericName=Scipyen
+Comment[en_GB]=Scientific Python Environment for Neurophysiology
+Comment=Scientific Python Environment for Neurophysiology
+Exec=scipyen
+GenericName[en_GB]=Scientific Python Environment for Neurophysiology
+GenericName=Scientific Python Environment for Neurophysiology
 Icon=pythonbackend
 MimeType=
 Name[en_GB]=Scipyen
 Name=Scipyen
 Path=
 StartupNotify=true
 Terminal=true
-TerminalOptions=\s--noclose
+TerminalOptions=\s
 Type=Application
 X-DBUS-ServiceName=
 X-DBUS-StartupType=

core/basescipyen.py

@@ -11,7 +11,8 @@
 from core import quantities as cq
 from core.triggerprotocols import TriggerProtocol
 from core.quantities import units_convertible
-from core.prog import (ArgumentError, OneOf, DescriptorTypeValidator, DescriptorGenericValidator,
+from core.prog import (ArgumentError, OneOf, 
+                       DescriptorTypeValidator, DescriptorGenericValidator,
                        AttributeAdapter, get_descriptors, get_properties,
                        parse_descriptor_specification, WithDescriptors,
                        setup_descriptor)

core/extipy_init.py

@@ -1,3 +1,7 @@
+"""Used for importing modules from site-package.
+Unless Scipyen is among those, import statements for Scipyen's modules won't
+work here.
+"""
 has_hdf5 = False
 
 try:

core/extipyutils_client.py

@@ -1,15 +1,15 @@
 # -*- coding: utf-8 -*-
 """Module with utilities for an external IPython kernel.
-To be used on the client side (i.e. Scipyen app side).
-Contains functions and types used ot communicate with the remote kernel via its
+To be used on the client side (i.e. Scipyen app side, NOT in the REMOTE kernel).
+Contains functions and types used to communicate with the remote kernel via its
 messaging API.
 """
 # on the client side, in a received execute_reply message the keys of
 # msg["content"]["user_expressions"] are identical to those in the
 # sent execute_request message["content"]["user_expressions"] NOTE that such
 # execute_request message is sent when client.execute(...) is called, and the 
 # user_expressions field of the message gets the valueof the user_expressions
-# parameter ot that call.
+# parameter to that call.
 
 # NOTE 2020-07-11 10:26:38
 # Strategies for getting information about the properties of an object located
@@ -86,13 +86,11 @@
 else:
     __virtual_site_packages__ = None
 
-# initialization script for ALL external IPython consoles
+# initialization script for ALL available external IPython consoles
 # private: call indirectly via init_commands!
 _ext_ipython_initialization_file = os.path.join(__module_path__, "extipy_init.py")
 _ext_ipython_initialization_cmd = " ".join(["get_ipython().run_cell(", "'run -i -n", _ext_ipython_initialization_file, "')"])
 
-#print("_ext_ipython_initialization_cmd", _ext_ipython_initialization_cmd)
-
 # initialization script for NEURON in external ipython process
 # only called when launching a NEURON external console
 # expected to be passed as 'code' parameter to tab with frontend factories in 
@@ -107,6 +105,8 @@
 # figure out how to use Qt5 Agg in the external ipython (mpl.use("Qt5Agg") crashes
 # the kernel when a mpl figure is shown)
 # for now stick with inline figures
+# TODO: 2022-02-06 22:28:41
+# consider consolidating these and the extipyutils_host if possible
 init_commands = [
     "import sys, os, io, warnings, numbers, types, typing, re, importlib",
     "import traceback, keyword, inspect, itertools, functools, collections",
@@ -117,6 +117,16 @@
 if os.path.isfile(_ext_ipython_initialization_file):
     init_commands.append(_ext_ipython_initialization_cmd)
 
+# NOTE: 2022-02-06 22:30:26
+# some of the commands below expose Scipyen API to external kernels; this may
+# be done via such init commands as below; 
+# These MIGHT also be imported from within extipyutils_host BUT they won't be 
+# directly available in the REMOTE kernel workspace - but only indirectly as
+# members of the 'hostutils' module available in the REMOTE kernel workspace.
+# Hence, hostutils (a.k.a extipyutils_host) shoud only contain API deemed as
+# housekeeping for the REMOTE python kernel, including for communication
+# between the Scipyen kernel and the REMOTE kernel, and not requiring regular
+# access by the user.
 init_commands.extend(
     [
     "import signal, pickle, json, csv",
@@ -133,9 +143,36 @@
     "from matplotlib import pyplot as plt",
     "from matplotlib import cm",
     "import matplotlib.mlab as mlb",
-    "mpl.use('Qt5Agg')",
+    #"mpl.use('Qt5Agg')",
     "".join(["sys.path.insert(2, '", __scipyen_path__, "')"]),
     "from core import extipyutils_host as hostutils",
+    "from core.workspacefunctions import *",
+    "from core import plots as plots",
+    "from core import datatypes as dt",
+    "from core import neoutils",
+    "from core.datatypes import * ",
+    "import core.signalprocessing as sigp",
+    "import core.curvefitting as crvf",
+    "import core.strutils as strutils",
+    "import core.data_analysis as anl",
+    "from core.utilities import (summarize_object_properties,standard_obj_summary_headers,safe_identity_test, unique, index_of, gethash,NestedFinder, normalized_index)",
+    "from core.prog import (safeWrapper, deprecation, iter_attribute,filter_type, filterfalse_type, filter_attribute, filterfalse_attribute)",
+    "from core import prog",
+    "from core.traitcontainers import DataBag",
+    "from core.triggerprotocols import TriggerProtocol",
+    "from core.triggerevent import (DataMark, TriggerEvent, TriggerEventType, )",
+    "from core.datasignal import (DataSignal, IrregularlySampledDataSignal,)",
+    "from core.datazone import DataZone",
+    "from ephys import (ephys, ltp, membrane, ivramp,)",
+    "from systems import *",
+    "from imaging import (imageprocessing as imgp, imgsim,)",
+    "from imaging import axisutils, vigrautils",
+    "from imaging.axisutils import (axisTypeFromString, axisTypeName,axisTypeStrings,axisTypeSymbol, axisTypeUnits,dimEnum,dimIter,evalAxisTypeExpression,getAxisTypeFlagsInt,getNonChannelDimensions,hasChannelAxis,)",
+    "from imaging.axiscalibration import (AxesCalibration,AxisCalibrationData, ChannelCalibrationData, CalibrationData)",
+    "from imaging.scandata import (AnalysisUnit, ScanData,)",
+    "from iolib import pictio as pio",
+    "from iolib import h5io, jsonio",
+    "print('To use matplotlib run %matplotlib magic')"
     ])
 
 #init_commands = ["import sys, os, io, warnings, numbers, types, typing, re, importlib",

core/extipyutils_host.py

@@ -1,9 +1,26 @@
 # -*- coding: utf-8 -*-
-"""Module with utilities for an external IPython kernel.
-To be imported in the remote kernel namespace - facilitates code execution by 
-Scipyen app and its internal console -- the "client" -- inside the remote kernel
--- the "host". the "host" is normally launched and interacted with by Scipyen's
-External IPython Console
+"""Module with housekeeping utilities for an external IPython kernel.
+
+To be run/imported inside a REMOTE ipython kernel (see extipyutils_client).
+
+This module includes code for communication between the external (or REMOTE)
+IPython kernel and Scipyen's internal (embedded) IPython kernel, as well as code
+managing custom magics, see below) which are not needed for regular user access.
+
+NOTE: 2022-02-06 22:26:01
+Some initialization has already occurred, driven by extipyutils_client.py (and
+from there, via extipy_init.py) BEFORE this module.
+
+In fact, this module is imported as hostutils via executing the init_commands in 
+extipyutils_client.
+
+To expose Scipyen API inside the REMOTE IPython kernel workspace, either insert
+relevant import statements in the init_commands list inside extipyutils_client
+module. NOTE: extipy_init module cannot be used for importing Scipyen API in 
+the REMOTE kernel namespace, as it is oblivious of Scipyen's module paths.
+
+NOTE: NeuronMagics are useful to start NEURON manually from an external IPython
+kernel, optionally with ('nrngui') or without NEURON GUI ('nrnpy')
 """
 
 import os, sys
@@ -73,13 +90,15 @@ class NeuronMagics(Magics):
     @line_magic
     @needs_local_scope
     def nrngui(self, line, local_ns):
-        """Starts NEURON modelling with gui
+        """Starts NEURON modelling with gui, in this kernel
         """
         get_ipython().run_line_magic("run", self.nrngui_magic_cmd)
 
     @line_magic
     @needs_local_scope
     def nrnpy(self, line, local_ns):
+        """Starts NEURON modelling (without gui), in this kernel
+        """
         get_ipython().run_line_magic("run", self.nrnpy_magic_cmd)
 
 #get_ipython().register_magics(NeuronMagics)

core/multiframeindex.py

@@ -22,6 +22,8 @@ class IndexProxy(object):
     in the 'map' attribute (a DataFrame) of the owner of this proxy, with 
     the owner being an instance of FrameIndexLookup.
 
+    Implements the descriptor protocol.
+
     """
 
     def __init__(self, field:str):
@@ -540,7 +542,12 @@ def where(self, field:str, value:int):
         """
         pds = self.__getitem__(field)
 
-        cond = pds == value
+        if self.missingFrameIndex is pd.NA:
+            ppds = pds.fillna(np.nan)
+        else:
+            ppds = pds
+
+        cond = ppds == value
 
         ret = list(pds.where(cond).dropna().index)
 

core/neoutils.py

@@ -163,6 +163,13 @@
 from neo.core.dataobject import (DataObject, ArrayDict)
 import matplotlib as mpl
 import pyqtgraph as pg
+
+try:
+    import pyabf
+    hasPyABF = True
+except:
+    hasPyABF = False
+
 #### END 3rd party modules
 
 #### BEGIN pict.core modules
@@ -240,6 +247,30 @@
 
     __debug_count__ = 0
 
+def getABF(obj):
+    import os
+    if not hasPyABF:
+        return
+
+    if not os.path.exists(getattr(obj, "file_origin", None)):
+        return
+
+    try:
+        return pyabf.ABF(obj.file_origin)
+    except:
+        pass
+
+def getABFProtocolEpochs(obj):
+    if not hasPyABF:
+        return
+
+    from core import pyabfbridge
+
+    abf = getABF(obj)
+
+    if abf:
+        return pyabfbridge.getEpochTables(abf, as_dataFrame=True)
+
 def copy_to_segment(obj:neo.core.dataobject.DataObject, new_seg:neo.Segment):
     new_obj = obj.copy()
     new_obj.segment = new_seg

core/patchneo.py

@@ -31,6 +31,7 @@
 
 def _patch_new_neo(original_f, *args, **kwargs):
     sig = signature2Dict(original_f)
+    #print(f"sig: {sig}")
     sig_named = list(sig.named.keys())
     named = dict()
     var = list()
@@ -59,6 +60,7 @@ def _patch_new_neo(original_f, *args, **kwargs):
         if k in sig.named:
             named[k] = v
 
+    #print(f" var {var}, named {named}, kwargs {kwargs}")
     return original_f(*var, **named, **kwargs)
 
 

core/pyabfbridge.py

@@ -0,0 +1,114 @@
+"""Bridge to pyabf package
+
+Requires pyabf.
+
+NOTE: PyABF (https://swharden.com/pyabf/) is not currently used to load 
+Axon ABF files - Scipyen uses neo package instead (https://neo.readthedocs.io/en/stable/)
+
+This is because all electorprhyiology signals are represented in Scipyen as
+objects of types defined in the neo framework. 
+
+However, pyABF does offer functionality that neo lacks, in particular, for the
+inspection of acquisition protocol data embedded in an axon file.
+
+Such functionality comes in handy when one needs to inspect the acquisition
+protocol post hoc.
+
+To use the functions in this module you need to load the ABF file as a pyabf.ABF
+object (see https://swharden.com/pyabf/tutorial/ and https://swharden.com/pyabf/)
+
+
+"""
+import typing
+import numpy as np
+import pandas as pd
+import quantities as pq
+
+from core import quantities as spq
+
+try:
+    import pyabf
+    hasPyABF=True
+except:
+    hasPyABF = False
+
+def getEpochTables(x:object, as_dataFrame:bool=False):
+    if not hasPyABF:
+        return
+
+    if not isinstance(x, pyabf.ABF):
+        raise TypeError(f"Expecting a pyabf.ABF object; got {type(x).__name__} instead")
+
+    etable = [pyabf.waveform.EpochTable(x, c) for c in x.channelList]
+
+    # NOTE: 2022-03-04 15:30:22
+    # only return the epoch tables that actually contain any non-OFF epochs
+
+    if as_dataFrame:
+        return [epochTable2DF(e, x) for e in etable if len(e.epochs)]
+
+    return [e for e in etable if len(e.epochs)]
+
+def epochTable2DF(x:object, abf:typing.Optional[pyabf.ABF] = None):
+    """Returns a pandas.DataFrame with the data from the epoch table 'x'
+    """
+    if not hasPyABF:
+        return
+
+    if not isinstance(x, pyabf.waveform.EpochTable):
+        raise TypeError(f"Expecting an EpochTable; got {type(x).__name__} instead")
+
+    # NOTE: 2022-03-04 15:38:31
+    # code below adapted from pyabf.waveform.EpochTable.text
+    #
+
+    rowIndex = ["Type", "First Level", "Delta Level", "First Duration (points)", "Delta Duration (points)",
+                "First duration (ms)", "Delta Duration (ms)",
+                "Digital Pattern #3-0", "Digital Pattern #7-4", "Train Period (points)", "Pulse Width (points)",
+                "Train Period (ms)", "Pulse Width (ms)"]
+
+    # prepare lists to hold values for each epoch
+
+    # NOTE: 2022-03-04 16:05:20 
+    # skip "Off" epochs
+    epochs = [e for e in x.epochs if e.epochTypeStr != "Off"]
+
+    if len(epochs):
+        epochCount = len(epochs)
+        epochLetters = [''] * epochCount
+
+        epochData = dict()
+
+        for i, epoch in enumerate(epochs):
+            assert isinstance(epoch, pyabf.waveform.Epoch)
+
+            if isinstance(abf, pyabf.ABF):
+                adcName, adcUnits = abf._getAdcNameAndUnits(x.channel)
+                dacName, dacUnits = abf._getDacNameAndUnits(x.channel)
+
+            else:
+                adcName = adcUnits = dacName = dacUnits = None
+
+            dacLevel = epoch.level*spq.unit_quantity_from_name_or_symbol(dacUnits) if isinstance(dacUnits, str) and len(dacUnits.strip()) else epoch.level
+            dacLevelDelta = epoch.levelDelta*spq.unit_quantity_from_name_or_symbol(dacUnits) if isinstance(dacUnits, str) and len(dacUnits.strip()) else epoch.levelDelta
+
+            epValues = np.array([epoch.epochTypeStr,                              
+                              dacLevel, dacLevelDelta,
+                              epoch.duration, epoch.durationDelta,
+                              epoch.duration/x.sampleRateHz * 1000 * pq.ms, 
+                              epoch.durationDelta/x.sampleRateHz * 1000 * pq.ms,
+                              epoch.digitalPattern[:4], epoch.digitalPattern[4:],
+                              epoch.pulsePeriod, epoch.pulseWidth,
+                              epoch.pulsePeriod/x.sampleRateHz * 1000 * pq.ms,
+                              epoch.pulseWidth/x.sampleRateHz * 1000 * pq.ms], dtype=object)
+
+            epochData[epoch.epochLetter] = epValues
+
+        #colIndex = epochLetters
+
+        return pd.DataFrame(epochData, index = rowIndex)
+
+
+
+
+