diff --git a/ODMTools.py b/ODMTools.py index ecbea16..6e57526 100755 --- a/ODMTools.py +++ b/ODMTools.py @@ -20,7 +20,7 @@ import pyodbc import pymysql -import psycopg2 +# import psycopg2 tool = LoggerTool() logger = tool.setupLogger('main', 'odmtools.log', 'a', logging.INFO) diff --git a/doc/wxFormBuilder/frmFillGaps.fbp b/doc/wxFormBuilder/frmFillGaps.fbp new file mode 100644 index 0000000..ddb8126 --- /dev/null +++ b/doc/wxFormBuilder/frmFillGaps.fbp @@ -0,0 +1,756 @@ + + + + + + Python + 1 + source_name + 0 + 0 + res + UTF-8 + connect + clsGapFill + 1000 + none + 0 + frmGapFill + + . + + 1 + 1 + 1 + 1 + UI + 0 + 0 + + 0 + wxAUI_MGR_DEFAULT + + wxBOTH + + 1 + 1 + decl + + + + 0 + wxID_ANY + 315,217 + 315,217 + dlgFill + + 315,217 + wxDEFAULT_DIALOG_STYLE + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + bsForm + wxVERTICAL + none + + 5 + wxALL + 0 + + 1 + 1 + 1 + 1 + + + + + + + + 1 + 0 + 1 + + 1 + 0 + Dock + 0 + Left + 1 + + 1 + + 0 + 0 + wxID_ANY + This function fills any gaps less than the gap duration with a no-data value at the fill frequency + + 0 + + + 0 + + 1 + lblInstructions + 1 + + + protected + 1 + + Resizable + 1 + + + + 0 + + + + + 300 + + + + + + + + + + + + + + + + + + + + + + + + + + + 5 + wxEXPAND + 1 + + 3 + wxBOTH + + + 0 + + fgSizer1 + wxFLEX_GROWMODE_SPECIFIED + none + 0 + 0 + + 5 + wxALL + 0 + + 1 + 1 + 1 + 1 + + + + + + + + 1 + 0 + 1 + + 1 + 0 + Dock + 0 + Left + 1 + + 1 + + 0 + 0 + wxID_ANY + Gap Duration: + + 0 + + + 0 + + 1 + lblGap + 1 + + + protected + 1 + + Resizable + 1 + + + + 0 + + + + + -1 + + + + + + + + + + + + + + + + + + + + + + + + + + + 5 + wxALL + 0 + + 1 + 1 + 1 + 1 + + + + + + + + 1 + 0 + 1 + + 1 + 0 + Dock + 0 + Left + 1 + + 1 + + 0 + 0 + wxID_ANY + + 0 + + + + 0 + + 1 + txtGap + 1 + + + protected + 1 + + Resizable + 1 + + + + 0 + + + wxFILTER_NONE + wxDefaultValidator + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 5 + wxALL + 1 + + 1 + 1 + 1 + 1 + + + + + + + + 1 + 0 + "second" "minute" "hour" "days" "week" "month" "day" "year" + 1 + + 1 + 0 + Dock + 0 + Left + 1 + + 1 + + 0 + 0 + wxID_ANY + + 0 + + + 0 + + 1 + cbGap + 1 + + + protected + 1 + + Resizable + 1 + 1 + + + + 0 + + + wxFILTER_NONE + wxDefaultValidator + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 5 + wxALL + 0 + + 1 + 1 + 1 + 1 + + + + + + + + 1 + 0 + 1 + + 1 + 0 + Dock + 0 + Left + 1 + + 1 + + 0 + 0 + wxID_ANY + Fill Frequency: + + 0 + + + 0 + + 1 + lblFill + 1 + + + protected + 1 + + Resizable + 1 + + + + 0 + + + + + -1 + + + + + + + + + + + + + + + + + + + + + + + + + + + 5 + wxALL + 0 + + 1 + 1 + 1 + 1 + + + + + + + + 1 + 0 + 1 + + 1 + 0 + Dock + 0 + Left + 1 + + 1 + + 0 + 0 + wxID_ANY + + 0 + + + + 0 + + 1 + txtFill + 1 + + + protected + 1 + + Resizable + 1 + + + + 0 + + + wxFILTER_NONE + wxDefaultValidator + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 5 + wxALL + 1 + + 1 + 1 + 1 + 1 + + + + + + + + 1 + 0 + "second" "minute" "hour" "day" "week" "month" "year" + 1 + + 1 + 0 + Dock + 0 + Left + 1 + + 1 + + 0 + 0 + wxID_ANY + + 0 + + + 0 + + 1 + cbFill + 1 + + + protected + 1 + + Resizable + 1 + 1 + + + + 0 + + + wxFILTER_NONE + wxDefaultValidator + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 5 + wxEXPAND + 1 + + 0 + 1 + 0 + 0 + 0 + 1 + 0 + 0 + + m_sdbSizer1 + protected + + OnCancelBtn + + + + onOKBtn + + + + + + + + diff --git a/odmtools/controller/frmGapFill.py b/odmtools/controller/frmGapFill.py new file mode 100644 index 0000000..ce34a3b --- /dev/null +++ b/odmtools/controller/frmGapFill.py @@ -0,0 +1,26 @@ +"""Subclass of dlgFill, which is generated by wxFormBuilder.""" + +from odmtools.view import clsGapFill + + +# Implementing dlgFill +class frmGapFill(clsGapFill.dlgFill): + def __init__(self, parent, record_service): + self.record_service = record_service + clsGapFill.dlgFill.__init__(self, parent) + + # Handlers for dlgFill events. + def onOKBtn(self, event): + #TODO add validation + gapvalue= self.txtGap.Value + gaptime = self.cbGap.Value + fillvalue = self.txtFill.Value + filltime= self.cbFill.Value + + self.record_service.fill_gap(gap=[gapvalue, gaptime], fill=[fillvalue, filltime]) + self.Close() + + def OnCancelBtn(self, event): + + self.Close() + self.Destroy() diff --git a/odmtools/controller/logicEditTools.py b/odmtools/controller/logicEditTools.py index 25b7ae2..40e64d8 100644 --- a/odmtools/controller/logicEditTools.py +++ b/odmtools/controller/logicEditTools.py @@ -50,6 +50,19 @@ def filter_date(self, endDate, startDate): else: return "Cannot filter: %s" % (self._edit_error) + + def fill_gap(self, gap, fill): + self._edit_service.fill_gap(gap , fill) + + self.refresh_edit() + if self._record: + self._script("edit_service.fill_gap(gap = %s, fill= %s)\n" % (gap, fill), 'black') + Publisher.sendMessage("scroll") + + + + + def data_gaps(self, value, time_period): self._edit_service.data_gaps(value, time_period) self.refresh_selection() @@ -57,7 +70,6 @@ def data_gaps(self, value, time_period): self._script("edit_service.data_gaps(%s, '%s')\n" % (value, time_period), 'black') Publisher.sendMessage("scroll") - def value_change_threshold(self, value, operator): self._edit_service.change_value_threshold(value, operator) self.refresh_selection() diff --git a/odmtools/gui/mnuRibbon.py b/odmtools/gui/mnuRibbon.py index 360f6cc..91efa14 100755 --- a/odmtools/gui/mnuRibbon.py +++ b/odmtools/gui/mnuRibbon.py @@ -14,6 +14,7 @@ from frmFlagValues import frmFlagValues from odmtools.controller.frmLinearDrift import frmLinearDrift from odmtools.controller.frmAbout import frmAbout +from odmtools.controller.frmGapFill import frmGapFill import wizSave from odmtools.common.icons import * import pandas as pd @@ -36,9 +37,9 @@ wxID_RIBBONEDITSCRIPTSAVE, wxID_RIBBONVIEWPLOT, wxID_RIBBONVIEWTABLE, wxID_RIBBONVIEWSERIES, wxID_RIBBONVIEWCONSOLE, wxID_RIBBONVIEWSCRIPT, wxID_RIBBONPLOTBLANKBTN, wxID_FileMenu, wxID_STARTDPDATE, wxID_ENDDPDATE, wxID_FRAME1SPINCTRL1, wxID_RIBBONEDITFILTER, wxID_RIBBONEDITRECORD, wxID_RIBBONEDITLINFILTER, wxID_RIBBONPLOTDATEAPPLY, - wxID_RIBBONEDITRESETFILTER, wxID_RIBBONRECORDNEW, wxID_RIBBONRECORDOPEN, wxID_RIBBONRECORDSAVE] = [wx.NewId() for + wxID_RIBBONEDITRESETFILTER, wxID_RIBBONRECORDNEW, wxID_RIBBONRECORDOPEN, wxID_RIBBONRECORDSAVE, wxID_GAPFILL] = [wx.NewId() for _init_ctrls in - range(46)] + range(47)] ## ################################# ## Build Menu and Toolbar @@ -155,6 +156,7 @@ def _init_ctrls(self, prnt): self.edit_bar.AddSimpleButton(wxID_RIBBONEDITFLAG, "Flag", flag.GetBitmap(), "") self.edit_bar.AddSimpleButton(wxID_RIBBONEDITADDPOINT, "Add Point", add.GetBitmap(), "") self.edit_bar.AddSimpleButton(wxID_RIBBONEDITDELPOINT, "Delete Point", delete.GetBitmap(), "") + self.edit_bar.AddSimpleButton(wxID_GAPFILL, "Fill Gap", add.GetBitmap(), "") #self.edit_bar.AddSimpleButton(wxID_RIBBONEDITRECORD, "Record", bitmap=record.GetBitmap(), help_string="",kind=0x4) self.edit_bar.EnableButton(wxID_RIBBONEDITFILTER, False) @@ -250,6 +252,7 @@ def bindEvents(self): self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onEditFlag, id=wxID_RIBBONEDITFLAG) self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onEditAddPoint, id=wxID_RIBBONEDITADDPOINT) self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onEditDelPoint, id=wxID_RIBBONEDITDELPOINT) + self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onGapFill, id = wxID_GAPFILL) self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onRecordNew, id=wxID_RIBBONRECORDNEW) self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onRecordOpen, id=wxID_RIBBONRECORDOPEN) @@ -466,7 +469,23 @@ def onEditAddPoint(self, event): addPoint.ShowModal() event.Skip() + # ################################### + # Gap fill + # ################################### + def onGapFill(self, event): + + gap_fill = frmGapFill(self.parent, self.parent.getRecordService()) + + if gap_fill.Show() == wx.OK: + gap_fill.Destroy() + + + + + + + event.Skip() # ################################### # Delete Point # ################################### diff --git a/odmtools/gui/plotProbability.py b/odmtools/gui/plotProbability.py index 6e23761..7cf1551 100644 --- a/odmtools/gui/plotProbability.py +++ b/odmtools/gui/plotProbability.py @@ -136,7 +136,9 @@ def updatePlot(self): #self.prob.append( #prop = oneSeries.Probability.plot(column="DataValue", ax=self.plots) #todo FutureWarning: order is deprecated, use sort_values(...) + #xValues = oneSeries.Probability.xAxis.order().values xValues = oneSeries.Probability.xAxis.order().values + # yValues = oneSeries.Probability.yAxis.order().values yValues = oneSeries.Probability.yAxis.order().values ax = self.plots.plot(xValues, yValues, 'bs', color=oneSeries.color, diff --git a/odmtools/odmdata/memory_database.py b/odmtools/odmdata/memory_database.py index 2ae1763..9b2ab59 100644 --- a/odmtools/odmdata/memory_database.py +++ b/odmtools/odmdata/memory_database.py @@ -201,7 +201,7 @@ def initEditValues(self, seriesID): """ if not self.editLoaded: logger.debug("Load series from db") - + self.series = self.series_service.get_series_by_id(seriesID) self.df = self.series_service.get_values_by_series(seriesID) self.editLoaded = True diff --git a/odmtools/odmservices/edit_service.py b/odmtools/odmservices/edit_service.py index d3448fd..ddfc019 100644 --- a/odmtools/odmservices/edit_service.py +++ b/odmtools/odmservices/edit_service.py @@ -17,6 +17,21 @@ # logger = tool.setupLogger(__name__, __name__ + '.log', 'w', logging.DEBUG) logger =logging.getLogger('main') +class time(object): + time_units = { + 'second': 's', + 'minute': 'm', + 'hour': 'h', + 'day': 'D', + 'week': 'W', + 'month': 'M', + 'year': 'Y' + } + + def __init__(self, value, time_period): + self.value = value + self.time_period = time_period + class EditService(): # Mutual exclusion: cursor, or connection_string def __init__(self, series_id, connection=None, connection_string="", debug=False): @@ -86,6 +101,7 @@ def _test_filter_previous(self): return df + def datetime2dataframe(self, datetime_list): """ Converts datetime_list to a pandas Dataframe @@ -141,19 +157,51 @@ def filter_date(self, before, after): if before and after: self.filtered_dataframe = df[(df.index < before) & (df.index > after)] + def fill_gap(self, gap, fill): + + df = self.memDB.getDataValuesDF() + gaps= self.find_gaps(df, gap[0], gap[1]) + points = [] + series= self.memDB.series + timegap = np.timedelta64(fill[0], self.time_units[fill[1]]) + + #if gaps is not of type dataframe- put it in a dataframe + #if not isinstance(gaps, pd.DataFrame + for g in gaps.iterrows(): + row = g[1] + e = row.datetime + s = row.dateprev + + #prime the loop + s = s + timegap + # for each gap time period in the larger gap ( until datetime = prev value) + while s < e: + utc_offset = (series.begin_date_time-series.begin_date_time_utc).total_seconds()/3600 + points.append((-9999, None, s, utc_offset, s, None, None, u'nc', None, None, series.site_id, series.variable_id, series.method_id, series.source_id, series.quality_control_level_id)) + #('-9999', None, DATE, series.begin_date_time_utc, UTCDATE, None, None, u'nc', None, None, + # series.site_id, series.variable_id, series.method_id, series.source_id, + # series.quality_control_level_id + + s = s + timegap + #print points + self.add_points(points) + + time_units = { + 'second': 's', + 'minute': 'm', + 'hour': 'h', + 'day': 'D', + 'week': 'W', + 'month': 'M', + 'year': 'Y' + } + # Data Gaps - def data_gaps(self, value, time_period): - df = self._test_filter_previous() - time_units = { - 'second': 's', - 'minute': 'm', - 'hour': 'h', - 'day': 'D', - 'week': 'W', - 'month': 'M', - 'year': 'Y' - } + + def find_gaps(self, df, value, time_period): + + # make a copy of the dataframe in order to modify it to be in the form we need to determine data gaps copy_df = df @@ -165,19 +213,29 @@ def data_gaps(self, value, time_period): value = int(value) # create a bool column indicating which rows meet condition - filtered_results = copy_df['datetime'].diff() >= np.timedelta64(value, time_units[time_period]) + filtered_results = copy_df['datetime'].diff() > np.timedelta64(value, self.time_units[time_period]) # filter on rows that passed previous condition - copy_df = copy_df[filtered_results] + return copy_df[filtered_results] + + + + def data_gaps(self, value, time_period): + df = self._test_filter_previous() + copy_df = self.find_gaps(df, value, time_period) + print (copy_df) # merge values and remove duplicates. this hack allows for both values to be marked when selecting data gaps newdf = pd.concat([copy_df['datetime'], copy_df['dateprev']], join='inner') - self.filtered_dataframe = df[df.index.isin(newdf.drop_duplicates().dropna())] # clean up del copy_df - del filtered_results - del newdf + + + self.filtered_dataframe= df[df.index.isin(newdf.drop_duplicates().dropna())] + + + def change_value_threshold(self, value, operator): @@ -310,7 +368,6 @@ def change_value(self, value, operator): def add_points(self, points): # todo: add the ability to send in multiple datetimes to a single 'point' - self.memDB.addPoints(points) self._populate_series() diff --git a/odmtools/view/clsGapFill.py b/odmtools/view/clsGapFill.py new file mode 100644 index 0000000..95c1208 --- /dev/null +++ b/odmtools/view/clsGapFill.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- + +########################################################################### +## Python code generated with wxFormBuilder (version Jun 17 2015) +## http://www.wxformbuilder.org/ +## +## PLEASE DO "NOT" EDIT THIS FILE! +########################################################################### + +import wx +import wx.xrc + + +########################################################################### +## Class dlgFill +########################################################################### + +class dlgFill(wx.Dialog): + def __init__(self, parent): + wx.Dialog.__init__(self, parent, id=wx.ID_ANY, title=wx.EmptyString, pos=wx.DefaultPosition, + size=wx.Size(315, 217), style=wx.DEFAULT_DIALOG_STYLE) + + self.SetSizeHintsSz(wx.Size(315, 217), wx.Size(315, 217)) + + bsForm = wx.BoxSizer(wx.VERTICAL) + + self.lblInstructions = wx.StaticText(self, wx.ID_ANY, + u"This function fills any gaps greater than the gap duration with a no-data value at the fill frequency.", + wx.DefaultPosition, wx.DefaultSize, 0) + self.lblInstructions.Wrap(300) + bsForm.Add(self.lblInstructions, 0, wx.ALL, 5) + + fgSizer1 = wx.FlexGridSizer(0, 3, 0, 0) + fgSizer1.SetFlexibleDirection(wx.BOTH) + fgSizer1.SetNonFlexibleGrowMode(wx.FLEX_GROWMODE_SPECIFIED) + + self.lblGap = wx.StaticText(self, wx.ID_ANY, u"Gap Duration:", wx.DefaultPosition, wx.DefaultSize, 0) + self.lblGap.Wrap(-1) + fgSizer1.Add(self.lblGap, 0, wx.ALL, 5) + + self.txtGap = wx.TextCtrl(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, 0) + fgSizer1.Add(self.txtGap, 0, wx.ALL, 5) + + cbGapChoices = [u"second", u"minute", u"hour", u"days", u"week", u"month", u"day", u"year"] + self.cbGap = wx.ComboBox(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, cbGapChoices, 0) + self.cbGap.SetSelection(1) + fgSizer1.Add(self.cbGap, 1, wx.ALL, 5) + + self.lblFill = wx.StaticText(self, wx.ID_ANY, u"Fill Frequency:", wx.DefaultPosition, wx.DefaultSize, 0) + self.lblFill.Wrap(-1) + fgSizer1.Add(self.lblFill, 0, wx.ALL, 5) + + self.txtFill = wx.TextCtrl(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, 0) + fgSizer1.Add(self.txtFill, 0, wx.ALL, 5) + + cbFillChoices = [u"second", u"minute", u"hour", u"day", u"week", u"month", u"year"] + self.cbFill = wx.ComboBox(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, cbFillChoices, 0) + self.cbFill.SetSelection(1) + fgSizer1.Add(self.cbFill, 1, wx.ALL, 5) + + bsForm.Add(fgSizer1, 1, wx.EXPAND, 5) + + m_sdbSizer1 = wx.StdDialogButtonSizer() + self.m_sdbSizer1OK = wx.Button(self, wx.ID_OK) + m_sdbSizer1.AddButton(self.m_sdbSizer1OK) + self.m_sdbSizer1Cancel = wx.Button(self, wx.ID_CANCEL) + m_sdbSizer1.AddButton(self.m_sdbSizer1Cancel) + m_sdbSizer1.Realize(); + + bsForm.Add(m_sdbSizer1, 1, wx.EXPAND, 5) + + self.SetSizer(bsForm) + self.Layout() + + self.Centre(wx.BOTH) + + # Connect Events + self.m_sdbSizer1Cancel.Bind(wx.EVT_BUTTON, self.OnCancelBtn) + self.m_sdbSizer1OK.Bind(wx.EVT_BUTTON, self.onOKBtn) + + def __del__(self): + pass + + # Virtual event handlers, overide them in your derived class + def OnCancelBtn(self, event): + event.Skip() + + def onOKBtn(self, event): + event.Skip() + + diff --git a/odmtools/view/clsSeriesSelector.py b/odmtools/view/clsSeriesSelector.py index fa4a00f..42ed9b1 100755 --- a/odmtools/view/clsSeriesSelector.py +++ b/odmtools/view/clsSeriesSelector.py @@ -155,7 +155,7 @@ def _init_ctrls(self): self.cbVariables = wx.ComboBox(choices=[], id=wxID_PNLSERIESSELECTORCBVARIABLES, name=u'cbVariables', parent=self.pnlVar, pos=wx.Point(100, 0), size=wx.Size(700, 25), style=wx.CB_READONLY, - value='comboBox4') + value='') self.cbVariables.SetLabel(u'') self.cbVariables.Enable(False)