summaryrefslogtreecommitdiffstats
path: root/Monitoring/src/main/python/DataProcessing/Data.py
blob: d6f6dbcefdc24e81627f00cdc72cbc1b6326a527 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
from __future__ import with_statement
'''
Created on Sep 1, 2011

@author: steger, jozsef
@organization: ELTE
@contact: steger@complex.elte.hu
@author: laki, sandor
'''

from threading import Lock, RLock
from DataProcessing.DataHeader import DataHeaderCell, DataHeader, DataError
from DataProcessing.Unit import UnitManager
from DataProcessing.DataSource import DataSource


class Data(DataSource):
    '''
    @author: steger, jozsef
    @summary: 
    This class contains the data in a tabular format. 
    All items in the same column are data of the same kind 
    whereas all data in the same record (row) are correlated.
    
    Contents of cells of a given column are either single items or Data objects 
    as dictated by the header of the table.
    
    New records can be added using the Record class, for which template generator is provided by this class.
    
    DataReaders and other consumers can register Events at new data insertion or at clear.
    '''
    
    class Record(object):
        '''
        @author: steger, jozsef
        @summary: 
        This class represents a given set of records that can be appended to a Data table. 
        It provides useful methods manipulate data within the record.
        '''
        def __init__(self, unitmanager, dataheader, size = 1):
            '''
            @summary: Constructor
            @param unitmanager: necessary to handle conversion
            @type unitmanager: UnitManager
            @param dataheader: the record conforms to the data header provided here
            @type dataheader: DataHeader
            @param size: the number of items to handle at once, default is 1
            @type size: integer 
            '''
            self.um = unitmanager
            self.record = {}
            self.units = {}
            self.subheaders = {}
            self.subrecords = {}
            self.size = size
            self.names = dataheader._cellnames
            for name, cell in dataheader._cells.iteritems():
                if isinstance(cell, DataHeaderCell):
                    self.record[name] = [ None ] * self.size
                    self.units[name] = cell.unit
                elif isinstance(cell, DataHeader):
                    self.subheaders[name] = cell
                else:
                    raise DataError("Data header declaration is wrong")
        
        def __str__(self):
            return "<DataRecord %s, size: %d (%s; %s)>: " % (id(self), self.size, ','.join(self.record.keys()), ','.join(self.subheaders.keys()))
        
        def clear(self, size = None):
            '''
            @summary: Clean the record containers and optionally resize the container
            @note: if DataRecord container is resized, sub record pointers are invalidated
            @param size: the requested new size of the container, default is None, which means keep the original size
            @type size: integer
            '''
            if size is None:
                for name in self.record.keys():
                    self.record[name] = [ None ] * self.size
                    if self.subrecords.has_key(name):
                        for r in self.subrecords[name]:
                            r.clear()
            else:
                self.size = size
                for name in self.record.keys():
                    self.record[name] = [ None ] * self.size
                self.subrecords.clear()
                    
        def getRecordTemplates(self, name, sizes = None):
            '''
            @summary: Sub record templates are pointing to table valued cells. This method allocates container to those data structures.
            @param name: the column name, that point to table valued columns
            @type name: string
            @param sizes: a list of integers that indicate the sizes of each sub tables. Default is None, which means the allocation of single row containers
            @type sizes: list/tuple of integers or None
            @return: a list of Record containers with size items
            @rtype: a list of Record
            @raise DataError: column name not found / wrong record sizes
            '''
            if sizes == None:
                sizes = [1] * self.size
            if len(sizes) != self.size:
                raise DataError("wrong record sizes requested")
            if not self.subheaders.has_key(name):
                raise DataError("Cannot find column name: %s" % name)
            hdr = self.subheaders[name]
            self.subrecords[name] = []
            while len(sizes):
                self.subrecords[name].append( Data.Record(unitmanager = self.um, dataheader = hdr, size = sizes.pop(0)) )
            return self.subrecords[name]
            
        def update(self, name, values, unit = None):
            '''
            @summary: Update a the column with the new value and make sure the unit is converted to the current unit of the model
            @param name: the name of the column
            @type name: string
            @param values: a list of data values to update the cells
            @type values: list
            @param unit: the unit of the values in the list, default is None, which means it is the same as the current unit stated in the unit model
            @type unit: string or None
            @raise DataError: missing column name / table valued cells / size mismatch
            '''
            if not self.record.has_key(name):
                raise DataError("Record has no column named %s" % name)
            if not self.units.has_key(name):
                raise DataError("Cannot update column named %s (table valued cells)" % name)
            if len(values) != self.size:
                raise DataError("The size of values don't match expected %d and got %d" % (len(values), self.size))
            if unit is None:
                self.record[name] = values[:]
            elif isinstance(unit, UnitManager.Unit):
                myunit = self.units[name]
                if unit == myunit:
                    self.record[name] = values[:]
                else:
                    self.record[name] = [ self.um.convert(value = quantity, from_unit = unit, to_unit = myunit) for quantity in values ]
            else:
                raise DataError("wrong type of unit")
        
        def updateMany(self, names, values, units = None):
            '''
            @summary: Update more columns with a single call
            @param names: a list of the non-table valued columns to update
            @type names: list/tuple of string
            @param values: a matrix of data values
            @type values: list of list of value
            @param units: a list of units corresponding to each columns, default is None, meaning everything is expected to be in the current unit
            @type units: list/tuple of sting or None
            @raise DataError: size mismatch / unknown column name
            '''
            names = list(names)
            if len(values) != self.size:
                raise DataError("The size of values don't match %d" % self.size)
            for name in names:
                if not self.record.has_key(name):
                    raise DataError("Record has no column named %s" % name)
            transpose = dict( map(lambda n: (n, []), names) )
            s = len(names)
            idxs = range(s)
            while len(values):
                value = values.pop(0)
                if len(value) == s:
                    for idx in idxs:
                        transpose[names[idx]].append(value.pop(0))
                else:
                    raise DataError("Record size does not match")
            if units is None:
                units = [ None ] * s
            else:
                units = list(units)
            while len(names):
                name = names.pop(0)
                unit = units.pop(0)
                self.update(name = name, values = transpose[name], unit = unit)
            
        def extract(self):
            '''
            @summary: Extract values stored in this record represented in a list in the order of names
            @return: a list of values
            @rtype: list
            '''
            retval = []
            idx = 0
            while idx < self.size:
                rec = []
                for name in self.names:
                    if self.record.has_key(name):
                        rec.append( self.record[name][idx] )
                    elif self.subrecords.has_key(name):
                        rec.append( self.subrecords[name][idx].extract() ) 
                idx += 1
                retval.append(tuple(rec))
            return retval
    
    def __init__(self, unitmanager, header):
        '''
        @summary: Constructor
        @param unitmanager: necessary to handle conversion
        @type unitmanager: UnitManager
        @param header: the header declaration of the data table
        @type header: DataHeader
        @raise DataError: raised upon wrong table header is given
        '''
        if not isinstance(header, DataHeader):
            raise DataError("attempt to allocate table with a wrong header")
        self.um = unitmanager
        DataSource.__init__(self, self)
        self.header = header
        self._rawrecords = []
        self._tail = 0
        self._seq = 0
        self._readlock = RLock()
        self._writelock = Lock()
        self._data = self

    def __str__(self):
        '''
        @summary: returns the name of the table and the python object id
        @return: abbreviated representation of the table
        @rtype: string
        '''
        return "<Data: %s %s>" % (self.header.name, id(self))
    
    def __len__(self):
        return len(self._rawrecords)

    def __getitem__(self, k):
        return self._rawrecords.__getitem__(k)
    
    def _get_data(self):
        return self

    def _get_readlock(self):
        return self._readlock

    def _get_writelock(self):
        return self._writelock
    
    def _get_name(self):
        '''
        @summary: the name of the data is defined by the header
        @return: the name of the header
        @rtype: string
        '''
        return self.header.name

    def _get_tail(self):
        '''
        @summary: Tail property indicates how many new records have been saved to the table in the last call
        @return: number of new records
        @rtype: integer
        '''
        return self._tail
                 
    def getTemplate(self, size = 1):
        '''
        @summary: Generate a helper class to extend the table with new values
        @param size: the size of the new records wished to handle together, default is 1
        @type size: integer
        @return: an empty row with the structure dictated by the header of the table
        @rtype: Record
        '''
        return self.Record(unitmanager = self.um, dataheader = self.header.getHeader(self.header.name), size = size)

    def saveRecord(self, record):
        '''
        @summary: append values stored in the record to the table
        @param record: a record with new data values
        @type record: DataRecord
        '''
        #TODO: check if record is not corrupted
        newrecords = record.extract()
        self._tail = len(newrecords)
        with self.writelock:
            self._rawrecords.extend( newrecords )
            self._onexpand()
    
    def clear(self):
        '''
        @summary: delete all data records stored
        '''
        with self.writelock:
            self._rawrecords = []
            self._tail = 0
            self._seq = 0
            self._onclear()
    

    readlock = property(_get_readlock,None,None)

    tail = property(_get_tail,None,None)

    data = property(_get_data,None,None)

    name = property(_get_name,None,None)

    writelock = property(_get_writelock,None,None)