summaryrefslogtreecommitdiffstats
path: root/Monitoring/MonitoringService/DataProcessing/Data.py
blob: 60ace6670d0a44687a582d5f8b0e2785764178e2 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
'''
Created on Sep 1, 2011

@author: steger, jozsef
@organization: ELTE
@contact: steger@complex.elte.hu
@author: laki, sandor
'''

from threading import RLock, Event
from DataProcessing.DataHeader import DataHeaderCell, DataHeader, DataError
from DataProcessing.Unit import UnitManager


class Data(object):
    '''
    @author: steger, jozsef
    @summary: 
    This class implements the representation of data provided by a tool or other sort of data source.
    This class contains the data in a tabular format. 
    The meta information of the columns are dictated by a DataHeader instance.
    All items in the same column are data of the same kind, 
    whereas all data in the same record (row) are correlated.
    
    Contents of cells of a given column are either single items or Data objects 
    as prescribed by the header of the table.
    
    New records can be added using the Record class, for which a template generator is provided by the class.
    
    DataReaders access the content of this class, they need to register Events to get notified 
    of new data insertion or of content erasure.
    '''
    
    class Record(object):
        '''
        @author: steger, jozsef
        @summary: 
        This class represents a given set of records that can be appended to a Data table. 
        It provides useful methods manipulate data within the record.
        '''
        def __init__(self, unitmanager, dataheader, size = 1):
            '''
            @summary: Constructor
            @param unitmanager: necessary to handle conversion
            @type unitmanager: UnitManager
            @param dataheader: the record conforms to the data header provided here
            @type dataheader: DataHeader
            @param size: the number of items to handle at once, default is 1
            @type size: integer 
            '''
            self.um = unitmanager
            self.record = {}
            self.units = {}
            self.subheaders = {}
            self.subrecords = {}
            self.size = size
            self.names = dataheader._cellnames
            for name, cell in dataheader._cells.iteritems():
                if isinstance(cell, DataHeaderCell):
                    self.record[name] = [ None ] * self.size
                    self.units[name] = cell.unit
                elif isinstance(cell, DataHeader):
                    self.subheaders[name] = cell
                else:
                    raise DataError("Data header declaration is wrong")
        
        def __str__(self):
            return "<DataRecord %s, size: %d (%s; %s)>: " % (id(self), self.size, ','.join(self.record.keys()), ','.join(self.subheaders.keys()))
        
        def clear(self, size = None):
            '''
            @summary: Clean the record containers and optionally resize the container
            @note: if DataRecord container is resized, sub record pointers are invalidated
            @param size: the requested new size of the container, default is None, which means keep the original size
            @type size: integer
            '''
            if size is None:
                for name in self.record.keys():
                    self.record[name] = [ None ] * self.size
                    if self.subrecords.has_key(name):
                        for r in self.subrecords[name]:
                            r.clear()
            else:
                self.size = size
                for name in self.record.keys():
                    self.record[name] = [ None ] * self.size
                self.subrecords.clear()
                    
        def getRecordTemplates(self, name, sizes = None):
            '''
            @summary: Sub record templates are pointing to table valued cells. This method allocates container to those data structures.
            @param name: the column name, that point to table valued columns
            @type name: string
            @param sizes: a list of integers that indicate the sizes of each sub tables. Default is None, which means the allocation of single row containers
            @type sizes: list/tuple of integers or None
            @return: a list of Record containers with size items
            @rtype: a list of Record
            @raise DataError: column name not found / wrong record sizes
            '''
            if sizes == None:
                sizes = [1] * self.size
            if len(sizes) != self.size:
                raise DataError("wrong record sizes requested")
            if not self.subheaders.has_key(name):
                raise DataError("Cannot find column name: %s" % name)
            hdr = self.subheaders[name]
            self.subrecords[name] = []
            while len(sizes):
                self.subrecords[name].append( Data.Record(unitmanager = self.um, dataheader = hdr, size = sizes.pop(0)) )
            return self.subrecords[name]
            
        def update(self, name, values, unit = None):
            '''
            @summary: Update a the column with the new value and make sure the unit is converted to the current unit of the model
            @param name: the name of the column
            @type name: string
            @param values: a list of data values to update the cells
            @type values: list
            @param unit: the unit of the values in the list, default is None, which means it is the same as the current unit stated in the unit model
            @type unit: string or None
            @raise DataError: missing column name / table valued cells / size mismatch
            '''
            if not self.record.has_key(name):
                raise DataError("Record has no column named %s" % name)
            if not self.units.has_key(name):
                raise DataError("Cannot update column named %s (table valued cells)" % name)
            if len(values) != self.size:
                raise DataError("The size of values don't match expected %d and got %d" % (len(values), self.size))
            if unit is None:
                self.record[name] = values[:]
            elif isinstance(unit, UnitManager.Unit):
                myunit = self.units[name]
                if unit == myunit:
                    self.record[name] = values[:]
                else:
                    self.record[name] = [ self.um.convert(value = quantity, from_unit = unit, to_unit = myunit) for quantity in values ]
            else:
                raise DataError("wrong type of unit")
        
        def updateMany(self, names, values, units = None):
            '''
            @summary: Update more columns with a single call
            @param names: a list of the non-table valued columns to update
            @type names: list/tuple of string
            @param values: a matrix of data values
            @type values: list of list of value
            @param units: a list of units corresponding to each columns, default is None, meaning everything is expected to be in the current unit
            @type units: list/tuple of sting or None
            @raise DataError: size mismatch / unknown column name
            '''
            names = list(names)
            if len(values) != self.size:
                raise DataError("The size of values don't match %d" % self.size)
            for name in names:
                if not self.record.has_key(name):
                    raise DataError("Record has no column named %s" % name)
            transpose = dict( map(lambda n: (n, []), names) )
            s = len(names)
            idxs = range(s)
            while len(values):
                value = values.pop(0)
                if len(value) == s:
                    for idx in idxs:
                        transpose[names[idx]].append(value.pop(0))
                else:
                    raise DataError("Record size does not match")
            if units is None:
                units = [ None ] * s
            else:
                units = list(units)
            while len(names):
                name = names.pop(0)
                unit = units.pop(0)
                self.update(name = name, values = transpose[name], unit = unit)
            
        def extract(self):
            '''
            @summary: Extract values stored in this record represented in a list in the order of names
            @return: a list of values
            @rtype: list
            '''
            retval = []
            idx = 0
            while idx < self.size:
                rec = []
                for name in self.names:
                    if self.record.has_key(name):
                        rec.append( self.record[name][idx] )
                    elif self.subrecords.has_key(name):
                        rec.append( self.subrecords[name][idx].extract() ) 
                idx += 1
                retval.append(tuple(rec))
            return retval
    
    def __init__(self, unitmanager, header):
        '''
        @summary: Constructor
        @param unitmanager: necessary to handle conversion
        @type unitmanager: UnitManager
        @param header: the header declaration of the data table
        @type header: DataHeader
        @raise DataError: raised upon wrong table header is given
        '''
        if not isinstance(header, DataHeader):
            raise DataError("attempt to allocate table with a wrong header")
        self.um = unitmanager
        self.header = header
        self._rawrecords = []
        self._chunks = []
        self.readlock = RLock()
        self.writelock = RLock()
        self.evExpanded = Event()
        self.evCleared = Event()

    def __str__(self):
        '''
        @summary: returns the name of the table and the python object id
        @return: abbreviated representation of the table
        @rtype: string
        '''
        return "<Data: %s %s>" % (self.header.name, id(self))
    
    def __len__(self):
        return len(self._rawrecords)

    def __getitem__(self, k):
        return self._rawrecords.__getitem__(k)
    
    @property
    def name(self):
        '''
        @summary: the name of the data is defined by the header
        @return: the name of the header
        @rtype: string
        '''
        return self.header.name

    @property
    def tail(self):
        '''
        @summary: Tail property indicates how many new records have been saved to the table in the last call
        @return: number of new records
        @rtype: integer
        '''
        try:
            return self._chunks[-1]
        except IndexError:
            return 0
                 
    def getTemplate(self, size = 1):
        '''
        @summary: Generate a helper class to extend the table with new values
        @param size: the size of the new records wished to handle together, default is 1
        @type size: integer
        @return: an empty row with the structure dictated by the header of the table
        @rtype: Record
        '''
        return self.Record(unitmanager = self.um, dataheader = self.header.getHeader(self.header.name), size = size)

    def saveRecord(self, record):
        '''
        @summary: append values stored in the record to the table
        @param record: a record with new data values
        @type record: DataRecord
        '''
        #TODO: check if record is not corrupted
        newrecords = record.extract()
        with self.writelock:
            self._rawrecords.extend( newrecords )
            self._chunks.append( len(newrecords) )
            self.evExpanded.set()
    
    def clear(self):
        '''
        @summary: delete all data records stored
        '''
        with self.writelock:
            self._rawrecords = []
            self._chunks = []
            self.evCleared.set()