generic.py

Mon, 26 Jan 2015 17:55:34 +0200

author
Iannis <ioannis@inoe.ro>
date
Mon, 26 Jan 2015 17:55:34 +0200
changeset 31
b243f896e5f4
parent 27
74f7617f5356
child 32
022f6f2bc09c
permissions
-rw-r--r--

Changes (again) duration in licel files.

binietoglou@0 1 # General imports
binietoglou@0 2 import datetime
binietoglou@0 3 from operator import itemgetter
binietoglou@0 4
binietoglou@0 5 # Science imports
binietoglou@0 6 import numpy as np
binietoglou@0 7 import matplotlib as mpl
ulalume3@27 8 from matplotlib.ticker import ScalarFormatter
binietoglou@0 9 from matplotlib import pyplot as plt
binietoglou@0 10 import netCDF4 as netcdf
binietoglou@0 11
binietoglou@1 12 netcdf_format = 'NETCDF3_CLASSIC' # choose one of 'NETCDF3_CLASSIC', 'NETCDF3_64BIT', 'NETCDF4_CLASSIC' and 'NETCDF4'
binietoglou@0 13
binietoglou@0 14
binietoglou@0 15 class BaseLidarMeasurement():
binietoglou@0 16 """ This is the general measurement object.
binietoglou@0 17 It is meant to become a general measurement object
binietoglou@0 18 independent of the input files.
binietoglou@0 19
binietoglou@0 20 Each subclass should implement the following:
binietoglou@0 21 * the import_file method.
binietoglou@0 22 * set the "extra_netcdf_parameters" variable to a dictionary that includes the appropriate parameters.
binietoglou@0 23
binietoglou@0 24 You can override the get_PT method to define a custom procedure to get ground temperature and pressure.
binietoglou@0 25 The one implemented by default is by using the MILOS meteorological station data.
binietoglou@0 26
binietoglou@0 27 """
binietoglou@0 28
binietoglou@0 29 def __init__(self, filelist= None):
binietoglou@0 30 self.info = {}
binietoglou@0 31 self.dimensions = {}
binietoglou@0 32 self.variables = {}
binietoglou@0 33 self.channels = {}
binietoglou@0 34 self.attributes = {}
binietoglou@0 35 self.files = []
binietoglou@0 36 self.dark_measurement = None
binietoglou@0 37 if filelist:
binietoglou@0 38 self.import_files(filelist)
binietoglou@0 39
binietoglou@0 40 def import_files(self,filelist):
binietoglou@0 41 for f in filelist:
binietoglou@0 42 self.import_file(f)
binietoglou@0 43 self.update()
binietoglou@0 44
binietoglou@0 45 def import_file(self,filename):
binietoglou@0 46 raise NotImplementedError('Importing files should be defined in the instrument-specific subclass.')
binietoglou@0 47
binietoglou@0 48 def update(self):
binietoglou@0 49 '''
binietoglou@0 50 Update the the info, variables and dimensions of the lidar measurement based
binietoglou@0 51 on the information found in the channels.
binietoglou@0 52
binietoglou@0 53 Reading of the scan_angles parameter is not implemented.
binietoglou@0 54 '''
binietoglou@0 55
binietoglou@0 56 # Initialize
binietoglou@0 57 start_time =[]
binietoglou@0 58 stop_time = []
binietoglou@0 59 points = []
binietoglou@0 60 all_time_scales = []
binietoglou@0 61 channel_name_list = []
binietoglou@0 62
binietoglou@0 63 # Get the information from all the channels
binietoglou@0 64 for channel_name, channel in self.channels.items():
binietoglou@0 65 channel.update()
binietoglou@0 66 start_time.append(channel.start_time)
binietoglou@0 67 stop_time.append(channel.stop_time)
binietoglou@0 68 points.append(channel.points)
binietoglou@0 69 all_time_scales.append(channel.time)
binietoglou@0 70 channel_name_list.append(channel_name)
binietoglou@0 71
binietoglou@0 72 # Find the unique time scales used in the channels
binietoglou@0 73 time_scales = set(all_time_scales)
binietoglou@0 74
binietoglou@0 75 # Update the info dictionary
binietoglou@0 76 self.info['start_time'] = min(start_time)
binietoglou@0 77 self.info['stop_time'] = max(stop_time)
binietoglou@0 78 self.info['duration'] = self.info['stop_time'] - self.info['start_time']
binietoglou@0 79
binietoglou@0 80 # Update the dimensions dictionary
binietoglou@0 81 self.dimensions['points'] = max(points)
binietoglou@0 82 self.dimensions['channels'] = len(self.channels)
binietoglou@0 83 # self.dimensions['scan angles'] = 1
binietoglou@0 84 self.dimensions['nb_of_time_scales'] = len(time_scales)
binietoglou@0 85
binietoglou@0 86 # Update the variables dictionary
binietoglou@0 87 # Write time scales in seconds
binietoglou@0 88 raw_Data_Start_Time = []
binietoglou@0 89 raw_Data_Stop_Time = []
binietoglou@0 90
binietoglou@0 91 for current_time_scale in list(time_scales):
binietoglou@0 92 raw_start_time = np.array(current_time_scale) - min(start_time) # Time since start_time
binietoglou@0 93 raw_start_in_seconds = np.array([t.seconds for t in raw_start_time]) # Convert in seconds
binietoglou@0 94 raw_Data_Start_Time.append(raw_start_in_seconds) # And add to the list
binietoglou@0 95 # Check if this time scale has measurements every 30 or 60 seconds.
binietoglou@0 96
binietoglou@0 97 duration = self._get_duration(raw_start_in_seconds)
binietoglou@0 98
binietoglou@0 99 raw_stop_in_seconds = raw_start_in_seconds + duration
binietoglou@0 100 raw_Data_Stop_Time.append(raw_stop_in_seconds)
binietoglou@0 101
ioannis@22 102 self.variables['Raw_Data_Start_Time'] = raw_Data_Start_Time
ioannis@22 103 self.variables['Raw_Data_Stop_Time'] = raw_Data_Stop_Time
binietoglou@0 104
binietoglou@0 105 # Make a dictionary to match time scales and channels
binietoglou@0 106 channel_timescales = []
binietoglou@0 107 for (channel_name, current_time_scale) in zip(channel_name_list, all_time_scales):
binietoglou@0 108 # The following lines are PEARL specific. The reason they are here is not clear.
binietoglou@0 109 # if channel_name =='1064BLR':
binietoglou@0 110 # channel_name = '1064'
binietoglou@0 111 for (ts,n) in zip(time_scales, range(len(time_scales))):
binietoglou@0 112 if current_time_scale == ts:
binietoglou@0 113 channel_timescales.append([channel_name,n])
binietoglou@0 114 self.variables['id_timescale'] = dict(channel_timescales)
binietoglou@0 115
binietoglou@0 116 def _get_duration(self, raw_start_in_seconds):
binietoglou@0 117 ''' Return the duration for a given time scale. In some files (ex. Licel) this
binietoglou@0 118 can be specified from the files themselves. In others this must be guessed.
binietoglou@0 119
binietoglou@0 120 '''
binietoglou@0 121 # The old method, kept here for reference
binietoglou@0 122 #dt = np.mean(np.diff(raw_start_in_seconds))
binietoglou@0 123 #for d in duration_list:
binietoglou@0 124 # if abs(dt - d) <15: #If the difference of measuremetns is 10s near the(30 or 60) seconds
binietoglou@0 125 # duration = d
binietoglou@0 126
binietoglou@0 127 duration = np.diff(raw_start_in_seconds)[0]
binietoglou@0 128
binietoglou@0 129 return duration
binietoglou@0 130
binietoglou@0 131 def subset_by_channels(self, channel_subset):
binietoglou@0 132 ''' Get a measurement object containing only the channels with names
binietoglou@0 133 contained in the channel_sublet list '''
binietoglou@0 134
binietoglou@0 135 m = self.__class__() # Create an object of the same type as this one.
binietoglou@0 136 m.channels = dict([(channel, self.channels[channel]) for channel
binietoglou@0 137 in channel_subset])
binietoglou@0 138 m.update()
binietoglou@0 139 return m
binietoglou@0 140
binietoglou@0 141 def subset_by_time(self, start_time, stop_time):
binietoglou@0 142
binietoglou@0 143 if start_time > stop_time:
binietoglou@0 144 raise ValueError('Stop time should be after start time')
binietoglou@0 145
binietoglou@0 146 if (start_time < self.info['start_time']) or (stop_time > self.info['stop_time']):
binietoglou@0 147 raise ValueError('The time interval specified is not part of the measurement')
binietoglou@0 148
binietoglou@0 149 m = self.__class__() # Create an object of the same type as this one.
binietoglou@0 150 for (channel_name, channel) in self.channels.items():
binietoglou@0 151 m.channels[channel_name] = channel.subset_by_time(start_time, stop_time)
binietoglou@0 152 m.update()
binietoglou@0 153 return m
binietoglou@0 154
ulalume3@27 155 def subset_by_bins(self, b_min = 0, b_max = None):
ulalume3@27 156 """Remove some height bins from the file. This could be needed to
ulalume3@27 157 remove aquisition artifacts at the start or the end of the files.
ulalume3@27 158 """
ulalume3@27 159
ulalume3@27 160 m = self.__class__() # Create an object of the same type as this one.
ulalume3@27 161
ulalume3@27 162 for (channel_name, channel) in self.channels.items():
ulalume3@27 163 m.channels[channel_name] = channel.subset_by_bins(b_min, b_max)
ulalume3@27 164
ulalume3@27 165 m.update()
ulalume3@27 166
ulalume3@27 167 return m
ulalume3@27 168
binietoglou@0 169 def r_plot(self):
binietoglou@0 170 #Make a basic plot of the data.
binietoglou@0 171 #Should include some dictionary with params to make plot stable.
binietoglou@0 172 pass
binietoglou@0 173
binietoglou@0 174 def r_pdf(self):
binietoglou@0 175 # Create a pdf report using a basic plot and meta-data.
binietoglou@0 176 pass
binietoglou@0 177
binietoglou@0 178 def save(self):
binietoglou@0 179 #Save the current state of the object to continue the analysis later.
binietoglou@0 180 pass
binietoglou@0 181
binietoglou@0 182 def get_PT(self):
binietoglou@0 183 ''' Sets the pressure and temperature at station level .
binietoglou@0 184 The results are stored in the info dictionary.
binietoglou@0 185 '''
binietoglou@0 186
binietoglou@0 187 self.info['Temperature'] = 10.0
binietoglou@0 188 self.info['Pressure'] = 930.0
binietoglou@0 189
binietoglou@15 190 def subtract_dark(self):
binietoglou@15 191
binietoglou@15 192 if not self.dark_measurement:
binietoglou@15 193 raise IOError('No dark measurements have been imported yet.')
binietoglou@15 194
binietoglou@15 195 for (channel_name, dark_channel) in self.dark_measurement.channels.iteritems():
binietoglou@15 196 dark_profile = dark_channel.average_profile()
binietoglou@15 197 channel = self.channels[channel_name]
binietoglou@15 198
binietoglou@15 199 for measurement_time, data in channel.data.iteritems():
binietoglou@15 200 channel.data[measurement_time] = data - dark_profile
binietoglou@15 201
binietoglou@15 202 channel.update()
binietoglou@15 203
ulalume3@23 204 def save_as_netcdf(self, filename = None):
binietoglou@0 205 """Saves the measurement in the netcdf format as required by the SCC.
ulalume3@23 206 Input: filename. If no filename is provided <measurement_id>.nc will
ulalume3@23 207 be used.
binietoglou@0 208 """
binietoglou@0 209 params = self.extra_netcdf_parameters
binietoglou@0 210 needed_parameters = ['Measurement_ID', 'Temperature', 'Pressure']
binietoglou@0 211
binietoglou@0 212 for parameter in needed_parameters:
binietoglou@0 213 stored_value = self.info.get(parameter, None)
binietoglou@0 214 if stored_value is None:
binietoglou@0 215 raise ValueError('A value needs to be specified for %s' % parameter)
binietoglou@0 216
ulalume3@23 217 if not filename:
ulalume3@23 218 filename = "%s.nc" % self.info['Measurement_ID']
binietoglou@0 219
binietoglou@0 220 dimensions = {'points': 1,
binietoglou@0 221 'channels': 1,
binietoglou@0 222 'time': None,
binietoglou@0 223 'nb_of_time_scales': 1,
binietoglou@0 224 'scan_angles': 1,} # Mandatory dimensions. Time bck not implemented
binietoglou@0 225
binietoglou@0 226 global_att = {'Measurement_ID': None,
binietoglou@0 227 'RawData_Start_Date': None,
binietoglou@0 228 'RawData_Start_Time_UT': None,
binietoglou@0 229 'RawData_Stop_Time_UT': None,
binietoglou@0 230 'RawBck_Start_Date': None,
binietoglou@0 231 'RawBck_Start_Time_UT': None,
binietoglou@0 232 'RawBck_Stop_Time_UT': None,
binietoglou@0 233 'Sounding_File_Name': None,
binietoglou@0 234 'LR_File_Name': None,
binietoglou@0 235 'Overlap_File_Name': None,
binietoglou@0 236 'Location': None,
binietoglou@0 237 'System': None,
binietoglou@0 238 'Latitude_degrees_north': None,
binietoglou@0 239 'Longitude_degrees_east': None,
binietoglou@0 240 'Altitude_meter_asl': None}
binietoglou@0 241
binietoglou@0 242 channel_variables = \
binietoglou@0 243 {'channel_ID': (('channels', ), 'i'),
binietoglou@0 244 'Background_Low': (('channels', ), 'd'),
binietoglou@0 245 'Background_High': (('channels', ), 'd'),
binietoglou@0 246 'LR_Input': (('channels', ), 'i'),
binietoglou@0 247 'DAQ_Range': (('channels', ), 'd'),
binietoglou@0 248 'Depolarization_Factor': (('channels', ), 'd'), }
binietoglou@0 249
binietoglou@0 250
binietoglou@0 251 channels = self.channels.keys()
binietoglou@0 252
binietoglou@0 253 input_values = dict(self.dimensions, **self.variables)
binietoglou@0 254
binietoglou@0 255 # Add some mandatory global attributes
binietoglou@0 256 input_values['Measurement_ID'] = self.info['Measurement_ID']
ioannis@21 257 input_values['RawData_Start_Date'] = self.info['start_time'].strftime('%Y%m%d')
ioannis@21 258 input_values['RawData_Start_Time_UT'] = self.info['start_time'].strftime('%H%M%S')
ioannis@21 259 input_values['RawData_Stop_Time_UT'] = self.info['stop_time'].strftime('%H%M%S')
binietoglou@0 260
binietoglou@0 261 # Add some optional global attributes
binietoglou@0 262 input_values['System'] = params.general_parameters['System']
binietoglou@0 263 input_values['Latitude_degrees_north'] = params.general_parameters['Latitude_degrees_north']
binietoglou@0 264 input_values['Longitude_degrees_east'] = params.general_parameters['Longitude_degrees_east']
binietoglou@0 265 input_values['Altitude_meter_asl'] = params.general_parameters['Altitude_meter_asl']
binietoglou@0 266
binietoglou@0 267 # Open a netCDF4 file
binietoglou@0 268 f = netcdf.Dataset(filename,'w', format = netcdf_format) # the format is specified in the begining of the file
binietoglou@0 269
binietoglou@0 270 # Create the dimensions in the file
binietoglou@0 271 for (d,v) in dimensions.iteritems():
binietoglou@0 272 v = input_values.pop(d, v)
binietoglou@0 273 f.createDimension(d,v)
binietoglou@0 274
binietoglou@0 275 # Create global attributes
binietoglou@0 276 for (attrib,value) in global_att.iteritems():
binietoglou@0 277 val = input_values.pop(attrib,value)
binietoglou@0 278 if val:
ioannis@21 279 setattr(f, attrib, val)
binietoglou@0 280
binietoglou@0 281 """ Variables """
binietoglou@0 282 # Write the values of fixes channel parameters
binietoglou@0 283 for (var,t) in channel_variables.iteritems():
binietoglou@0 284 temp_v = f.createVariable(var,t[1],t[0])
binietoglou@0 285 for (channel, n) in zip(channels, range(len(channels))):
binietoglou@0 286 temp_v[n] = params.channel_parameters[channel][var]
binietoglou@0 287
binietoglou@0 288 # Write the id_timescale values
binietoglou@0 289 temp_id_timescale = f.createVariable('id_timescale','i',('channels',))
binietoglou@0 290 for (channel, n) in zip(channels, range(len(channels))):
binietoglou@0 291 temp_id_timescale[n] = self.variables['id_timescale'][channel]
binietoglou@0 292
binietoglou@0 293 # Laser pointing angle
binietoglou@0 294 temp_v = f.createVariable('Laser_Pointing_Angle','d',('scan_angles',))
binietoglou@0 295 temp_v[:] = params.general_parameters['Laser_Pointing_Angle']
binietoglou@0 296
binietoglou@0 297 # Molecular calculation
binietoglou@0 298 temp_v = f.createVariable('Molecular_Calc','i')
binietoglou@0 299 temp_v[:] = params.general_parameters['Molecular_Calc']
binietoglou@0 300
binietoglou@0 301 # Laser pointing angles of profiles
binietoglou@0 302 temp_v = f.createVariable('Laser_Pointing_Angle_of_Profiles','i',('time','nb_of_time_scales'))
binietoglou@0 303 for (time_scale,n) in zip(self.variables['Raw_Data_Start_Time'],
binietoglou@0 304 range(len(self.variables['Raw_Data_Start_Time']))):
binietoglou@0 305 temp_v[:len(time_scale), n] = 0 # The lidar has only one laser pointing angle
binietoglou@0 306
binietoglou@0 307 # Raw data start/stop time
binietoglou@0 308 temp_raw_start = f.createVariable('Raw_Data_Start_Time','i',('time','nb_of_time_scales'))
binietoglou@0 309 temp_raw_stop = f.createVariable('Raw_Data_Stop_Time','i',('time','nb_of_time_scales'))
binietoglou@0 310 for (start_time, stop_time,n) in zip(self.variables['Raw_Data_Start_Time'],
binietoglou@0 311 self.variables['Raw_Data_Stop_Time'],
binietoglou@0 312 range(len(self.variables['Raw_Data_Start_Time']))):
binietoglou@0 313 temp_raw_start[:len(start_time),n] = start_time
binietoglou@0 314 temp_raw_stop[:len(stop_time),n] = stop_time
binietoglou@0 315
binietoglou@0 316 #Laser shots
binietoglou@0 317 temp_v = f.createVariable('Laser_Shots','i',('time','channels'))
binietoglou@0 318 for (channel,n) in zip(channels, range(len(channels))):
binietoglou@0 319 time_length = len(self.variables['Raw_Data_Start_Time'][self.variables['id_timescale'][channel]])
ulalume3@24 320 # Array slicing stoped working as usual ex. temp_v[:10] = 100 does not work. ??? np.ones was added.
ulalume3@24 321 temp_v[:time_length, n] = np.ones(time_length) * params.channel_parameters[channel]['Laser_Shots']
ulalume3@24 322
binietoglou@0 323 #Raw lidar data
binietoglou@0 324 temp_v = f.createVariable('Raw_Lidar_Data','d',('time', 'channels','points'))
binietoglou@0 325 for (channel,n) in zip(channels, range(len(channels))):
binietoglou@0 326 c = self.channels[channel]
binietoglou@0 327 temp_v[:len(c.time),n, :c.points] = c.matrix
binietoglou@0 328
binietoglou@0 329 self.add_dark_measurements_to_netcdf(f, channels)
binietoglou@0 330
binietoglou@0 331 #Pressure at lidar station
binietoglou@0 332 temp_v = f.createVariable('Pressure_at_Lidar_Station','d')
binietoglou@0 333 temp_v[:] = self.info['Pressure']
binietoglou@0 334
binietoglou@0 335 #Temperature at lidar station
binietoglou@0 336 temp_v = f.createVariable('Temperature_at_Lidar_Station','d')
binietoglou@0 337 temp_v[:] = self.info['Temperature']
binietoglou@0 338
binietoglou@0 339 self.save_netcdf_extra(f)
binietoglou@0 340 f.close()
binietoglou@0 341
binietoglou@0 342 def add_dark_measurements_to_netcdf(self, f, channels):
binietoglou@0 343
binietoglou@0 344 # Get dark measurements. If it is not given in self.dark_measurement
binietoglou@0 345 # try to get it using the get_dark_measurements method. If none is found
binietoglou@0 346 # return without adding something.
binietoglou@0 347 if self.dark_measurement is None:
binietoglou@0 348 self.dark_measurement = self.get_dark_measurements()
binietoglou@0 349
binietoglou@0 350 if self.dark_measurement is None:
binietoglou@0 351 return
binietoglou@0 352
binietoglou@0 353 dark_measurement = self.dark_measurement
binietoglou@0 354
binietoglou@0 355 # Calculate the length of the time_bck dimensions
binietoglou@0 356 number_of_profiles = [len(c.time) for c in dark_measurement.channels.values()]
binietoglou@0 357 max_number_of_profiles = np.max(number_of_profiles)
binietoglou@0 358
binietoglou@0 359 # Create the dimension
binietoglou@0 360 f.createDimension('time_bck', max_number_of_profiles)
binietoglou@0 361
binietoglou@0 362 # Save the dark measurement data
binietoglou@0 363 temp_v = f.createVariable('Background_Profile','d',('time_bck', 'channels', 'points'))
binietoglou@0 364 for (channel,n) in zip(channels, range(len(channels))):
binietoglou@0 365 c = dark_measurement.channels[channel]
binietoglou@0 366 temp_v[:len(c.time),n, :c.points] = c.matrix
binietoglou@0 367
binietoglou@0 368 # Dark profile start/stop time
binietoglou@14 369 temp_raw_start = f.createVariable('Raw_Bck_Start_Time','i',('time_bck','nb_of_time_scales'))
binietoglou@14 370 temp_raw_stop = f.createVariable('Raw_Bck_Stop_Time','i',('time_bck','nb_of_time_scales'))
binietoglou@0 371 for (start_time, stop_time,n) in zip(dark_measurement.variables['Raw_Data_Start_Time'],
binietoglou@0 372 dark_measurement.variables['Raw_Data_Stop_Time'],
binietoglou@0 373 range(len(dark_measurement.variables['Raw_Data_Start_Time']))):
binietoglou@0 374 temp_raw_start[:len(start_time),n] = start_time
binietoglou@0 375 temp_raw_stop[:len(stop_time),n] = stop_time
binietoglou@0 376
binietoglou@0 377 # Dark measurement start/stop time
binietoglou@0 378 f.RawBck_Start_Date = dark_measurement.info['start_time'].strftime('%Y%m%d')
binietoglou@0 379 f.RawBck_Start_Time_UT = dark_measurement.info['start_time'].strftime('%H%M%S')
binietoglou@0 380 f.RawBck_Stop_Time_UT = dark_measurement.info['stop_time'].strftime('%H%M%S')
binietoglou@0 381
binietoglou@0 382
binietoglou@0 383
binietoglou@0 384 def save_netcdf_extra(self, f):
binietoglou@0 385 pass
binietoglou@0 386
binietoglou@0 387 def _gettime(self, date_str, time_str):
binietoglou@0 388 t = datetime.datetime.strptime(date_str+time_str,'%d/%m/%Y%H.%M.%S')
binietoglou@0 389 return t
binietoglou@0 390
binietoglou@0 391 def plot(self):
binietoglou@0 392 for channel in self.channels:
binietoglou@0 393 self.channels[channel].plot(show_plot = False)
binietoglou@0 394 plt.show()
binietoglou@0 395
binietoglou@0 396 def get_dark_measurements(self):
binietoglou@0 397 return None
ioannis@22 398
ioannis@22 399 @property
ioannis@22 400 def mean_time(self):
ioannis@22 401 start_time = self.info['start_time']
ioannis@22 402 stop_time = self.info['stop_time']
ioannis@22 403 dt = stop_time - start_time
ioannis@22 404 t_mean = start_time + dt / 2
ioannis@22 405 return t_mean
binietoglou@0 406
binietoglou@0 407
ulalume3@27 408 class LidarChannel:
binietoglou@0 409
ulalume3@27 410 def __init__(self, channel_parameters):
ulalume3@27 411 c = 299792458 # Speed of light
binietoglou@0 412 self.wavelength = channel_parameters['name']
binietoglou@0 413 self.name = str(self.wavelength)
ulalume3@27 414 self.binwidth = float(channel_parameters['binwidth']) # in microseconds
binietoglou@0 415 self.data = {}
binietoglou@0 416 self.resolution = self.binwidth * c / 2
binietoglou@0 417 self.z = np.arange(len(channel_parameters['data'])) * self.resolution + self.resolution/2.0 # Change: add half bin in the z
binietoglou@0 418 self.points = len(channel_parameters['data'])
binietoglou@0 419 self.rc = []
binietoglou@0 420 self.duration = 60
binietoglou@0 421
binietoglou@0 422 def calculate_rc(self):
ulalume3@27 423 background = np.mean(self.matrix[:,4000:], axis = 1) # Calculate the background from 30000m and above
binietoglou@0 424 self.rc = (self.matrix.transpose()- background).transpose() * (self.z **2)
binietoglou@0 425
binietoglou@0 426
binietoglou@0 427 def update(self):
binietoglou@0 428 self.start_time = min(self.data.keys())
binietoglou@0 429 self.stop_time = max(self.data.keys()) + datetime.timedelta(seconds = self.duration)
binietoglou@0 430 self.time = tuple(sorted(self.data.keys()))
binietoglou@0 431 sorted_data = sorted(self.data.iteritems(), key=itemgetter(0))
binietoglou@0 432 self.matrix = np.array(map(itemgetter(1),sorted_data))
binietoglou@0 433
binietoglou@0 434 def _nearest_dt(self,dtime):
binietoglou@0 435 margin = datetime.timedelta(seconds = 300)
binietoglou@0 436 if ((dtime + margin) < self.start_time)| ((dtime - margin) > self.stop_time):
binietoglou@0 437 print "Requested date not covered in this file"
binietoglou@0 438 raise
binietoglou@0 439 dt = abs(self.time - np.array(dtime))
binietoglou@0 440 dtmin = min(dt)
binietoglou@0 441
binietoglou@0 442 if dtmin > datetime.timedelta(seconds = 60):
binietoglou@0 443 print "Nearest profile more than 60 seconds away. dt = %s." % dtmin
binietoglou@0 444 ind_t = np.where(dt == dtmin)
binietoglou@0 445 ind_a= ind_t[0]
binietoglou@0 446 if len(ind_a) > 1:
binietoglou@0 447 ind_a = ind_a[0]
binietoglou@0 448 chosen_time = self.time[ind_a]
binietoglou@0 449 return chosen_time, ind_a
binietoglou@0 450
binietoglou@0 451 def subset_by_time(self, start_time, stop_time):
binietoglou@0 452
binietoglou@0 453 time_array = np.array(self.time)
binietoglou@0 454 condition = (time_array >= start_time) & (time_array <= stop_time)
binietoglou@0 455
binietoglou@0 456 subset_time = time_array[condition]
binietoglou@0 457 subset_data = dict([(c_time, self.data[c_time]) for c_time in subset_time])
binietoglou@0 458
binietoglou@0 459 #Create a list with the values needed by channel's __init__()
ulalume3@27 460 parameter_values = {'name': self.wavelength,
binietoglou@0 461 'binwidth': self.binwidth,
binietoglou@0 462 'data': subset_data[subset_time[0]],}
ulalume3@27 463
ulalume3@27 464 # We should use __class__ instead of class name, so that this works properly
ulalume3@27 465 # with subclasses
ulalume3@27 466 # Ex: c = self.__class__(parameters_values)
ulalume3@27 467 # This does not currently work with Licel files though
ulalume3@27 468 c = LidarChannel(parameter_values)
ulalume3@27 469 c.data = subset_data
ulalume3@27 470 c.update()
ulalume3@27 471 return c
ulalume3@27 472
ulalume3@27 473 def subset_by_bins(self, b_min = 0, b_max = None):
ulalume3@27 474 """Remove some height bins from the file. This could be needed to
ulalume3@27 475 remove aquisition artifacts at the start or the end of the files.
ulalume3@27 476 """
ulalume3@27 477
ulalume3@27 478 subset_data = {}
ulalume3@27 479
ulalume3@27 480 for ctime, cdata in self.data.items():
ulalume3@27 481 subset_data[ctime] = cdata[b_min:b_max]
ulalume3@27 482
ulalume3@27 483 #Create a list with the values needed by channel's __init__()
ulalume3@27 484 parameters_values = {'name': self.wavelength,
ulalume3@27 485 'binwidth': self.binwidth,
ulalume3@27 486 'data': subset_data[subset_data.keys()[0]],} # We just need any array with the correct length
binietoglou@0 487
ulalume3@27 488 c = LidarChannel(parameters_values)
binietoglou@0 489 c.data = subset_data
binietoglou@0 490 c.update()
binietoglou@0 491 return c
binietoglou@0 492
binietoglou@0 493 def profile(self,dtime, signal_type = 'rc'):
binietoglou@0 494 t, idx = self._nearest_dt(dtime)
binietoglou@0 495 if signal_type == 'rc':
binietoglou@0 496 data = self.rc
binietoglou@0 497 else:
binietoglou@0 498 data = self.matrix
binietoglou@0 499
binietoglou@0 500 prof = data[idx,:][0]
binietoglou@0 501 return prof, t
binietoglou@0 502
binietoglou@0 503 def get_slice(self, starttime, endtime, signal_type = 'rc'):
binietoglou@0 504 if signal_type == 'rc':
binietoglou@0 505 data = self.rc
binietoglou@0 506 else:
binietoglou@0 507 data = self.matrix
binietoglou@0 508 tim = np.array(self.time)
binietoglou@0 509 starttime = self._nearest_dt(starttime)[0]
binietoglou@0 510 endtime = self._nearest_dt(endtime)[0]
binietoglou@0 511 condition = (tim >= starttime) & (tim <= endtime)
binietoglou@0 512 sl = data[condition, :]
binietoglou@0 513 t = tim[condition]
binietoglou@0 514 return sl,t
binietoglou@0 515
binietoglou@15 516 def profile_for_duration(self, tim, duration = datetime.timedelta(seconds = 0), signal_type = 'rc'):
binietoglou@15 517 """ Calculates the profile around a specific time (tim). """
binietoglou@0 518 starttime = tim - duration/2
binietoglou@0 519 endtime = tim + duration/2
binietoglou@0 520 d,t = self.get_slice(starttime, endtime, signal_type = signal_type)
binietoglou@0 521 prof = np.mean(d, axis = 0)
binietoglou@0 522 tmin = min(t)
binietoglou@0 523 tmax = max(t)
binietoglou@0 524 tav = tmin + (tmax-tmin)/2
binietoglou@0 525 return prof,(tav, tmin,tmax)
binietoglou@15 526
binietoglou@15 527 def average_profile(self):
binietoglou@15 528 """ Return the average profile (NOT range corrected) for all the duration of the measurement. """
binietoglou@15 529 prof = np.mean(self.matrix, axis = 0)
binietoglou@15 530 return prof
binietoglou@15 531
binietoglou@17 532 def plot(self, signal_type = 'rc', filename = None, zoom = [0,12000,0,-1], show_plot = True, cmap = plt.cm.jet, z0 = None, title = None, vmin = 0, vmax = 1.3 * 10 ** 7):
binietoglou@0 533 #if filename is not None:
binietoglou@0 534 # matplotlib.use('Agg')
binietoglou@0 535
binietoglou@0 536 fig = plt.figure()
binietoglou@0 537 ax1 = fig.add_subplot(111)
binietoglou@17 538 self.draw_plot(ax1, cmap = cmap, signal_type = signal_type, zoom = zoom, z0 = z0, vmin = vmin, vmax = vmax)
binietoglou@17 539
binietoglou@17 540 if title:
binietoglou@17 541 ax1.set_title(title)
binietoglou@17 542 else:
binietoglou@17 543 ax1.set_title("%s signal - %s" % (signal_type.upper(), self.name))
binietoglou@0 544
binietoglou@0 545 if filename is not None:
binietoglou@0 546 pass
binietoglou@0 547 #plt.savefig(filename)
binietoglou@0 548 else:
binietoglou@0 549 if show_plot:
binietoglou@0 550 plt.show()
binietoglou@0 551 #plt.close() ???
binietoglou@0 552
ulalume3@27 553 def draw_plot(self,ax1, cmap = plt.cm.jet, signal_type = 'rc',
ulalume3@27 554 zoom = [0,12000,0,-1], z0 = None,
ulalume3@27 555 add_colorbar = True, cmap_label = 'a.u.', cb_format = None,
ulalume3@27 556 vmin = 0, vmax = 1.3 * 10 ** 7):
binietoglou@0 557
binietoglou@0 558 if signal_type == 'rc':
binietoglou@0 559 if len(self.rc) == 0:
binietoglou@0 560 self.calculate_rc()
binietoglou@0 561 data = self.rc
binietoglou@0 562 else:
binietoglou@0 563 data = self.matrix
binietoglou@0 564
binietoglou@0 565 hmax_idx = self.index_at_height(zoom[1])
binietoglou@0 566
binietoglou@17 567 # If z0 is given, then the plot is a.s.l.
binietoglou@17 568 if z0:
binietoglou@17 569 ax1.set_ylabel('Altitude a.s.l. [km]')
binietoglou@17 570 else:
binietoglou@17 571 ax1.set_ylabel('Altitude a.g.l. [km]')
binietoglou@17 572 z0 = 0
binietoglou@17 573
binietoglou@17 574 ax1.set_xlabel('Time UTC [hh:mm]')
binietoglou@0 575 #y axis in km, xaxis /2 to make 30s measurements in minutes. Only for 1064
binietoglou@0 576 #dateFormatter = mpl.dates.DateFormatter('%H.%M')
binietoglou@0 577 #hourlocator = mpl.dates.HourLocator()
binietoglou@0 578
binietoglou@0 579 #dayFormatter = mpl.dates.DateFormatter('\n\n%d/%m')
binietoglou@0 580 #daylocator = mpl.dates.DayLocator()
binietoglou@17 581 hourFormatter = mpl.dates.DateFormatter('%H:%M')
binietoglou@19 582 hourlocator = mpl.dates.AutoDateLocator(minticks = 3, maxticks = 8, interval_multiples=True)
binietoglou@0 583
binietoglou@0 584
binietoglou@0 585 #ax1.axes.xaxis.set_major_formatter(dayFormatter)
binietoglou@0 586 #ax1.axes.xaxis.set_major_locator(daylocator)
binietoglou@0 587 ax1.axes.xaxis.set_major_formatter(hourFormatter)
binietoglou@0 588 ax1.axes.xaxis.set_major_locator(hourlocator)
binietoglou@0 589
binietoglou@0 590
binietoglou@0 591 ts1 = mpl.dates.date2num(self.start_time)
binietoglou@0 592 ts2 = mpl.dates.date2num(self.stop_time)
binietoglou@0 593
binietoglou@0 594
binietoglou@0 595 im1 = ax1.imshow(data.transpose()[zoom[0]:hmax_idx,zoom[2]:zoom[3]],
binietoglou@0 596 aspect = 'auto',
binietoglou@0 597 origin = 'lower',
binietoglou@0 598 cmap = cmap,
binietoglou@17 599 vmin = vmin,
binietoglou@17 600 #vmin = data[:,10:400].max() * 0.1,
binietoglou@17 601 vmax = vmax,
binietoglou@17 602 #vmax = data[:,10:400].max() * 0.9,
binietoglou@17 603 extent = [ts1,ts2,self.z[zoom[0]]/1000.0 + z0/1000., self.z[hmax_idx]/1000.0 + z0/1000.],
binietoglou@0 604 )
ulalume3@27 605
ulalume3@27 606 if add_colorbar:
ulalume3@27 607 if cb_format:
ulalume3@27 608 cb1 = plt.colorbar(im1, format = cb_format)
ulalume3@27 609 else:
ulalume3@27 610 cb1 = plt.colorbar(im1)
ulalume3@27 611 cb1.ax.set_ylabel(cmap_label)
ulalume3@27 612
ulalume3@27 613 # Make the ticks of the colorbar smaller, two points smaller than the default font size
ulalume3@27 614 cb_font_size = mpl.rcParams['font.size'] - 2
ulalume3@27 615 for ticklabels in cb1.ax.get_yticklabels():
ulalume3@27 616 ticklabels.set_fontsize(cb_font_size)
ulalume3@27 617 cb1.ax.yaxis.get_offset_text().set_fontsize(cb_font_size)
ulalume3@27 618
ulalume3@27 619
ulalume3@27 620 def draw_plot_new(self, ax1, cmap = plt.cm.jet, signal_type = 'rc',
ulalume3@27 621 zoom = [0, 12000, 0, None], z0 = None,
ulalume3@27 622 add_colorbar = True, cmap_label = 'a.u.',
ulalume3@27 623 cb_format = None, power_limits = (-2, 2),
ulalume3@27 624 date_labels = False,
ulalume3@27 625 vmin = 0, vmax = 1.3 * 10 ** 7):
binietoglou@0 626
ulalume3@27 627 if signal_type == 'rc':
ulalume3@27 628 if len(self.rc) == 0:
ulalume3@27 629 self.calculate_rc()
ulalume3@27 630 data = self.rc
ulalume3@24 631 else:
ulalume3@27 632 data = self.matrix
ulalume3@27 633
ulalume3@27 634 hmax_idx = self.index_at_height(zoom[1])
ulalume3@27 635 hmin_idx = self.index_at_height(zoom[0])
ulalume3@27 636
ulalume3@27 637 # If z0 is given, then the plot is a.s.l.
ulalume3@27 638 if z0:
ulalume3@27 639 ax1.set_ylabel('Altitude a.s.l. [km]')
ulalume3@27 640 else:
ulalume3@27 641 ax1.set_ylabel('Altitude a.g.l. [km]')
ulalume3@27 642 z0 = 0
ulalume3@27 643
ulalume3@27 644 ax1.set_xlabel('Time UTC [hh:mm]')
ulalume3@27 645 #y axis in km, xaxis /2 to make 30s measurements in minutes. Only for 1064
ulalume3@27 646 #dateFormatter = mpl.dates.DateFormatter('%H.%M')
ulalume3@27 647 #hourlocator = mpl.dates.HourLocator()
ulalume3@27 648
binietoglou@19 649
ulalume3@27 650 if date_labels:
ulalume3@27 651 dayFormatter = mpl.dates.DateFormatter('%H:%M\n%d/%m/%Y')
ulalume3@27 652 daylocator = mpl.dates.AutoDateLocator(minticks = 3, maxticks = 8, interval_multiples=True)
ulalume3@27 653 ax1.axes.xaxis.set_major_formatter(dayFormatter)
ulalume3@27 654 ax1.axes.xaxis.set_major_locator(daylocator)
ulalume3@27 655 else:
ulalume3@27 656 hourFormatter = mpl.dates.DateFormatter('%H:%M')
ulalume3@27 657 hourlocator = mpl.dates.AutoDateLocator(minticks = 3, maxticks = 8, interval_multiples=True)
ulalume3@27 658 ax1.axes.xaxis.set_major_formatter(hourFormatter)
ulalume3@27 659 ax1.axes.xaxis.set_major_locator(hourlocator)
binietoglou@0 660
ulalume3@27 661
ulalume3@27 662 # Get the values of the time axis
ulalume3@27 663 dt = datetime.timedelta(seconds = self.duration)
ulalume3@27 664
ulalume3@27 665 time_cut = self.time[zoom[2]:zoom[3]]
ulalume3@27 666 time_last = time_cut[-1] + dt # The last element needed for pcolormesh
ulalume3@27 667
ulalume3@27 668 time_all = time_cut + (time_last,)
ulalume3@27 669
ulalume3@27 670 t_axis = mpl.dates.date2num(time_all)
ulalume3@27 671
ulalume3@27 672 # Get the values of the z axis
ulalume3@27 673 z_cut = self.z[hmin_idx:hmax_idx] - self.resolution / 2.
ulalume3@27 674 z_last = z_cut[-1] + self.resolution
ulalume3@27 675
ulalume3@27 676 z_axis = np.append(z_cut, z_last) / 1000. + z0 / 1000. # Convert to km
ulalume3@27 677
ulalume3@27 678 # Plot
ulalume3@27 679 im1 = ax1.pcolormesh(t_axis, z_axis, data.T[hmin_idx:hmax_idx, zoom[2]:zoom[3]],
ulalume3@27 680 cmap = cmap,
ulalume3@27 681 vmin = vmin,
ulalume3@27 682 vmax = vmax,
ulalume3@27 683 )
ulalume3@27 684
ulalume3@27 685 if add_colorbar:
ulalume3@27 686 if cb_format:
ulalume3@27 687 cb1 = plt.colorbar(im1, format = cb_format)
ulalume3@27 688 else:
ulalume3@27 689 cb_formatter = ScalarFormatter()
ulalume3@27 690 cb_formatter.set_powerlimits(power_limits)
ulalume3@27 691 cb1 = plt.colorbar(im1, format = cb_formatter)
ulalume3@27 692 cb1.ax.set_ylabel(cmap_label)
ulalume3@27 693
ulalume3@27 694 # Make the ticks of the colorbar smaller, two points smaller than the default font size
ulalume3@27 695 cb_font_size = mpl.rcParams['font.size'] - 2
ulalume3@27 696 for ticklabels in cb1.ax.get_yticklabels():
ulalume3@27 697 ticklabels.set_fontsize(cb_font_size)
ulalume3@27 698 cb1.ax.yaxis.get_offset_text().set_fontsize(cb_font_size)
ulalume3@27 699
binietoglou@0 700 def index_at_height(self, height):
binietoglou@0 701 idx = np.array(np.abs(self.z - height).argmin())
ulalume3@24 702 if idx.size > 1:
binietoglou@0 703 idx =idx[0]
binietoglou@0 704 return idx
binietoglou@0 705
binietoglou@0 706 def netcdf_from_files(LidarClass, filename, files, channels, measurement_ID):
binietoglou@0 707 #Read the lidar files and select channels
binietoglou@0 708 temp_m = LidarClass(files)
binietoglou@0 709 m = temp_m.subset_by_channels(channels)
binietoglou@0 710 m.get_PT()
binietoglou@0 711 m.info['Measurement_ID'] = measurement_ID
binietoglou@0 712 m.save_as_netcdf(filename)
binietoglou@0 713

mercurial