generic.py

Mon, 22 Feb 2016 16:44:55 +0200

author
Ioannis <devnull@localhost>
date
Mon, 22 Feb 2016 16:44:55 +0200
changeset 34
fbd77d0f5076
parent 33
2984158468e6
permissions
-rwxr-xr-x

Adding new files.

binietoglou@0 1 # General imports
binietoglou@0 2 import datetime
binietoglou@0 3 from operator import itemgetter
binietoglou@0 4
binietoglou@0 5 # Science imports
binietoglou@0 6 import numpy as np
binietoglou@0 7 import matplotlib as mpl
ulalume3@27 8 from matplotlib.ticker import ScalarFormatter
binietoglou@0 9 from matplotlib import pyplot as plt
binietoglou@0 10 import netCDF4 as netcdf
binietoglou@0 11
binietoglou@1 12 netcdf_format = 'NETCDF3_CLASSIC' # choose one of 'NETCDF3_CLASSIC', 'NETCDF3_64BIT', 'NETCDF4_CLASSIC' and 'NETCDF4'
binietoglou@0 13
binietoglou@0 14
binietoglou@0 15 class BaseLidarMeasurement():
binietoglou@0 16 """ This is the general measurement object.
binietoglou@0 17 It is meant to become a general measurement object
binietoglou@0 18 independent of the input files.
binietoglou@0 19
binietoglou@0 20 Each subclass should implement the following:
binietoglou@0 21 * the import_file method.
binietoglou@0 22 * set the "extra_netcdf_parameters" variable to a dictionary that includes the appropriate parameters.
binietoglou@0 23
binietoglou@0 24 You can override the get_PT method to define a custom procedure to get ground temperature and pressure.
binietoglou@0 25 The one implemented by default is by using the MILOS meteorological station data.
binietoglou@0 26
binietoglou@0 27 """
binietoglou@0 28
binietoglou@33 29 def __init__(self, filelist = None):
binietoglou@0 30 self.info = {}
binietoglou@0 31 self.dimensions = {}
binietoglou@0 32 self.variables = {}
binietoglou@0 33 self.channels = {}
binietoglou@0 34 self.attributes = {}
binietoglou@0 35 self.files = []
binietoglou@0 36 self.dark_measurement = None
binietoglou@33 37
binietoglou@0 38 if filelist:
binietoglou@0 39 self.import_files(filelist)
binietoglou@0 40
binietoglou@33 41 def import_files(self, filelist):
binietoglou@0 42 for f in filelist:
binietoglou@0 43 self.import_file(f)
binietoglou@0 44 self.update()
binietoglou@0 45
binietoglou@0 46 def import_file(self,filename):
binietoglou@0 47 raise NotImplementedError('Importing files should be defined in the instrument-specific subclass.')
binietoglou@0 48
binietoglou@0 49 def update(self):
binietoglou@0 50 '''
binietoglou@0 51 Update the the info, variables and dimensions of the lidar measurement based
binietoglou@0 52 on the information found in the channels.
binietoglou@0 53
binietoglou@0 54 Reading of the scan_angles parameter is not implemented.
binietoglou@0 55 '''
binietoglou@0 56
binietoglou@0 57 # Initialize
binietoglou@0 58 start_time =[]
binietoglou@0 59 stop_time = []
binietoglou@0 60 points = []
binietoglou@0 61 all_time_scales = []
binietoglou@0 62 channel_name_list = []
binietoglou@0 63
binietoglou@0 64 # Get the information from all the channels
binietoglou@0 65 for channel_name, channel in self.channels.items():
binietoglou@0 66 channel.update()
binietoglou@0 67 start_time.append(channel.start_time)
binietoglou@0 68 stop_time.append(channel.stop_time)
binietoglou@0 69 points.append(channel.points)
binietoglou@0 70 all_time_scales.append(channel.time)
binietoglou@0 71 channel_name_list.append(channel_name)
binietoglou@0 72
binietoglou@0 73 # Find the unique time scales used in the channels
binietoglou@0 74 time_scales = set(all_time_scales)
binietoglou@0 75
binietoglou@0 76 # Update the info dictionary
binietoglou@0 77 self.info['start_time'] = min(start_time)
binietoglou@0 78 self.info['stop_time'] = max(stop_time)
binietoglou@0 79 self.info['duration'] = self.info['stop_time'] - self.info['start_time']
binietoglou@0 80
binietoglou@0 81 # Update the dimensions dictionary
binietoglou@0 82 self.dimensions['points'] = max(points)
binietoglou@0 83 self.dimensions['channels'] = len(self.channels)
binietoglou@0 84 # self.dimensions['scan angles'] = 1
binietoglou@0 85 self.dimensions['nb_of_time_scales'] = len(time_scales)
binietoglou@0 86
binietoglou@0 87 # Update the variables dictionary
binietoglou@0 88 # Write time scales in seconds
binietoglou@0 89 raw_Data_Start_Time = []
binietoglou@0 90 raw_Data_Stop_Time = []
binietoglou@0 91
binietoglou@0 92 for current_time_scale in list(time_scales):
binietoglou@0 93 raw_start_time = np.array(current_time_scale) - min(start_time) # Time since start_time
binietoglou@0 94 raw_start_in_seconds = np.array([t.seconds for t in raw_start_time]) # Convert in seconds
binietoglou@0 95 raw_Data_Start_Time.append(raw_start_in_seconds) # And add to the list
binietoglou@0 96 # Check if this time scale has measurements every 30 or 60 seconds.
binietoglou@0 97
binietoglou@0 98 duration = self._get_duration(raw_start_in_seconds)
binietoglou@0 99
binietoglou@0 100 raw_stop_in_seconds = raw_start_in_seconds + duration
binietoglou@0 101 raw_Data_Stop_Time.append(raw_stop_in_seconds)
binietoglou@0 102
ioannis@22 103 self.variables['Raw_Data_Start_Time'] = raw_Data_Start_Time
ioannis@22 104 self.variables['Raw_Data_Stop_Time'] = raw_Data_Stop_Time
binietoglou@0 105
binietoglou@0 106 # Make a dictionary to match time scales and channels
binietoglou@0 107 channel_timescales = []
binietoglou@0 108 for (channel_name, current_time_scale) in zip(channel_name_list, all_time_scales):
binietoglou@0 109 # The following lines are PEARL specific. The reason they are here is not clear.
binietoglou@0 110 # if channel_name =='1064BLR':
binietoglou@0 111 # channel_name = '1064'
binietoglou@0 112 for (ts,n) in zip(time_scales, range(len(time_scales))):
binietoglou@0 113 if current_time_scale == ts:
binietoglou@0 114 channel_timescales.append([channel_name,n])
binietoglou@0 115 self.variables['id_timescale'] = dict(channel_timescales)
binietoglou@0 116
binietoglou@0 117 def _get_duration(self, raw_start_in_seconds):
binietoglou@33 118 ''' Return the duration for a given time scale. In some files (e.g. Licel) this
binietoglou@0 119 can be specified from the files themselves. In others this must be guessed.
binietoglou@0 120
binietoglou@0 121 '''
binietoglou@0 122 # The old method, kept here for reference
binietoglou@0 123 #dt = np.mean(np.diff(raw_start_in_seconds))
binietoglou@0 124 #for d in duration_list:
binietoglou@0 125 # if abs(dt - d) <15: #If the difference of measuremetns is 10s near the(30 or 60) seconds
binietoglou@0 126 # duration = d
binietoglou@0 127
binietoglou@0 128 duration = np.diff(raw_start_in_seconds)[0]
binietoglou@0 129
binietoglou@0 130 return duration
binietoglou@0 131
binietoglou@0 132 def subset_by_channels(self, channel_subset):
binietoglou@0 133 ''' Get a measurement object containing only the channels with names
binietoglou@0 134 contained in the channel_sublet list '''
binietoglou@0 135
binietoglou@0 136 m = self.__class__() # Create an object of the same type as this one.
binietoglou@0 137 m.channels = dict([(channel, self.channels[channel]) for channel
binietoglou@0 138 in channel_subset])
binietoglou@0 139 m.update()
binietoglou@0 140 return m
binietoglou@0 141
binietoglou@0 142 def subset_by_time(self, start_time, stop_time):
binietoglou@0 143
binietoglou@0 144 if start_time > stop_time:
binietoglou@0 145 raise ValueError('Stop time should be after start time')
binietoglou@0 146
binietoglou@0 147 if (start_time < self.info['start_time']) or (stop_time > self.info['stop_time']):
binietoglou@0 148 raise ValueError('The time interval specified is not part of the measurement')
binietoglou@0 149
binietoglou@0 150 m = self.__class__() # Create an object of the same type as this one.
binietoglou@0 151 for (channel_name, channel) in self.channels.items():
binietoglou@0 152 m.channels[channel_name] = channel.subset_by_time(start_time, stop_time)
binietoglou@0 153 m.update()
binietoglou@0 154 return m
binietoglou@0 155
ulalume3@27 156 def subset_by_bins(self, b_min = 0, b_max = None):
ulalume3@27 157 """Remove some height bins from the file. This could be needed to
ulalume3@27 158 remove aquisition artifacts at the start or the end of the files.
ulalume3@27 159 """
ulalume3@27 160
ulalume3@27 161 m = self.__class__() # Create an object of the same type as this one.
ulalume3@27 162
ulalume3@27 163 for (channel_name, channel) in self.channels.items():
ulalume3@27 164 m.channels[channel_name] = channel.subset_by_bins(b_min, b_max)
ulalume3@27 165
ulalume3@27 166 m.update()
ulalume3@27 167
ulalume3@27 168 return m
ulalume3@27 169
binietoglou@0 170 def r_plot(self):
binietoglou@0 171 #Make a basic plot of the data.
binietoglou@0 172 #Should include some dictionary with params to make plot stable.
binietoglou@0 173 pass
binietoglou@0 174
binietoglou@0 175 def r_pdf(self):
binietoglou@0 176 # Create a pdf report using a basic plot and meta-data.
binietoglou@0 177 pass
binietoglou@0 178
binietoglou@0 179 def save(self):
binietoglou@0 180 #Save the current state of the object to continue the analysis later.
binietoglou@0 181 pass
binietoglou@0 182
binietoglou@0 183 def get_PT(self):
binietoglou@0 184 ''' Sets the pressure and temperature at station level .
binietoglou@0 185 The results are stored in the info dictionary.
binietoglou@0 186 '''
binietoglou@0 187
binietoglou@0 188 self.info['Temperature'] = 10.0
binietoglou@0 189 self.info['Pressure'] = 930.0
binietoglou@0 190
binietoglou@15 191 def subtract_dark(self):
binietoglou@15 192
binietoglou@15 193 if not self.dark_measurement:
binietoglou@15 194 raise IOError('No dark measurements have been imported yet.')
binietoglou@15 195
binietoglou@15 196 for (channel_name, dark_channel) in self.dark_measurement.channels.iteritems():
binietoglou@15 197 dark_profile = dark_channel.average_profile()
binietoglou@15 198 channel = self.channels[channel_name]
binietoglou@15 199
binietoglou@15 200 for measurement_time, data in channel.data.iteritems():
binietoglou@15 201 channel.data[measurement_time] = data - dark_profile
binietoglou@15 202
binietoglou@15 203 channel.update()
binietoglou@15 204
ulalume3@23 205 def save_as_netcdf(self, filename = None):
binietoglou@0 206 """Saves the measurement in the netcdf format as required by the SCC.
ulalume3@23 207 Input: filename. If no filename is provided <measurement_id>.nc will
ulalume3@23 208 be used.
binietoglou@0 209 """
binietoglou@0 210 params = self.extra_netcdf_parameters
binietoglou@0 211 needed_parameters = ['Measurement_ID', 'Temperature', 'Pressure']
binietoglou@0 212
binietoglou@0 213 for parameter in needed_parameters:
binietoglou@0 214 stored_value = self.info.get(parameter, None)
binietoglou@0 215 if stored_value is None:
binietoglou@0 216 raise ValueError('A value needs to be specified for %s' % parameter)
binietoglou@0 217
ulalume3@23 218 if not filename:
ulalume3@23 219 filename = "%s.nc" % self.info['Measurement_ID']
binietoglou@0 220
binietoglou@0 221 dimensions = {'points': 1,
binietoglou@0 222 'channels': 1,
binietoglou@0 223 'time': None,
binietoglou@0 224 'nb_of_time_scales': 1,
binietoglou@0 225 'scan_angles': 1,} # Mandatory dimensions. Time bck not implemented
binietoglou@0 226
binietoglou@0 227 global_att = {'Measurement_ID': None,
binietoglou@0 228 'RawData_Start_Date': None,
binietoglou@0 229 'RawData_Start_Time_UT': None,
binietoglou@0 230 'RawData_Stop_Time_UT': None,
binietoglou@0 231 'RawBck_Start_Date': None,
binietoglou@0 232 'RawBck_Start_Time_UT': None,
binietoglou@0 233 'RawBck_Stop_Time_UT': None,
binietoglou@0 234 'Sounding_File_Name': None,
binietoglou@0 235 'LR_File_Name': None,
binietoglou@0 236 'Overlap_File_Name': None,
binietoglou@0 237 'Location': None,
binietoglou@0 238 'System': None,
binietoglou@0 239 'Latitude_degrees_north': None,
binietoglou@0 240 'Longitude_degrees_east': None,
binietoglou@0 241 'Altitude_meter_asl': None}
binietoglou@0 242
binietoglou@0 243 channel_variables = \
binietoglou@0 244 {'channel_ID': (('channels', ), 'i'),
binietoglou@0 245 'Background_Low': (('channels', ), 'd'),
binietoglou@0 246 'Background_High': (('channels', ), 'd'),
binietoglou@0 247 'LR_Input': (('channels', ), 'i'),
binietoglou@0 248 'DAQ_Range': (('channels', ), 'd'),
binietoglou@0 249 'Depolarization_Factor': (('channels', ), 'd'), }
binietoglou@0 250
binietoglou@0 251
binietoglou@0 252 channels = self.channels.keys()
binietoglou@0 253
binietoglou@0 254 input_values = dict(self.dimensions, **self.variables)
binietoglou@0 255
binietoglou@0 256 # Add some mandatory global attributes
binietoglou@0 257 input_values['Measurement_ID'] = self.info['Measurement_ID']
ioannis@21 258 input_values['RawData_Start_Date'] = self.info['start_time'].strftime('%Y%m%d')
ioannis@21 259 input_values['RawData_Start_Time_UT'] = self.info['start_time'].strftime('%H%M%S')
ioannis@21 260 input_values['RawData_Stop_Time_UT'] = self.info['stop_time'].strftime('%H%M%S')
binietoglou@0 261
binietoglou@0 262 # Add some optional global attributes
binietoglou@0 263 input_values['System'] = params.general_parameters['System']
binietoglou@0 264 input_values['Latitude_degrees_north'] = params.general_parameters['Latitude_degrees_north']
binietoglou@0 265 input_values['Longitude_degrees_east'] = params.general_parameters['Longitude_degrees_east']
binietoglou@0 266 input_values['Altitude_meter_asl'] = params.general_parameters['Altitude_meter_asl']
binietoglou@0 267
binietoglou@0 268 # Open a netCDF4 file
binietoglou@0 269 f = netcdf.Dataset(filename,'w', format = netcdf_format) # the format is specified in the begining of the file
binietoglou@0 270
binietoglou@0 271 # Create the dimensions in the file
binietoglou@0 272 for (d,v) in dimensions.iteritems():
binietoglou@0 273 v = input_values.pop(d, v)
binietoglou@0 274 f.createDimension(d,v)
binietoglou@0 275
binietoglou@0 276 # Create global attributes
binietoglou@0 277 for (attrib,value) in global_att.iteritems():
binietoglou@0 278 val = input_values.pop(attrib,value)
binietoglou@0 279 if val:
ioannis@21 280 setattr(f, attrib, val)
binietoglou@0 281
binietoglou@0 282 """ Variables """
binietoglou@0 283 # Write the values of fixes channel parameters
binietoglou@0 284 for (var,t) in channel_variables.iteritems():
binietoglou@0 285 temp_v = f.createVariable(var,t[1],t[0])
binietoglou@0 286 for (channel, n) in zip(channels, range(len(channels))):
binietoglou@0 287 temp_v[n] = params.channel_parameters[channel][var]
binietoglou@0 288
binietoglou@0 289 # Write the id_timescale values
binietoglou@0 290 temp_id_timescale = f.createVariable('id_timescale','i',('channels',))
binietoglou@0 291 for (channel, n) in zip(channels, range(len(channels))):
binietoglou@0 292 temp_id_timescale[n] = self.variables['id_timescale'][channel]
binietoglou@0 293
binietoglou@0 294 # Laser pointing angle
binietoglou@0 295 temp_v = f.createVariable('Laser_Pointing_Angle','d',('scan_angles',))
binietoglou@0 296 temp_v[:] = params.general_parameters['Laser_Pointing_Angle']
binietoglou@0 297
binietoglou@0 298 # Molecular calculation
binietoglou@0 299 temp_v = f.createVariable('Molecular_Calc','i')
binietoglou@0 300 temp_v[:] = params.general_parameters['Molecular_Calc']
binietoglou@0 301
binietoglou@0 302 # Laser pointing angles of profiles
binietoglou@0 303 temp_v = f.createVariable('Laser_Pointing_Angle_of_Profiles','i',('time','nb_of_time_scales'))
binietoglou@0 304 for (time_scale,n) in zip(self.variables['Raw_Data_Start_Time'],
binietoglou@0 305 range(len(self.variables['Raw_Data_Start_Time']))):
binietoglou@0 306 temp_v[:len(time_scale), n] = 0 # The lidar has only one laser pointing angle
binietoglou@0 307
binietoglou@0 308 # Raw data start/stop time
binietoglou@0 309 temp_raw_start = f.createVariable('Raw_Data_Start_Time','i',('time','nb_of_time_scales'))
binietoglou@0 310 temp_raw_stop = f.createVariable('Raw_Data_Stop_Time','i',('time','nb_of_time_scales'))
binietoglou@0 311 for (start_time, stop_time,n) in zip(self.variables['Raw_Data_Start_Time'],
binietoglou@0 312 self.variables['Raw_Data_Stop_Time'],
binietoglou@0 313 range(len(self.variables['Raw_Data_Start_Time']))):
binietoglou@0 314 temp_raw_start[:len(start_time),n] = start_time
binietoglou@0 315 temp_raw_stop[:len(stop_time),n] = stop_time
binietoglou@0 316
binietoglou@0 317 #Laser shots
binietoglou@0 318 temp_v = f.createVariable('Laser_Shots','i',('time','channels'))
binietoglou@0 319 for (channel,n) in zip(channels, range(len(channels))):
binietoglou@0 320 time_length = len(self.variables['Raw_Data_Start_Time'][self.variables['id_timescale'][channel]])
ulalume3@24 321 # Array slicing stoped working as usual ex. temp_v[:10] = 100 does not work. ??? np.ones was added.
ulalume3@24 322 temp_v[:time_length, n] = np.ones(time_length) * params.channel_parameters[channel]['Laser_Shots']
ulalume3@24 323
binietoglou@0 324 #Raw lidar data
binietoglou@0 325 temp_v = f.createVariable('Raw_Lidar_Data','d',('time', 'channels','points'))
binietoglou@0 326 for (channel,n) in zip(channels, range(len(channels))):
binietoglou@0 327 c = self.channels[channel]
binietoglou@0 328 temp_v[:len(c.time),n, :c.points] = c.matrix
binietoglou@0 329
binietoglou@0 330 self.add_dark_measurements_to_netcdf(f, channels)
binietoglou@0 331
binietoglou@0 332 #Pressure at lidar station
binietoglou@0 333 temp_v = f.createVariable('Pressure_at_Lidar_Station','d')
binietoglou@0 334 temp_v[:] = self.info['Pressure']
binietoglou@0 335
binietoglou@0 336 #Temperature at lidar station
binietoglou@0 337 temp_v = f.createVariable('Temperature_at_Lidar_Station','d')
binietoglou@0 338 temp_v[:] = self.info['Temperature']
binietoglou@0 339
binietoglou@0 340 self.save_netcdf_extra(f)
binietoglou@0 341 f.close()
binietoglou@0 342
binietoglou@0 343 def add_dark_measurements_to_netcdf(self, f, channels):
binietoglou@0 344
binietoglou@0 345 # Get dark measurements. If it is not given in self.dark_measurement
binietoglou@0 346 # try to get it using the get_dark_measurements method. If none is found
binietoglou@0 347 # return without adding something.
binietoglou@0 348 if self.dark_measurement is None:
binietoglou@0 349 self.dark_measurement = self.get_dark_measurements()
binietoglou@0 350
binietoglou@0 351 if self.dark_measurement is None:
binietoglou@0 352 return
binietoglou@0 353
binietoglou@0 354 dark_measurement = self.dark_measurement
binietoglou@0 355
binietoglou@0 356 # Calculate the length of the time_bck dimensions
binietoglou@0 357 number_of_profiles = [len(c.time) for c in dark_measurement.channels.values()]
binietoglou@0 358 max_number_of_profiles = np.max(number_of_profiles)
binietoglou@0 359
binietoglou@0 360 # Create the dimension
binietoglou@0 361 f.createDimension('time_bck', max_number_of_profiles)
binietoglou@0 362
binietoglou@0 363 # Save the dark measurement data
binietoglou@0 364 temp_v = f.createVariable('Background_Profile','d',('time_bck', 'channels', 'points'))
binietoglou@0 365 for (channel,n) in zip(channels, range(len(channels))):
binietoglou@0 366 c = dark_measurement.channels[channel]
binietoglou@0 367 temp_v[:len(c.time),n, :c.points] = c.matrix
binietoglou@0 368
binietoglou@0 369 # Dark profile start/stop time
binietoglou@14 370 temp_raw_start = f.createVariable('Raw_Bck_Start_Time','i',('time_bck','nb_of_time_scales'))
binietoglou@14 371 temp_raw_stop = f.createVariable('Raw_Bck_Stop_Time','i',('time_bck','nb_of_time_scales'))
binietoglou@0 372 for (start_time, stop_time,n) in zip(dark_measurement.variables['Raw_Data_Start_Time'],
binietoglou@0 373 dark_measurement.variables['Raw_Data_Stop_Time'],
binietoglou@0 374 range(len(dark_measurement.variables['Raw_Data_Start_Time']))):
binietoglou@0 375 temp_raw_start[:len(start_time),n] = start_time
binietoglou@0 376 temp_raw_stop[:len(stop_time),n] = stop_time
binietoglou@0 377
binietoglou@0 378 # Dark measurement start/stop time
binietoglou@0 379 f.RawBck_Start_Date = dark_measurement.info['start_time'].strftime('%Y%m%d')
binietoglou@0 380 f.RawBck_Start_Time_UT = dark_measurement.info['start_time'].strftime('%H%M%S')
binietoglou@0 381 f.RawBck_Stop_Time_UT = dark_measurement.info['stop_time'].strftime('%H%M%S')
binietoglou@0 382
binietoglou@0 383
binietoglou@0 384
binietoglou@0 385 def save_netcdf_extra(self, f):
binietoglou@0 386 pass
binietoglou@0 387
binietoglou@0 388 def _gettime(self, date_str, time_str):
binietoglou@0 389 t = datetime.datetime.strptime(date_str+time_str,'%d/%m/%Y%H.%M.%S')
binietoglou@0 390 return t
binietoglou@0 391
binietoglou@0 392 def plot(self):
binietoglou@0 393 for channel in self.channels:
binietoglou@0 394 self.channels[channel].plot(show_plot = False)
binietoglou@0 395 plt.show()
binietoglou@0 396
binietoglou@0 397 def get_dark_measurements(self):
binietoglou@0 398 return None
ioannis@22 399
ioannis@22 400 @property
ioannis@22 401 def mean_time(self):
ioannis@22 402 start_time = self.info['start_time']
ioannis@22 403 stop_time = self.info['stop_time']
ioannis@22 404 dt = stop_time - start_time
ioannis@22 405 t_mean = start_time + dt / 2
ioannis@22 406 return t_mean
binietoglou@0 407
binietoglou@0 408
ulalume3@27 409 class LidarChannel:
binietoglou@0 410
ulalume3@27 411 def __init__(self, channel_parameters):
ulalume3@27 412 c = 299792458 # Speed of light
binietoglou@0 413 self.wavelength = channel_parameters['name']
binietoglou@0 414 self.name = str(self.wavelength)
ulalume3@27 415 self.binwidth = float(channel_parameters['binwidth']) # in microseconds
binietoglou@0 416 self.data = {}
binietoglou@0 417 self.resolution = self.binwidth * c / 2
binietoglou@33 418 self.z = np.arange(len(channel_parameters['data'])) * self.resolution + self.resolution / 2.0 # Change: add half bin in the z
binietoglou@0 419 self.points = len(channel_parameters['data'])
binietoglou@0 420 self.rc = []
binietoglou@0 421 self.duration = 60
binietoglou@0 422
ioannis@32 423 def calculate_rc(self, idx_min = 4000, idx_max = 5000):
ioannis@32 424 background = np.mean(self.matrix[:,idx_min:idx_max], axis = 1) # Calculate the background from 30000m and above
binietoglou@0 425 self.rc = (self.matrix.transpose()- background).transpose() * (self.z **2)
binietoglou@0 426
binietoglou@0 427
binietoglou@0 428 def update(self):
binietoglou@0 429 self.start_time = min(self.data.keys())
binietoglou@0 430 self.stop_time = max(self.data.keys()) + datetime.timedelta(seconds = self.duration)
binietoglou@0 431 self.time = tuple(sorted(self.data.keys()))
binietoglou@0 432 sorted_data = sorted(self.data.iteritems(), key=itemgetter(0))
binietoglou@0 433 self.matrix = np.array(map(itemgetter(1),sorted_data))
binietoglou@0 434
binietoglou@0 435 def _nearest_dt(self,dtime):
binietoglou@0 436 margin = datetime.timedelta(seconds = 300)
binietoglou@0 437 if ((dtime + margin) < self.start_time)| ((dtime - margin) > self.stop_time):
binietoglou@0 438 print "Requested date not covered in this file"
binietoglou@0 439 raise
binietoglou@0 440 dt = abs(self.time - np.array(dtime))
binietoglou@0 441 dtmin = min(dt)
binietoglou@0 442
binietoglou@0 443 if dtmin > datetime.timedelta(seconds = 60):
binietoglou@0 444 print "Nearest profile more than 60 seconds away. dt = %s." % dtmin
binietoglou@0 445 ind_t = np.where(dt == dtmin)
binietoglou@0 446 ind_a= ind_t[0]
binietoglou@0 447 if len(ind_a) > 1:
binietoglou@0 448 ind_a = ind_a[0]
binietoglou@0 449 chosen_time = self.time[ind_a]
binietoglou@0 450 return chosen_time, ind_a
binietoglou@0 451
binietoglou@0 452 def subset_by_time(self, start_time, stop_time):
binietoglou@0 453
binietoglou@0 454 time_array = np.array(self.time)
binietoglou@0 455 condition = (time_array >= start_time) & (time_array <= stop_time)
binietoglou@0 456
binietoglou@0 457 subset_time = time_array[condition]
binietoglou@0 458 subset_data = dict([(c_time, self.data[c_time]) for c_time in subset_time])
binietoglou@0 459
binietoglou@0 460 #Create a list with the values needed by channel's __init__()
ulalume3@27 461 parameter_values = {'name': self.wavelength,
binietoglou@0 462 'binwidth': self.binwidth,
binietoglou@0 463 'data': subset_data[subset_time[0]],}
ulalume3@27 464
ulalume3@27 465 # We should use __class__ instead of class name, so that this works properly
ulalume3@27 466 # with subclasses
ulalume3@27 467 # Ex: c = self.__class__(parameters_values)
ulalume3@27 468 # This does not currently work with Licel files though
ulalume3@27 469 c = LidarChannel(parameter_values)
ulalume3@27 470 c.data = subset_data
ulalume3@27 471 c.update()
ulalume3@27 472 return c
ulalume3@27 473
ulalume3@27 474 def subset_by_bins(self, b_min = 0, b_max = None):
ulalume3@27 475 """Remove some height bins from the file. This could be needed to
ulalume3@27 476 remove aquisition artifacts at the start or the end of the files.
ulalume3@27 477 """
ulalume3@27 478
ulalume3@27 479 subset_data = {}
ulalume3@27 480
ulalume3@27 481 for ctime, cdata in self.data.items():
ulalume3@27 482 subset_data[ctime] = cdata[b_min:b_max]
ulalume3@27 483
ulalume3@27 484 #Create a list with the values needed by channel's __init__()
ulalume3@27 485 parameters_values = {'name': self.wavelength,
ulalume3@27 486 'binwidth': self.binwidth,
ulalume3@27 487 'data': subset_data[subset_data.keys()[0]],} # We just need any array with the correct length
binietoglou@0 488
ulalume3@27 489 c = LidarChannel(parameters_values)
binietoglou@0 490 c.data = subset_data
binietoglou@0 491 c.update()
binietoglou@0 492 return c
binietoglou@0 493
binietoglou@0 494 def profile(self,dtime, signal_type = 'rc'):
binietoglou@0 495 t, idx = self._nearest_dt(dtime)
binietoglou@0 496 if signal_type == 'rc':
binietoglou@0 497 data = self.rc
binietoglou@0 498 else:
binietoglou@0 499 data = self.matrix
binietoglou@0 500
binietoglou@0 501 prof = data[idx,:][0]
binietoglou@0 502 return prof, t
binietoglou@0 503
binietoglou@0 504 def get_slice(self, starttime, endtime, signal_type = 'rc'):
binietoglou@0 505 if signal_type == 'rc':
binietoglou@0 506 data = self.rc
binietoglou@0 507 else:
binietoglou@0 508 data = self.matrix
binietoglou@0 509 tim = np.array(self.time)
binietoglou@0 510 starttime = self._nearest_dt(starttime)[0]
binietoglou@0 511 endtime = self._nearest_dt(endtime)[0]
binietoglou@0 512 condition = (tim >= starttime) & (tim <= endtime)
binietoglou@0 513 sl = data[condition, :]
binietoglou@0 514 t = tim[condition]
binietoglou@0 515 return sl,t
binietoglou@0 516
binietoglou@15 517 def profile_for_duration(self, tim, duration = datetime.timedelta(seconds = 0), signal_type = 'rc'):
binietoglou@15 518 """ Calculates the profile around a specific time (tim). """
binietoglou@0 519 starttime = tim - duration/2
binietoglou@0 520 endtime = tim + duration/2
binietoglou@0 521 d,t = self.get_slice(starttime, endtime, signal_type = signal_type)
binietoglou@0 522 prof = np.mean(d, axis = 0)
binietoglou@0 523 tmin = min(t)
binietoglou@0 524 tmax = max(t)
binietoglou@0 525 tav = tmin + (tmax-tmin)/2
binietoglou@0 526 return prof,(tav, tmin,tmax)
binietoglou@15 527
binietoglou@15 528 def average_profile(self):
binietoglou@15 529 """ Return the average profile (NOT range corrected) for all the duration of the measurement. """
binietoglou@15 530 prof = np.mean(self.matrix, axis = 0)
binietoglou@15 531 return prof
binietoglou@15 532
binietoglou@17 533 def plot(self, signal_type = 'rc', filename = None, zoom = [0,12000,0,-1], show_plot = True, cmap = plt.cm.jet, z0 = None, title = None, vmin = 0, vmax = 1.3 * 10 ** 7):
binietoglou@0 534 #if filename is not None:
binietoglou@0 535 # matplotlib.use('Agg')
binietoglou@0 536
binietoglou@0 537 fig = plt.figure()
binietoglou@0 538 ax1 = fig.add_subplot(111)
binietoglou@17 539 self.draw_plot(ax1, cmap = cmap, signal_type = signal_type, zoom = zoom, z0 = z0, vmin = vmin, vmax = vmax)
binietoglou@17 540
binietoglou@17 541 if title:
binietoglou@17 542 ax1.set_title(title)
binietoglou@17 543 else:
binietoglou@17 544 ax1.set_title("%s signal - %s" % (signal_type.upper(), self.name))
binietoglou@0 545
binietoglou@0 546 if filename is not None:
binietoglou@0 547 pass
binietoglou@0 548 #plt.savefig(filename)
binietoglou@0 549 else:
binietoglou@0 550 if show_plot:
binietoglou@0 551 plt.show()
binietoglou@0 552 #plt.close() ???
binietoglou@0 553
ulalume3@27 554 def draw_plot(self,ax1, cmap = plt.cm.jet, signal_type = 'rc',
ulalume3@27 555 zoom = [0,12000,0,-1], z0 = None,
ulalume3@27 556 add_colorbar = True, cmap_label = 'a.u.', cb_format = None,
ulalume3@27 557 vmin = 0, vmax = 1.3 * 10 ** 7):
binietoglou@0 558
binietoglou@0 559 if signal_type == 'rc':
binietoglou@0 560 if len(self.rc) == 0:
binietoglou@0 561 self.calculate_rc()
binietoglou@0 562 data = self.rc
binietoglou@0 563 else:
binietoglou@0 564 data = self.matrix
binietoglou@0 565
binietoglou@0 566 hmax_idx = self.index_at_height(zoom[1])
binietoglou@0 567
binietoglou@17 568 # If z0 is given, then the plot is a.s.l.
binietoglou@17 569 if z0:
binietoglou@17 570 ax1.set_ylabel('Altitude a.s.l. [km]')
binietoglou@17 571 else:
binietoglou@17 572 ax1.set_ylabel('Altitude a.g.l. [km]')
binietoglou@17 573 z0 = 0
binietoglou@17 574
binietoglou@17 575 ax1.set_xlabel('Time UTC [hh:mm]')
binietoglou@0 576 #y axis in km, xaxis /2 to make 30s measurements in minutes. Only for 1064
binietoglou@0 577 #dateFormatter = mpl.dates.DateFormatter('%H.%M')
binietoglou@0 578 #hourlocator = mpl.dates.HourLocator()
binietoglou@0 579
binietoglou@0 580 #dayFormatter = mpl.dates.DateFormatter('\n\n%d/%m')
binietoglou@0 581 #daylocator = mpl.dates.DayLocator()
binietoglou@17 582 hourFormatter = mpl.dates.DateFormatter('%H:%M')
binietoglou@19 583 hourlocator = mpl.dates.AutoDateLocator(minticks = 3, maxticks = 8, interval_multiples=True)
binietoglou@0 584
binietoglou@0 585
binietoglou@0 586 #ax1.axes.xaxis.set_major_formatter(dayFormatter)
binietoglou@0 587 #ax1.axes.xaxis.set_major_locator(daylocator)
binietoglou@0 588 ax1.axes.xaxis.set_major_formatter(hourFormatter)
binietoglou@0 589 ax1.axes.xaxis.set_major_locator(hourlocator)
binietoglou@0 590
binietoglou@0 591
binietoglou@0 592 ts1 = mpl.dates.date2num(self.start_time)
binietoglou@0 593 ts2 = mpl.dates.date2num(self.stop_time)
binietoglou@0 594
binietoglou@0 595
binietoglou@0 596 im1 = ax1.imshow(data.transpose()[zoom[0]:hmax_idx,zoom[2]:zoom[3]],
binietoglou@0 597 aspect = 'auto',
binietoglou@0 598 origin = 'lower',
binietoglou@0 599 cmap = cmap,
binietoglou@17 600 vmin = vmin,
binietoglou@17 601 #vmin = data[:,10:400].max() * 0.1,
binietoglou@17 602 vmax = vmax,
binietoglou@17 603 #vmax = data[:,10:400].max() * 0.9,
binietoglou@17 604 extent = [ts1,ts2,self.z[zoom[0]]/1000.0 + z0/1000., self.z[hmax_idx]/1000.0 + z0/1000.],
binietoglou@0 605 )
ulalume3@27 606
ulalume3@27 607 if add_colorbar:
ulalume3@27 608 if cb_format:
ulalume3@27 609 cb1 = plt.colorbar(im1, format = cb_format)
ulalume3@27 610 else:
ulalume3@27 611 cb1 = plt.colorbar(im1)
ulalume3@27 612 cb1.ax.set_ylabel(cmap_label)
ulalume3@27 613
ulalume3@27 614 # Make the ticks of the colorbar smaller, two points smaller than the default font size
ulalume3@27 615 cb_font_size = mpl.rcParams['font.size'] - 2
ulalume3@27 616 for ticklabels in cb1.ax.get_yticklabels():
ulalume3@27 617 ticklabels.set_fontsize(cb_font_size)
ulalume3@27 618 cb1.ax.yaxis.get_offset_text().set_fontsize(cb_font_size)
ulalume3@27 619
ulalume3@27 620
ulalume3@27 621 def draw_plot_new(self, ax1, cmap = plt.cm.jet, signal_type = 'rc',
ulalume3@27 622 zoom = [0, 12000, 0, None], z0 = None,
ulalume3@27 623 add_colorbar = True, cmap_label = 'a.u.',
ulalume3@27 624 cb_format = None, power_limits = (-2, 2),
ulalume3@27 625 date_labels = False,
ulalume3@27 626 vmin = 0, vmax = 1.3 * 10 ** 7):
binietoglou@0 627
ulalume3@27 628 if signal_type == 'rc':
ulalume3@27 629 if len(self.rc) == 0:
ulalume3@27 630 self.calculate_rc()
ulalume3@27 631 data = self.rc
ulalume3@24 632 else:
ulalume3@27 633 data = self.matrix
ulalume3@27 634
ulalume3@27 635 hmax_idx = self.index_at_height(zoom[1])
ulalume3@27 636 hmin_idx = self.index_at_height(zoom[0])
ulalume3@27 637
ulalume3@27 638 # If z0 is given, then the plot is a.s.l.
ulalume3@27 639 if z0:
ulalume3@27 640 ax1.set_ylabel('Altitude a.s.l. [km]')
ulalume3@27 641 else:
ulalume3@27 642 ax1.set_ylabel('Altitude a.g.l. [km]')
ulalume3@27 643 z0 = 0
ulalume3@27 644
ulalume3@27 645 ax1.set_xlabel('Time UTC [hh:mm]')
ulalume3@27 646 #y axis in km, xaxis /2 to make 30s measurements in minutes. Only for 1064
ulalume3@27 647 #dateFormatter = mpl.dates.DateFormatter('%H.%M')
ulalume3@27 648 #hourlocator = mpl.dates.HourLocator()
ulalume3@27 649
binietoglou@19 650
ulalume3@27 651 if date_labels:
ulalume3@27 652 dayFormatter = mpl.dates.DateFormatter('%H:%M\n%d/%m/%Y')
ulalume3@27 653 daylocator = mpl.dates.AutoDateLocator(minticks = 3, maxticks = 8, interval_multiples=True)
ulalume3@27 654 ax1.axes.xaxis.set_major_formatter(dayFormatter)
ulalume3@27 655 ax1.axes.xaxis.set_major_locator(daylocator)
ulalume3@27 656 else:
ulalume3@27 657 hourFormatter = mpl.dates.DateFormatter('%H:%M')
ulalume3@27 658 hourlocator = mpl.dates.AutoDateLocator(minticks = 3, maxticks = 8, interval_multiples=True)
ulalume3@27 659 ax1.axes.xaxis.set_major_formatter(hourFormatter)
ulalume3@27 660 ax1.axes.xaxis.set_major_locator(hourlocator)
binietoglou@0 661
ulalume3@27 662
ulalume3@27 663 # Get the values of the time axis
ulalume3@27 664 dt = datetime.timedelta(seconds = self.duration)
ulalume3@27 665
ulalume3@27 666 time_cut = self.time[zoom[2]:zoom[3]]
ulalume3@27 667 time_last = time_cut[-1] + dt # The last element needed for pcolormesh
ulalume3@27 668
ulalume3@27 669 time_all = time_cut + (time_last,)
ulalume3@27 670
ulalume3@27 671 t_axis = mpl.dates.date2num(time_all)
ulalume3@27 672
ulalume3@27 673 # Get the values of the z axis
ulalume3@27 674 z_cut = self.z[hmin_idx:hmax_idx] - self.resolution / 2.
ulalume3@27 675 z_last = z_cut[-1] + self.resolution
ulalume3@27 676
ulalume3@27 677 z_axis = np.append(z_cut, z_last) / 1000. + z0 / 1000. # Convert to km
ulalume3@27 678
ulalume3@27 679 # Plot
ulalume3@27 680 im1 = ax1.pcolormesh(t_axis, z_axis, data.T[hmin_idx:hmax_idx, zoom[2]:zoom[3]],
ulalume3@27 681 cmap = cmap,
ulalume3@27 682 vmin = vmin,
ulalume3@27 683 vmax = vmax,
ulalume3@27 684 )
ulalume3@27 685
ulalume3@27 686 if add_colorbar:
ulalume3@27 687 if cb_format:
ulalume3@27 688 cb1 = plt.colorbar(im1, format = cb_format)
ulalume3@27 689 else:
ulalume3@27 690 cb_formatter = ScalarFormatter()
ulalume3@27 691 cb_formatter.set_powerlimits(power_limits)
ulalume3@27 692 cb1 = plt.colorbar(im1, format = cb_formatter)
ulalume3@27 693 cb1.ax.set_ylabel(cmap_label)
ulalume3@27 694
ulalume3@27 695 # Make the ticks of the colorbar smaller, two points smaller than the default font size
ulalume3@27 696 cb_font_size = mpl.rcParams['font.size'] - 2
ulalume3@27 697 for ticklabels in cb1.ax.get_yticklabels():
ulalume3@27 698 ticklabels.set_fontsize(cb_font_size)
ulalume3@27 699 cb1.ax.yaxis.get_offset_text().set_fontsize(cb_font_size)
ulalume3@27 700
binietoglou@0 701 def index_at_height(self, height):
binietoglou@0 702 idx = np.array(np.abs(self.z - height).argmin())
ulalume3@24 703 if idx.size > 1:
binietoglou@0 704 idx =idx[0]
binietoglou@0 705 return idx
binietoglou@0 706
binietoglou@0 707 def netcdf_from_files(LidarClass, filename, files, channels, measurement_ID):
binietoglou@0 708 #Read the lidar files and select channels
binietoglou@0 709 temp_m = LidarClass(files)
binietoglou@0 710 m = temp_m.subset_by_channels(channels)
binietoglou@0 711 m.get_PT()
binietoglou@0 712 m.info['Measurement_ID'] = measurement_ID
binietoglou@0 713 m.save_as_netcdf(filename)
binietoglou@0 714

mercurial