# HG changeset patch # User Iannis # Date 1519142848 -7200 # Node ID a8670c4038239c7a8b562c3b1f9c0c1c5d8c079f # Parent a1788b77f33b9fe59b085122f5c283770e34dd1c Adding first attempts to support Raymetrics-only files artifacts: * New header lines for scanning measurements * Photodiode channels. diff -r a1788b77f33b -r a8670c403823 atmospheric_lidar/licel.py --- a/atmospheric_lidar/licel.py Fri Feb 16 14:39:54 2018 +0200 +++ b/atmospheric_lidar/licel.py Tue Feb 20 18:07:28 2018 +0200 @@ -9,18 +9,20 @@ logger = logging.getLogger(__name__) -licel_file_header_format = ['Filename', - 'StartDate StartTime EndDate EndTime Altitude Longtitude Latitude ZenithAngle', - # Appart from Site that is read manually - 'LS1 Rate1 LS2 Rate2 DataSets', ] -licel_file_channel_format = 'Active AnalogPhoton LaserUsed DataPoints 1 HV BinW Wavelength d1 d2 d3 d4 ADCbits NShots Discriminator ID' - c = 299792458.0 # Speed of light class LicelFile: """ A class representing a single binary Licel file. """ + licel_file_header_format = ['Filename', + 'StartDate StartTime EndDate EndTime Altitude Longtitude Latitude ZenithAngle', + # Appart from Site that is read manually + 'LS1 Rate1 LS2 Rate2 DataSets', ] + licel_file_channel_format = 'Active AnalogPhoton LaserUsed DataPoints 1 HV BinW Wavelength d1 d2 d3 d4 ADCbits NShots Discriminator ID' + + file_channel_class = LicelFileChannel + def __init__(self, file_path, use_id_as_name=False, licel_timezone="UTC"): """ This is run when creating a new object. @@ -74,7 +76,7 @@ if (a[0] != 13) | (b[0] != 10): logging.warning("No end of line found after record. File could be corrupt: %s" % file_path) - channel = LicelFileChannel(current_channel_info, raw_data, self.duration(), + channel = self.file_channel_class(current_channel_info, raw_data, self.duration(), use_id_as_name=self.use_id_as_name) channel_name = channel.channel_name @@ -102,23 +104,9 @@ # Read first line raw_info['Filename'] = f.readline().strip() - # Read second line - second_line = f.readline() - - # Many Licel files don't follow the licel standard. Specifically, the - # measurement site is not always 8 characters, and can include white - # spaces. For this, the site name is detect everything before the first - # date. For efficiency, the first date is found by the first '/'. - # e.g. assuming a string like 'Site name 01/01/2010 ...' + raw_info.update(self._read_second_header_line(f)) - site_name = second_line.split('/')[0][:-2] - clean_site_name = site_name.strip() - raw_info['Site'] = clean_site_name - raw_info.update(self.match_lines(second_line[len(clean_site_name) + 1:], licel_file_header_format[1])) - - # Read third line - third_line = f.readline() - raw_info.update(self.match_lines(third_line, licel_file_header_format[2])) + raw_info.update(self._read_rest_of_header(f)) # Update the object properties based on the raw info start_string = '%s %s' % (raw_info['StartDate'], raw_info['StartTime']) @@ -144,11 +132,36 @@ # Read the rest of the header. for c1 in range(int(raw_info['DataSets'])): - channel_info.append(self.match_lines(f.readline(), licel_file_channel_format)) + channel_info.append(self.match_lines(f.readline(), self.licel_file_channel_format)) self.raw_info = raw_info self.channel_info = channel_info + def _read_second_header_line(self, f): + """ Read the second line of a licel file. """ + raw_info = {} + + second_line = f.readline() + # Many Licel files don't follow the licel standard. Specifically, the + # measurement site is not always 8 characters, and can include white + # spaces. For this, the site name is detect everything before the first + # date. For efficiency, the first date is found by the first '/'. + # e.g. assuming a string like 'Site name 01/01/2010 ...' + + site_name = second_line.split('/')[0][:-2] + clean_site_name = site_name.strip() + raw_info['Site'] = clean_site_name + + raw_info.update(self.match_lines(second_line[len(clean_site_name) + 1:], self.licel_file_header_format[1])) + return raw_info + + def _read_rest_of_header(self, f): + """ Read the rest of the header lines, after line 2. """ + # Read third line + third_line = f.readline() + raw_dict = self.match_lines(third_line, self.licel_file_header_format[2]) + return raw_dict + def duration(self): """ Return the duration of the file. @@ -278,8 +291,8 @@ if self.raw_info['AnalogPhoton'] == '0': # If the channel is in analog mode - ADCrange = self.discriminator * 1000 # Value in mV - channel_data = norm * ADCrange / ((2 ** self.adcbits) - 1) + ADCrange = self.discriminator # Discriminator value already in mV + channel_data = norm * ADCrange / ((2 ** self.adcbits) - 1) # TODO: Check this. Agrees with Licel docs, but differs from their LabView code. # print ADCb, ADCRange,cdata,norm else: @@ -306,6 +319,9 @@ class LicelLidarMeasurement(BaseLidarMeasurement): + file_class = LicelFile + channel_class = LicelChannel + def __init__(self, file_list=None, use_id_as_name=False, licel_timezone='UTC'): self.raw_info = {} # Keep the raw info from the files self.durations = {} # Keep the duration of the files @@ -320,20 +336,24 @@ logger.warning("File has been imported already: %s" % filename) else: logger.debug('Importing file {0}'.format(filename)) - current_file = LicelFile(filename, use_id_as_name=self.use_id_as_name, licel_timezone=self.licel_timezone) + current_file = self.file_class(filename, use_id_as_name=self.use_id_as_name, licel_timezone=self.licel_timezone) self.raw_info[current_file.filename] = current_file.raw_info self.durations[current_file.filename] = current_file.duration() file_laser_shots = [] - for channel_name, channel in current_file.channels.items(): - if channel_name not in self.channels: - self.channels[channel_name] = LicelChannel() - self.channels[channel_name].append_file(current_file.start_time, channel) + self._create_or_append_channel(current_file) self.laser_shots.append(file_laser_shots) self.files.append(current_file.filename) + def _create_or_append_channel(self, current_file): + + for channel_name, channel in current_file.channels.items(): + if channel_name not in self.channels: + self.channels[channel_name] = self.channel_class() + self.channels[channel_name].append_file(current_file.start_time, channel) + def append(self, other): self.start_times.extend(other.start_times) @@ -468,15 +488,7 @@ def append_file(self, file_start_time, file_channel): """ Append file to the current object """ - self._assign_unique_property('name', file_channel.channel_name) - self._assign_unique_property('resolution', file_channel.dz) - self._assign_unique_property('wavelength', file_channel.wavelength) - self._assign_unique_property('points', file_channel.data_points) - self._assign_unique_property('adcbits', file_channel.adcbits) - self._assign_unique_property('active', file_channel.active) - self._assign_unique_property('laser_user', file_channel.laser_user) - self._assign_unique_property('adcbints', file_channel.adcbits) - self._assign_unique_property('analog_photon_string', file_channel.analog_photon_string) + self._assign_properties(file_channel) self.binwidth = self.resolution * 2. / c # in seconds self.z = file_channel.z @@ -489,6 +501,17 @@ self.discriminator.append(file_channel.discriminator) self.hv.append(file_channel.hv) + def _assign_properties(self, file_channel): + self._assign_unique_property('name', file_channel.channel_name) + self._assign_unique_property('resolution', file_channel.dz) + self._assign_unique_property('wavelength', file_channel.wavelength) + self._assign_unique_property('points', file_channel.data_points) + self._assign_unique_property('adcbits', file_channel.adcbits) + self._assign_unique_property('active', file_channel.active) + self._assign_unique_property('laser_user', file_channel.laser_user) + self._assign_unique_property('adcbints', file_channel.adcbits) + self._assign_unique_property('analog_photon_string', file_channel.analog_photon_string) + def _assign_unique_property(self, property_name, value): current_value = getattr(self, property_name, None) diff -r a1788b77f33b -r a8670c403823 atmospheric_lidar/raymetrics.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/atmospheric_lidar/raymetrics.py Tue Feb 20 18:07:28 2018 +0200 @@ -0,0 +1,69 @@ +""" Code to read Raymetrics version of Licel binary files.""" +import datetime +import logging + +import pytz + +from .licel import LicelFile, LicelLidarMeasurement, LicelChannel + +logger = logging.getLogger(__name__) + + +class RaymetricsLidarMeasurement(LicelLidarMeasurement): + + def __init__(self, file_list=None, use_id_as_name=False, licel_timezone='UTC'): + self.photodiodes = {} # Add photodiode dictionary + super(RaymetricsLidarMeasurement, self).__init__(file_list=file_list, + use_id_as_name=use_id_as_name, + licel_timezone=licel_timezone) + + def _create_or_append_channel(self, current_file): + + for channel_name, channel in current_file.channels.items(): + if channel_name[0:2] == 'PD': + if channel_name not in self.photodiodes: + self.photodiodes[channel_name] = PhotodiodeChannel() + self.photodiodes[channel_name].append_file(current_file.start_time, channel) + else: + if channel_name not in self.channels: + self.channels[channel_name] = self.channel_class() + self.channels[channel_name].append_file(current_file.start_time, channel) + + +class ScanningFile(LicelFile): + licel_file_header_format = ['Filename', + 'StartDate StartTime EndDate EndTime Altitude Longtitude Latitude ZenithAngle Temperature Pressure', # Appart from Site that is read manually + 'azimuth_start azimuth_finish azimuth_step zenith_start zenith_finish zenith_step azimuth_offset', + 'LS1 Rate1 LS2 Rate2 DataSets', ] + licel_file_channel_format = 'Active AnalogPhoton LaserUsed DataPoints 1 HV BinW Wavelength d1 d2 d3 d4 ADCbits NShots Discriminator ID' + + + def _read_rest_of_header(self, f): + """ Read the rest of the header lines, after line 2. """ + raw_info = {} + + third_line = f.readline() + raw_info.update(self.match_lines(third_line, self.licel_file_header_format[2])) + + fourth_line = f.readline() + raw_info.update(self.match_lines(fourth_line, self.licel_file_header_format[3])) + return raw_info + + +class ScanningLidarMeasurement(RaymetricsLidarMeasurement): + file_class = ScanningFile + channel_class = LicelChannel + + +class PhotodiodeChannel(LicelChannel): + + def _assign_properties(self, file_channel): + """ In contrast with normal channels, don't check for constant points.""" + self._assign_unique_property('name', file_channel.channel_name) + self._assign_unique_property('resolution', file_channel.dz) + self._assign_unique_property('wavelength', file_channel.wavelength) + self._assign_unique_property('adcbits', file_channel.adcbits) + self._assign_unique_property('active', file_channel.active) + self._assign_unique_property('laser_user', file_channel.laser_user) + self._assign_unique_property('adcbints', file_channel.adcbits) + self._assign_unique_property('analog_photon_string', file_channel.analog_photon_string) \ No newline at end of file