# HG changeset patch # User Ioannis # Date 1456151963 -7200 # Node ID 2984158468e611f3dea4788f968c0e807f62826e # Parent 022f6f2bc09c0f43dbf145f10a574dbd8ba5cc15 Commiting all (bug in EasyHG?) diff -r 022f6f2bc09c -r 2984158468e6 .hgignore diff -r 022f6f2bc09c -r 2984158468e6 __init__.py diff -r 022f6f2bc09c -r 2984158468e6 aias_netcdf_parameters.py diff -r 022f6f2bc09c -r 2984158468e6 at_netcdf_parameters.py diff -r 022f6f2bc09c -r 2984158468e6 cf_netcdf_parameters.py diff -r 022f6f2bc09c -r 2984158468e6 cf_raymetrics.py diff -r 022f6f2bc09c -r 2984158468e6 ciao.py diff -r 022f6f2bc09c -r 2984158468e6 eole.py diff -r 022f6f2bc09c -r 2984158468e6 eole_netcdf_parameters.py diff -r 022f6f2bc09c -r 2984158468e6 generic.py --- a/generic.py Wed Apr 01 13:06:08 2015 +0300 +++ b/generic.py Mon Feb 22 16:39:23 2016 +0200 @@ -26,7 +26,7 @@ """ - def __init__(self, filelist= None): + def __init__(self, filelist = None): self.info = {} self.dimensions = {} self.variables = {} @@ -34,10 +34,11 @@ self.attributes = {} self.files = [] self.dark_measurement = None + if filelist: self.import_files(filelist) - def import_files(self,filelist): + def import_files(self, filelist): for f in filelist: self.import_file(f) self.update() @@ -114,7 +115,7 @@ self.variables['id_timescale'] = dict(channel_timescales) def _get_duration(self, raw_start_in_seconds): - ''' Return the duration for a given time scale. In some files (ex. Licel) this + ''' Return the duration for a given time scale. In some files (e.g. Licel) this can be specified from the files themselves. In others this must be guessed. ''' @@ -414,7 +415,7 @@ self.binwidth = float(channel_parameters['binwidth']) # in microseconds self.data = {} self.resolution = self.binwidth * c / 2 - self.z = np.arange(len(channel_parameters['data'])) * self.resolution + self.resolution/2.0 # Change: add half bin in the z + self.z = np.arange(len(channel_parameters['data'])) * self.resolution + self.resolution / 2.0 # Change: add half bin in the z self.points = len(channel_parameters['data']) self.rc = [] self.duration = 60 diff -r 022f6f2bc09c -r 2984158468e6 licel.py --- a/licel.py Wed Apr 01 13:06:08 2015 +0300 +++ b/licel.py Mon Feb 22 16:39:23 2016 +0200 @@ -5,7 +5,7 @@ import musa_2009_netcdf_parameters licel_file_header_format = ['Filename', - 'Site StartDate StartTime EndDate EndTime Altitude Longtitude Latitude ZenithAngle', + 'StartDate StartTime EndDate EndTime Altitude Longtitude Latitude ZenithAngle', # Appart from Site that is read manually 'LS1 Rate1 LS2 Rate2 DataSets', ] licel_file_channel_format = 'Active AnalogPhoton LaserUsed DataPoints 1 HV BinW Wavelength d1 d2 d3 d4 ADCbits NShots Discriminator ID' @@ -28,53 +28,83 @@ Input: filename Output: object """ - raw_info = {} channels = {} - channel_info = [] + + with open(filename, 'rb') as f: + + self.read_header(f) + + # Check the complete header is read + a = f.readline() - f = open(filename, 'rb') - + # Import the data + for current_channel_info in self.channel_info: + raw_data = np.fromfile(f, 'i4', int(current_channel_info['DataPoints'])) + a = np.fromfile(f, 'b', 1) + b = np.fromfile(f, 'b', 1) + + if (a[0] != 13) | (b[0] != 10): + print "Warning: No end of line found after record. File could be corrupt" + channel = LicelFileChannel(current_channel_info, raw_data, self.duration()) + + channel_name = channel.channel_name + + if channel_name in channels.keys(): + # If the analog/photon naming scheme is not enough, find a new one! + raise IOError('Trying to import two channels with the same name') + + channels[channel_name] = channel + + self.channels = channels + + + def read_header(self, f): + """ Read the header of a open file f. + + Returns raw_info and channel_info. Updates some object properties. """ + #Read the first 3 lines of the header raw_info = {} - for c1 in range(3): - raw_info.update(match_lines(f.readline(), licel_file_header_format[c1])) - + channel_info = [] + + # Read first line + raw_info['Filename'] = f.readline().strip() + + # Read second line + second_line = f.readline() + + # Many licel files don't follow the licel standard. Specifically, the + # measurement site is not always 8 characters, and can include white + # spaces. For this, the site name is detect everything before the first + # date. For efficiency, the first date is found by the first '/'. + # e.g. assuming a string like 'Site name 01/01/2010 ...' + + site_name = second_line.split('/')[0][:-2] + clean_site_name = site_name.strip() + raw_info['Site'] = clean_site_name + raw_info.update(match_lines(second_line[len(clean_site_name) + 1:], licel_file_header_format[1])) + + # Read third line + third_line = f.readline() + raw_info.update(match_lines(third_line, licel_file_header_format[2])) + + # Update the object properties based on the raw info start_string = '%s %s' % (raw_info['StartDate'], raw_info['StartTime']) stop_string = '%s %s' % (raw_info['EndDate'], raw_info['EndTime']) date_format = '%d/%m/%Y %H:%M:%S' + self.start_time = datetime.datetime.strptime(start_string, date_format) self.stop_time = datetime.datetime.strptime(stop_string, date_format) self.latitude = float(raw_info['Latitude']) self.longitude = float(raw_info['Longtitude']) - + # Read the rest of the header. for c1 in range(int(raw_info['DataSets'])): channel_info.append(match_lines(f.readline(), licel_file_channel_format)) - - # Check the complete header is read - a = f.readline() - - # Import the data - for current_channel_info in channel_info: - raw_data = np.fromfile(f, 'i4', int(current_channel_info['DataPoints'])) - a = np.fromfile(f, 'b', 1) - b = np.fromfile(f, 'b', 1) + + self.raw_info = raw_info + self.channel_info = channel_info - if (a[0] != 13) | (b[0] != 10): - print "Warning: No end of line found after record. File could be corrupt" - channel = LicelFileChannel(current_channel_info, raw_data, self.duration()) - - channel_name = channel.channel_name - if channel_name in channels.keys(): - # If the analog/photon naming scheme is not enough, find a new one! - raise IOError('Trying to import two channels with the same name') - - channels[channel_name] = channel - f.close() - - self.raw_info = raw_info - self.channels = channels - def duration(self): """ Return the duration of the file. """ dt = self.stop_time - self.start_time @@ -194,7 +224,7 @@ class LicelChannel(LidarChannel): def __init__(self, channel_file): - c = 299792458.0 # Speed of light + c = 299792458.0 # Speed of light self.wavelength = channel_file.wavelength self.name = channel_file.channel_name self.binwidth = channel_file.dz * 2 / c # in seconds diff -r 022f6f2bc09c -r 2984158468e6 musa.py diff -r 022f6f2bc09c -r 2984158468e6 musa_2009_netcdf_parameters.py diff -r 022f6f2bc09c -r 2984158468e6 musa_netcdf_parameters.py diff -r 022f6f2bc09c -r 2984158468e6 pearl.py diff -r 022f6f2bc09c -r 2984158468e6 pearl_netcdf_parameters.py diff -r 022f6f2bc09c -r 2984158468e6 rali.py diff -r 022f6f2bc09c -r 2984158468e6 rali_netcdf_parameters.py diff -r 022f6f2bc09c -r 2984158468e6 readme.md