# HG changeset patch # User Victor Nicolae # Date 1513250692 -7200 # Node ID 2f3f75e5b99e93adc7db9e0d03e1a7ef424d901e # Parent 6511136e69375302d1934b8532857af282c75398# Parent ef3b6f838da1d5d83d16c1fe0c55fe62865dbef5 Merge. diff -r 6511136e6937 -r 2f3f75e5b99e atmospheric_lidar/__init__.py --- a/atmospheric_lidar/__init__.py Thu Dec 14 13:24:15 2017 +0200 +++ b/atmospheric_lidar/__init__.py Thu Dec 14 13:24:52 2017 +0200 @@ -1,1 +1,1 @@ -__version__ = '0.2.11' \ No newline at end of file +__version__ = '0.2.13' \ No newline at end of file diff -r 6511136e6937 -r 2f3f75e5b99e atmospheric_lidar/generic.py --- a/atmospheric_lidar/generic.py Thu Dec 14 13:24:15 2017 +0200 +++ b/atmospheric_lidar/generic.py Thu Dec 14 13:24:52 2017 +0200 @@ -10,6 +10,8 @@ NETCDF_FORMAT = 'NETCDF4' # choose one of 'NETCDF3_CLASSIC', 'NETCDF3_64BIT', 'NETCDF4_CLASSIC' and 'NETCDF4' +logger = logging.getLogger(__name__) + class BaseLidarMeasurement(object): """ @@ -23,6 +25,8 @@ The class assumes that the input files are consecutive, i.e. there are no measurements gaps. """ + extra_netcdf_parameters = None + def __init__(self, file_list=None): """ This is run when creating a new object. @@ -39,7 +43,6 @@ self.attributes = {} self.files = [] self.dark_measurement = None - self.extra_netcdf_parameters = None if file_list: self._import_files(file_list) @@ -175,10 +178,13 @@ m = self.__class__() # Create an object of the same type as this one. m.channels = dict([(channel, self.channels[channel]) for channel in channel_subset]) + m.files = self.files + m.update() - m.files = self.files - - m.update() + # Dark measurements should also be subseted. + if self.dark_measurement is not None: + dark_subset = self.dark_measurement.subset_by_channels(channel_subset) + m.dark_measurement = dark_subset return m @@ -233,6 +239,9 @@ m.channels[channel_name] = channel.subset_by_time(start_time, stop_time) m.update() + + # Transfer dark measurement to the new object. They don't need time subsetting. + m.dark_measurement = self.dark_measurement return m def subset_by_bins(self, b_min=0, b_max=None): @@ -261,6 +270,11 @@ m.update() + # Dark measurements should also be subseted. + if self.dark_measurement is not None: + dark_subset = self.dark_measurement.subset_by_bins(b_min, b_max) + m.dark_measurement = dark_subset + return m def rename_channels(self, prefix="", suffix=""): @@ -294,7 +308,7 @@ def subtract_dark(self): """ Subtract dark measurements from the raw lidar signals. - + This method is here just for testing. Note diff -r 6511136e6937 -r 2f3f75e5b99e atmospheric_lidar/licel.py --- a/atmospheric_lidar/licel.py Thu Dec 14 13:24:15 2017 +0200 +++ b/atmospheric_lidar/licel.py Thu Dec 14 13:24:52 2017 +0200 @@ -2,12 +2,12 @@ import logging import numpy as np - -from .systems.musa import musa_2009_netcdf_parameters -from .systems.musa import musa_netcdf_parameters +import pytz from generic import BaseLidarMeasurement, LidarChannel +logger = logging.getLogger(__name__) + licel_file_header_format = ['Filename', 'StartDate StartTime EndDate EndTime Altitude Longtitude Latitude ZenithAngle', # Appart from Site that is read manually @@ -16,31 +16,51 @@ class LicelFile: - def __init__(self, filename, use_id_as_name=False): - self.filename = filename + """ A class representing a single binary Licel file. """ + def __init__(self, file_path, use_id_as_name=False, licel_timezone="UTC"): + """ + This is run when creating a new object. + + Parameters + ---------- + file_path : str + The path to the Licel file. + use_id_as_name : bool + If True, the transient digitizer name (e.g. BT0) is used as a channel + name. If False, a more descriptive name is used (e.g. '01064.o_an'). + licel_timezone : str + The timezone of dates found in the Licel files. Should match the available + timezones in the TZ database. + """ + self.filename = file_path self.use_id_as_name = use_id_as_name self.start_time = None self.stop_time = None - self.import_file(filename) + self.licel_timezone = licel_timezone + self._import_file(file_path) self.calculate_physical() def calculate_physical(self): + """ Calculate physical quantities from raw data for all channels in the file. """ for channel in self.channels.itervalues(): channel.calculate_physical() - def import_file(self, filename): - """Imports a licel file. - Input: filename - Output: object """ - + def _import_file(self, file_path): + """ Read the content of the Licel file. + + Parameters + ---------- + file_path : str + The path to the Licel file. + """ channels = {} - with open(filename, 'rb') as f: + with open(file_path, 'rb') as f: self.read_header(f) # Check the complete header is read - a = f.readline() + f.readline() # Import the data for current_channel_info in self.channel_info: @@ -49,7 +69,7 @@ b = np.fromfile(f, 'b', 1) if (a[0] != 13) | (b[0] != 10): - logging.warning("No end of line found after record. File could be corrupt: %s" % filename) + logging.warning("No end of line found after record. File could be corrupt: %s" % file_path) channel = LicelFileChannel(current_channel_info, raw_data, self.duration(), use_id_as_name=self.use_id_as_name) @@ -64,10 +84,13 @@ self.channels = channels def read_header(self, f): - """ Read the header of a open file f. + """ Read the header of an open Licel file. - Returns raw_info and channel_info. Updates some object properties. """ - + Parameters + ---------- + f : file-like object + An open file object. + """ # Read the first 3 lines of the header raw_info = {} channel_info = [] @@ -98,8 +121,20 @@ stop_string = '%s %s' % (raw_info['EndDate'], raw_info['EndTime']) date_format = '%d/%m/%Y %H:%M:%S' - self.start_time = datetime.datetime.strptime(start_string, date_format) - self.stop_time = datetime.datetime.strptime(stop_string, date_format) + try: + logger.debug('Creating timezone object %s' % self.licel_timezone) + timezone = pytz.timezone(self.licel_timezone) + except: + raise ValueError("Cloud not create time zone object %s" % self.licel_timezone) + + # According to pytz docs, timezones do not work with default datetime constructor. + local_start_time = timezone.localize(datetime.datetime.strptime(start_string, date_format)) + local_stop_time = timezone.localize(datetime.datetime.strptime(stop_string, date_format)) + + # Only save UTC time. + self.start_time = local_start_time.astimezone(pytz.utc) + self.stop_time = local_stop_time.astimezone(pytz.utc) + self.latitude = float(raw_info['Latitude']) self.longitude = float(raw_info['Longtitude']) @@ -111,13 +146,35 @@ self.channel_info = channel_info def duration(self): - """ Return the duration of the file. """ + """ Return the duration of the file. + + Returns + ------- + : float + The duration of the file in seconds. + """ dt = self.stop_time - self.start_time return dt.seconds class LicelFileChannel: + """ A class representing a single channel found in a single Licel file.""" def __init__(self, raw_info=None, raw_data=None, duration=None, use_id_as_name=False): + """ + This is run when creating a new object. + + Parameters + ---------- + raw_info : dict + A dictionary containing raw channel information. + raw_data : dict + An array with raw channel data. + duration : float + Duration of the file, in seconds + use_id_as_name : bool + If True, the transient digitizer name (e.g. BT0) is used as a channel + name. If False, a more descriptive name is used (e.g. '01064.o_an'). + """ self.raw_info = raw_info self.raw_data = raw_data self.duration = duration @@ -125,6 +182,14 @@ @property def wavelength(self): + """ Property describing the nominal wavelength of the channel. + + Returns + ------- + : int or None + The integer value describing the wavelength. If no raw_info have been provided, + returns None. + """ if self.raw_info is not None: wave_str = self.raw_info['Wavelength'] wavelength = wave_str.split('.')[0] @@ -134,13 +199,18 @@ @property def channel_name(self): - ''' + """ Construct the channel name adding analog photon info to avoid duplicates If use_id_as_name is True, the channel name will be the transient digitizer ID (e.g. BT01). This could be useful if the lidar system has multiple telescopes, so the descriptive name is not unique. - ''' + + Returns + ------- + channel_name : str + The channel name + """ if self.use_id_as_name: channel_name = self.raw_info['ID'] else: @@ -148,7 +218,20 @@ channel_name = "%s_%s" % (self.raw_info['Wavelength'], acquisition_type) return channel_name - def analog_photon_string(self, analog_photon_number): + @staticmethod + def analog_photon_string(analog_photon_number): + """ Convert the analog/photon flag found in the Licel file to a proper sting. + + Parameters + ---------- + analog_photon_number : int + 0 or 1 indicating analog or photon counting channel. + + Returns + ------- + string : str + 'an' or 'ph' string, for analog or photon-counting channel respectively. + """ if analog_photon_number == '0': string = 'an' else: @@ -156,6 +239,13 @@ return string def calculate_physical(self): + """ Calculate physically-meaningful data from raw channel data: + + * In case of analog signals, the data are converted to mV. + * In case of photon counting signals, data are stored as number of photons. + + In addition, some ancillary variables are also calculated (z, dz, number_of_bins). + """ data = self.raw_data number_of_shots = float(self.raw_info['NShots']) @@ -189,26 +279,23 @@ class LicelLidarMeasurement(BaseLidarMeasurement): - ''' - - ''' - extra_netcdf_parameters = musa_netcdf_parameters raw_info = {} # Keep the raw info from the files durations = {} # Keep the duration of the files laser_shots = [] - def __init__(self, file_list=None, use_id_as_name=False): + def __init__(self, file_list=None, use_id_as_name=False, licel_timezone='UTC'): self.use_id_as_name = use_id_as_name + self.licel_timezone = licel_timezone super(LicelLidarMeasurement, self).__init__(file_list) def _import_file(self, filename): if filename in self.files: logging.warning("File has been imported already: %s" % filename) else: - current_file = LicelFile(filename, use_id_as_name=self.use_id_as_name) + current_file = LicelFile(filename, use_id_as_name=self.use_id_as_name, licel_timezone=self.licel_timezone) self.raw_info[current_file.filename] = current_file.raw_info self.durations[current_file.filename] = current_file.duration() - + file_laser_shots = [] for channel_name, channel in current_file.channels.items(): @@ -216,7 +303,7 @@ self.channels[channel_name] = LicelChannel(channel) self.channels[channel_name].data[current_file.start_time] = channel.data file_laser_shots.append(channel.raw_info['NShots']) - + self.laser_shots.append(file_laser_shots) self.files.append(current_file.filename) @@ -241,39 +328,39 @@ duration_sec = np.diff(raw_start_in_seconds)[0] return duration_sec - + def get_custom_channel_parameters(self): params = [{ - "name": "DAQ_Range", - "dimensions": ('channels',), - "type": 'd', - "values": [self.channels[x].raw_info['Discriminator'] for x in self.channels.keys()] - }, { - "name": "LR_Input", - "dimensions": ('channels',), - "type": 'i', - "values": [self.channels[x].raw_info['LaserUsed'] for x in self.channels.keys()] - }, { - "name": "Laser_Shots", - "dimensions": ('time', 'channels',), - "type": 'i', - "values": self.laser_shots - }, + "name": "DAQ_Range", + "dimensions": ('channels',), + "type": 'd', + "values": [self.channels[x].raw_info['Discriminator'] for x in self.channels.keys()] + }, { + "name": "LR_Input", + "dimensions": ('channels',), + "type": 'i', + "values": [self.channels[x].raw_info['LaserUsed'] for x in self.channels.keys()] + }, { + "name": "Laser_Shots", + "dimensions": ('time', 'channels',), + "type": 'i', + "values": self.laser_shots + }, ] - + return params - + def get_custom_general_parameters(self): params = [{ - "name": "Altitude_meter_asl", - "value": self.raw_info[ self.files[0] ]["Altitude"] - }, { - "name": "Latitude_degrees_north", - "value": self.raw_info[ self.files[0] ]["Latitude"] - }, { - "name": "Longitude_degrees_east", - "value": self.raw_info[ self.files[0] ]["Longtitude"] - }, + "name": "Altitude_meter_asl", + "value": self.raw_info[self.files[0]]["Altitude"] + }, { + "name": "Latitude_degrees_north", + "value": self.raw_info[self.files[0]]["Latitude"] + }, { + "name": "Longitude_degrees_east", + "value": self.raw_info[self.files[0]]["Longtitude"] + }, ] return params @@ -293,7 +380,7 @@ self.rc = [] self.duration = channel_file.duration self.raw_info = channel_file.raw_info - + def append(self, other): if self.info != other.info: raise ValueError('Channel info are different. Data can not be combined.') @@ -307,10 +394,6 @@ return unicode(self).encode('utf-8') -class Licel2009LidarMeasurement(LicelLidarMeasurement): - extra_netcdf_parameters = musa_2009_netcdf_parameters - - def match_lines(f1, f2): list1 = f1.split() list2 = f2.split() diff -r 6511136e6937 -r 2f3f75e5b99e atmospheric_lidar/licel_depol.py --- a/atmospheric_lidar/licel_depol.py Thu Dec 14 13:24:15 2017 +0200 +++ b/atmospheric_lidar/licel_depol.py Thu Dec 14 13:24:52 2017 +0200 @@ -1,15 +1,17 @@ -import datetime +import logging import numpy as np from licel import LicelLidarMeasurement +logger = logging.getLogger(__name__) + class LicelCalibrationMeasurement(LicelLidarMeasurement): - def __init__(self, plus45_files=None, minus45_files=None, use_id_as_name=False): + def __init__(self, plus45_files=None, minus45_files=None, use_id_as_name=False, licel_timezone='UTC'): # Setup the empty class - super(LicelCalibrationMeasurement, self).__init__(use_id_as_name=use_id_as_name) + super(LicelCalibrationMeasurement, self).__init__(use_id_as_name=use_id_as_name, licel_timezone=licel_timezone) self.plus45_files = plus45_files self.minus45_files = minus45_files @@ -39,10 +41,10 @@ def read_channel_data(self): # Read plus and minus 45 measurements - self.plus45_measurement = LicelLidarMeasurement(self.plus45_files, self.use_id_as_name) + self.plus45_measurement = LicelLidarMeasurement(self.plus45_files, self.use_id_as_name, self.licel_timezone) self.plus45_measurement.rename_channels(suffix='_p45') - self.minus45_measurement = LicelLidarMeasurement(self.minus45_files, self.use_id_as_name) + self.minus45_measurement = LicelLidarMeasurement(self.minus45_files, self.use_id_as_name, self.licel_timezone) self.minus45_measurement.rename_channels(suffix='_m45') # Combine them in this object diff -r 6511136e6937 -r 2f3f75e5b99e atmospheric_lidar/scripts/licel2scc.py --- a/atmospheric_lidar/scripts/licel2scc.py Thu Dec 14 13:24:15 2017 +0200 +++ b/atmospheric_lidar/scripts/licel2scc.py Thu Dec 14 13:24:52 2017 +0200 @@ -7,39 +7,43 @@ import os import sys - from ..licel import LicelLidarMeasurement from ..__init__ import __version__ +logger = logging.getLogger(__name__) -def create_custom_class(custom_netcdf_parameter_path, use_id_as_name=False, temperature=25., pressure=1020.): + +def create_custom_class(custom_netcdf_parameter_path, use_id_as_name=False, temperature=25., pressure=1020., + licel_timezone='UTC'): """ This funtion creates a custom LicelLidarMeasurement subclass, based on the input provided by the users. Parameters ---------- - custom_netcdf_parameter_path: str + custom_netcdf_parameter_path : str The path to the custom channels parameters. - use_id_as_name: bool + use_id_as_name : bool Defines if channels names are descriptive or transient digitizer IDs. - temperature: float + temperature : float The ground temperature in degrees C (default 25.0). - pressure: float + pressure : float The ground pressure in hPa (default: 1020.0). + licel_timezone : str + String describing the timezone according to the tz database. Returns ------- CustomLidarMeasurement: A custom sub-class of LicelLidarMeasurement """ - + logger.debug('Reading parameter files: %s' % custom_netcdf_parameter_path) custom_netcdf_parameters = read_settings_file(custom_netcdf_parameter_path) class CustomLidarMeasurement(LicelLidarMeasurement): extra_netcdf_parameters = custom_netcdf_parameters def __init__(self, file_list=None): - super(CustomLidarMeasurement, self).__init__(file_list, use_id_as_name) + super(CustomLidarMeasurement, self).__init__(file_list, use_id_as_name, licel_timezone=licel_timezone) def set_PT(self): ''' Sets the pressure and temperature at station level. This is used if molecular_calc parameter is @@ -89,6 +93,12 @@ parser.add_argument("-p", "--pressure", type=float, help="The pressure (in hPa) at lidar level, required if using US Standard atmosphere", default="1020") + parser.add_argument('-D', '--dark_measurements', help="Location of files containing dark measurements. Use relative path and filename wildcars, see 'files' parameter for example.", + default="", dest="dark_files" + ) + parser.add_argument('--licel_timezone', help="String describing the timezone according to the tz database.", + default="UTC", dest="licel_timezone", + ) # Verbosity settings from http://stackoverflow.com/a/20663028 parser.add_argument('-d', '--debug', help="Print dubuging information.", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO, @@ -96,9 +106,6 @@ parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const", dest="loglevel", const=logging.WARNING ) - parser.add_argument('-D', '--dark_measurements', help="Location of files containing dark measurements. Use relative path and filename wildcars, see 'files' parameter for example.", - default="", dest="dark_files" - ) parser.add_argument('--version', help="Show current version.", action='store_true') args = parser.parse_args() @@ -107,7 +114,7 @@ logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel) logger = logging.getLogger(__name__) - #coloredlogs.install(fmt='%(levelname)s: %(message)s', level=args.loglevel) + # coloredlogs.install(fmt='%(levelname)s: %(message)s', level=args.loglevel) # Check for version if args.version: @@ -125,7 +132,7 @@ # If everything OK, proceed logger.info("Found {0} files matching {1}".format(len(files), os.path.abspath(args.files))) CustomLidarMeasurement = create_custom_class(args.parameter_file, args.id_as_name, args.temperature, - args.pressure) + args.pressure, args.licel_timezone) measurement = CustomLidarMeasurement(files) # Get a list of files containing dark measurements diff -r 6511136e6937 -r 2f3f75e5b99e atmospheric_lidar/scripts/licel2scc_depol.py --- a/atmospheric_lidar/scripts/licel2scc_depol.py Thu Dec 14 13:24:15 2017 +0200 +++ b/atmospheric_lidar/scripts/licel2scc_depol.py Thu Dec 14 13:24:52 2017 +0200 @@ -7,25 +7,29 @@ import os import sys - from ..licel_depol import LicelCalibrationMeasurement from ..__init__ import __version__ +logger = logging.getLogger(__name__) -def create_custom_class(custom_netcdf_parameter_path, use_id_as_name=False, temperature=25., pressure=1020.): + +def create_custom_class(custom_netcdf_parameter_path, use_id_as_name=False, temperature=25., pressure=1020., + licel_timezone='UTC'): """ This funtion creates a custom LicelLidarMeasurement subclass, based on the input provided by the users. Parameters ---------- - custom_netcdf_parameter_path: str + custom_netcdf_parameter_path : str The path to the custom channels parameters. - use_id_as_name: bool + use_id_as_name : bool Defines if channels names are descriptive or transient digitizer IDs. - temperature: float + temperature : float The ground temperature in degrees C (default 25.0). - pressure: float + pressure : float The ground pressure in hPa (default: 1020.0). + licel_timezone : str + String describing the timezone according to the tz database. Returns ------- @@ -39,7 +43,8 @@ extra_netcdf_parameters = custom_netcdf_parameters def __init__(self, plus45_files=None, minus45_files=None): - super(CustomLidarMeasurement, self).__init__(plus45_files, minus45_files, use_id_as_name) + super(CustomLidarMeasurement, self).__init__(plus45_files, minus45_files, use_id_as_name, + licel_timezone=licel_timezone) def set_PT(self): ''' Sets the pressure and temperature at station level. This is used if molecular_calc parameter is @@ -89,6 +94,9 @@ parser.add_argument("-p", "--pressure", type=float, help="The pressure (in hPa) at lidar level, required if using US Standard atmosphere", default="1020") + parser.add_argument('--licel_timezone', help="String describing the timezone according to the tz database.", + default="UTC", dest="licel_timezone", + ) # Verbosity settings from http://stackoverflow.com/a/20663028 parser.add_argument('-d', '--debug', help="Print dubuging information.", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO, @@ -126,7 +134,7 @@ logger.info("Reading {0} files from {1}".format(len(minus45_files), args.minus45_string)) CustomLidarMeasurement = create_custom_class(args.parameter_file, args.id_as_name, args.temperature, - args.pressure) + args.pressure, args.licel_timezone) measurement = CustomLidarMeasurement(plus45_files, minus45_files) diff -r 6511136e6937 -r 2f3f75e5b99e setup.py --- a/setup.py Thu Dec 14 13:24:15 2017 +0200 +++ b/setup.py Thu Dec 14 13:24:52 2017 +0200 @@ -52,6 +52,7 @@ "numpy", "matplotlib", "sphinx", + "pytz", ], entry_points={ 'console_scripts': ['licel2scc = atmospheric_lidar.scripts.licel2scc:main',