# HG changeset patch # User Ioannis # Date 1487255742 -7200 # Node ID e1961b511b3d579d03130c5dd8b92a92f53ea81d # Parent 357138ffbeaf7ce0667495c6101a072aaab0a0b1 Added logging and collored logging through CLI. diff -r 357138ffbeaf -r e1961b511b3d atmospheric_lidar/__init__.py --- a/atmospheric_lidar/__init__.py Thu Feb 16 12:23:32 2017 +0200 +++ b/atmospheric_lidar/__init__.py Thu Feb 16 16:35:42 2017 +0200 @@ -1,1 +1,1 @@ -__version__ = '0.2.0' \ No newline at end of file +__version__ = '0.2.1' \ No newline at end of file diff -r 357138ffbeaf -r e1961b511b3d atmospheric_lidar/generic.py --- a/atmospheric_lidar/generic.py Thu Feb 16 12:23:32 2017 +0200 +++ b/atmospheric_lidar/generic.py Thu Feb 16 16:35:42 2017 +0200 @@ -1,13 +1,13 @@ # General imports import datetime from operator import itemgetter +import logging -# Science imports -import numpy as np import matplotlib as mpl -from matplotlib.ticker import ScalarFormatter +import netCDF4 as netcdf +import numpy as np from matplotlib import pyplot as plt -import netCDF4 as netcdf +from matplotlib.ticker import ScalarFormatter netcdf_format = 'NETCDF3_CLASSIC' # choose one of 'NETCDF3_CLASSIC', 'NETCDF3_64BIT', 'NETCDF4_CLASSIC' and 'NETCDF4' @@ -143,8 +143,15 @@ """ Subset the measurement based on the channels provided in the extra_netecdf_parameter file. """ - extra_channels = self.extra_netcdf_parameters.channel_parameters.keys() - return self.subset_by_channels(extra_channels) + scc_channels = self.extra_netcdf_parameters.channel_parameters.keys() + common_channels = list(set(scc_channels).intersection(self.channels.keys())) + + if not common_channels: + logging.debug("Config channels: %s." % ','.join(scc_channels)) + logging.debug("Licel channels: %s." % ','.join(self.channels.keys())) + raise ValueError('No common channels between licel and configuration files.') + + return self.subset_by_channels(common_channels) def subset_by_time(self, start_time, stop_time): @@ -324,8 +331,8 @@ else: raise ValueError('Channel parameters should define either "chanel_id" or "channel_string_ID".') - temp_v = f.createVariable(channel_var, variable_type, ('channels', )) - for channel, n in enumerate(channels): + temp_v = f.createVariable(channel_var, variable_type, ('channels',)) + for n, channel in enumerate(channels): temp_v[n] = params.channel_parameters[channel][channel_var] # Write the values of fixed channel parameters @@ -489,13 +496,13 @@ def _nearest_dt(self, dtime): margin = datetime.timedelta(seconds=300) if ((dtime + margin) < self.start_time) | ((dtime - margin) > self.stop_time): - print "Requested date not covered in this file" - raise + logging.error("Requested date not covered in this file") + raise ValueError("Requested date not covered in this file") dt = abs(self.time - np.array(dtime)) dtmin = min(dt) if dtmin > datetime.timedelta(seconds=60): - print "Nearest profile more than 60 seconds away. dt = %s." % dtmin + logging.warning("Nearest profile more than 60 seconds away. dt = %s." % dtmin) ind_t = np.where(dt == dtmin) ind_a = ind_t[0] if len(ind_a) > 1: diff -r 357138ffbeaf -r e1961b511b3d atmospheric_lidar/licel.py --- a/atmospheric_lidar/licel.py Thu Feb 16 12:23:32 2017 +0200 +++ b/atmospheric_lidar/licel.py Thu Feb 16 16:35:42 2017 +0200 @@ -1,8 +1,11 @@ +import datetime +import logging + import numpy as np -import datetime + +import musa_2009_netcdf_parameters +import musa_netcdf_parameters from generic import BaseLidarMeasurement, LidarChannel -import musa_netcdf_parameters -import musa_2009_netcdf_parameters licel_file_header_format = ['Filename', 'StartDate StartTime EndDate EndTime Altitude Longtitude Latitude ZenithAngle', @@ -11,11 +14,10 @@ licel_file_channel_format = 'Active AnalogPhoton LaserUsed DataPoints 1 HV BinW Wavelength d1 d2 d3 d4 ADCbits NShots Discriminator ID' - class LicelFile: def __init__(self, filename, use_id_as_name=False): self.filename = filename - self.use_id_as_name=use_id_as_name + self.use_id_as_name = use_id_as_name self.start_time = None self.stop_time = None self.import_file(filename) @@ -46,8 +48,9 @@ b = np.fromfile(f, 'b', 1) if (a[0] != 13) | (b[0] != 10): - print "Warning: No end of line found after record. File could be corrupt" - channel = LicelFileChannel(current_channel_info, raw_data, self.duration(), use_id_as_name=self.use_id_as_name) + logging.warning("No end of line found after record. File could be corrupt: %s" % filename) + channel = LicelFileChannel(current_channel_info, raw_data, self.duration(), + use_id_as_name=self.use_id_as_name) channel_name = channel.channel_name @@ -198,7 +201,7 @@ def import_file(self, filename): if filename in self.files: - print "File has been imported already:" + filename + logging.warning("File has been imported already: %s" % filename) else: current_file = LicelFile(filename, use_id_as_name=self.use_id_as_name) self.raw_info[current_file.filename] = current_file.raw_info @@ -269,9 +272,9 @@ list2 = f2.split() if len(list1) != len(list2): - print "Warning: Combining lists of different lengths." - print "List 1: %s" % list1 - print "List 2: %s" % list2 + logging.debug("Channel parameter list has different length from licel specifications.") + logging.debug("List 1: %s" % list1) + logging.debug("List 2: %s" % list2) combined = zip(list2, list1) combined = dict(combined) return combined diff -r 357138ffbeaf -r e1961b511b3d atmospheric_lidar/scripts/licel2scc.py --- a/atmospheric_lidar/scripts/licel2scc.py Thu Feb 16 12:23:32 2017 +0200 +++ b/atmospheric_lidar/scripts/licel2scc.py Thu Feb 16 16:35:42 2017 +0200 @@ -1,10 +1,13 @@ """ Command line tool to convert Licel binary files to SCC NetCDF format. """ +import argparse +import glob +import importlib +import logging import os import sys -import glob -import argparse -import importlib + +import coloredlogs from ..licel import LicelLidarMeasurement @@ -71,7 +74,8 @@ parser = argparse.ArgumentParser(description="A program to convert LICEL binary files to the SCC NetCDF format.") parser.add_argument("parameter_file", help="The path to a parameter file linking licel and SCC channels.") parser.add_argument("directory", nargs='?', help="Directory containing licel files (default '.')", default='.') - parser.add_argument("search_string", nargs='?', help="Search string for files in directory (default '*.*')", default="*.*") + parser.add_argument("search_string", nargs='?', help="Search string for files in directory (default '*.*')", + default="*.*") parser.add_argument("-i", '--id_as_name', help="Use transient digitizer ids as channel names, instead of descriptive names", action="store_true") @@ -85,25 +89,42 @@ parser.add_argument("-p", "--pressure", type=float, help="The pressure (in hPa) at lidar level, required if using US Standard atmosphere", default="1020") + # Verbosity settings from http://stackoverflow.com/a/20663028 + parser.add_argument('-d', '--debug', help="Print dubuging information.", action="store_const", + dest="loglevel", const=logging.DEBUG, default=logging.INFO, + ) + parser.add_argument('-s', '--silent', help="Show processing information.", action="store_const", + dest="loglevel", const=logging.WARNING + ) + args = parser.parse_args() + # Get the logger with the appropriate level + #logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel) + logger = logging.getLogger(__name__) + + coloredlogs.install(fmt='%(levelname)s: %(message)s', level=args.loglevel) + # Get a list of files to convert search_str = os.path.join(args.directory, args.search_string) files = glob.glob(search_str) - print(args.parameter_file) if files: # Read the files - print "Reading {0} files from {1}".format(len(files), args.directory) + logger.info("Reading {0} files from {1}".format(len(files), args.directory)) CustomLidarMeasurement = create_custom_class(args.parameter_file, args.id_as_name, args.temperature, args.pressure) measurement = CustomLidarMeasurement(files) - measurement = measurement.subset_by_scc_channels() - + try: + measurement = measurement.subset_by_scc_channels() + except ValueError as err: + logging.error(err) + sys.exit(1) # Save the netcdf - print "Saving netcdf." + logger.info("Saving netcdf") measurement.set_measurement_id(args.measurement_id, args.measurement_number) measurement.save_as_netcdf() - print "Created file ", measurement.scc_filename + logger.info("Created file %s" % measurement.scc_filename) else: - print "No files found when searching for ", search_str + logger.error("No files found when searching for %s." % search_str) + sys.exit(1) \ No newline at end of file diff -r 357138ffbeaf -r e1961b511b3d setup.py --- a/setup.py Thu Feb 16 12:23:32 2017 +0200 +++ b/setup.py Thu Feb 16 16:35:42 2017 +0200 @@ -7,7 +7,7 @@ # Read the long description from the readme file with open("readme.rst", "rb") as f: - long_descr = f.read().decode("utf-8") + long_description = f.read().decode("utf-8") # Read the version parameters from the __init__.py file. In this way @@ -33,8 +33,8 @@ setup(name='atmospheric_lidar_reader', packages=['atmospheric_lidar', 'atmospheric_lidar.scripts'], version=find_version("atmospheric_lidar", "__init__.py"), - description='Classes for reading raw atmospheric lidar data.', - long_description=long_descr, + description='Package for reading raw atmospheric lidar data.', + long_description=long_description, url='https://bitbucket.org/iannis_b/atmospheric-lidar/', author='Ioannis Binietoglou', author_email='ioannis@inoe.ro', @@ -53,6 +53,7 @@ "sphinx", "pytest", "netcdf4", + "coloredlogs", ], entry_points={ 'console_scripts': ['licel2scc = atmospheric_lidar.scripts.licel2scc:main'],