Mercurial > public > atmospheric_lidar / changeset

--- a/.hgignore	Sun Feb 26 22:30:13 2017 +0200
+++ b/.hgignore	Sun Feb 26 22:30:27 2017 +0200
@@ -8,3 +8,5 @@
 re:^atmospheric_lidar\.egg-info/
 *.nc
 re:^atmospheric_lidar_reader\.egg-info/
+re:^build/
+re:^dist/

--- a/atmospheric_lidar/__init__.py	Sun Feb 26 22:30:13 2017 +0200
+++ b/atmospheric_lidar/__init__.py	Sun Feb 26 22:30:27 2017 +0200
@@ -1,1 +1,1 @@
-__version__ = '0.2.0'
\ No newline at end of file
+__version__ = '0.2.8'
\ No newline at end of file

--- a/atmospheric_lidar/generic.py	Sun Feb 26 22:30:13 2017 +0200
+++ b/atmospheric_lidar/generic.py	Sun Feb 26 22:30:27 2017 +0200
@@ -1,13 +1,13 @@
 # General imports
 import datetime
 from operator import itemgetter
+import logging
 
-# Science imports
-import numpy as np
 import matplotlib as mpl
-from matplotlib.ticker import ScalarFormatter
+import netCDF4 as netcdf
+import numpy as np
 from matplotlib import pyplot as plt
-import netCDF4 as netcdf
+from matplotlib.ticker import ScalarFormatter
 
 netcdf_format = 'NETCDF3_CLASSIC'  # choose one of 'NETCDF3_CLASSIC', 'NETCDF3_64BIT', 'NETCDF4_CLASSIC' and 'NETCDF4'
 
@@ -143,8 +143,15 @@
         """
         Subset the measurement based on the channels provided in the extra_netecdf_parameter file.
         """
-        extra_channels = self.extra_netcdf_parameters.channel_parameters.keys()
-        return self.subset_by_channels(extra_channels)
+        scc_channels = self.extra_netcdf_parameters.channel_parameters.keys()
+        common_channels = list(set(scc_channels).intersection(self.channels.keys()))
+
+        if not common_channels:
+            logging.debug("Config channels: %s." % ','.join(scc_channels))
+            logging.debug("Licel channels: %s." % ','.join(self.channels.keys()))
+            raise ValueError('No common channels between licel and configuration files.')
+
+        return self.subset_by_channels(common_channels)
 
     def subset_by_time(self, start_time, stop_time):
 
@@ -317,15 +324,15 @@
         first_channel_keys = params.channel_parameters.items()[0][1].keys()
         if "channel_ID" in first_channel_keys:
             channel_var = 'channel_ID'
-            variable_type = 'd'
+            variable_type = 'i'
         elif "channel string ID" in first_channel_keys:
             channel_var = 'channel string ID'
             variable_type = str
         else:
             raise ValueError('Channel parameters should define either "chanel_id" or "channel_string_ID".')
 
-        temp_v = f.createVariable(channel_var, variable_type, ('channels', ))
-        for channel, n in enumerate(channels):
+        temp_v = f.createVariable(channel_var, variable_type, ('channels',))
+        for n, channel in enumerate(channels):
             temp_v[n] = params.channel_parameters[channel][channel_var]
 
         # Write the values of fixed channel parameters
@@ -489,13 +496,13 @@
     def _nearest_dt(self, dtime):
         margin = datetime.timedelta(seconds=300)
         if ((dtime + margin) < self.start_time) | ((dtime - margin) > self.stop_time):
-            print "Requested date not covered in this file"
-            raise
+            logging.error("Requested date not covered in this file")
+            raise ValueError("Requested date not covered in this file")
         dt = abs(self.time - np.array(dtime))
         dtmin = min(dt)
 
         if dtmin > datetime.timedelta(seconds=60):
-            print "Nearest profile more than 60 seconds away. dt = %s." % dtmin
+            logging.warning("Nearest profile more than 60 seconds away. dt = %s." % dtmin)
         ind_t = np.where(dt == dtmin)
         ind_a = ind_t[0]
         if len(ind_a) > 1:

--- a/atmospheric_lidar/licel.py	Sun Feb 26 22:30:13 2017 +0200
+++ b/atmospheric_lidar/licel.py	Sun Feb 26 22:30:27 2017 +0200
@@ -1,8 +1,11 @@
+import datetime
+import logging
+
 import numpy as np
-import datetime
+
+import musa_2009_netcdf_parameters
+import musa_netcdf_parameters
 from generic import BaseLidarMeasurement, LidarChannel
-import musa_netcdf_parameters
-import musa_2009_netcdf_parameters
 
 licel_file_header_format = ['Filename',
                             'StartDate StartTime EndDate EndTime Altitude Longtitude Latitude ZenithAngle',
@@ -11,11 +14,10 @@
 licel_file_channel_format = 'Active AnalogPhoton LaserUsed DataPoints 1 HV BinW Wavelength d1 d2 d3 d4 ADCbits NShots Discriminator ID'
 
 
-
 class LicelFile:
     def __init__(self, filename, use_id_as_name=False):
         self.filename = filename
-        self.use_id_as_name=use_id_as_name
+        self.use_id_as_name = use_id_as_name
         self.start_time = None
         self.stop_time = None
         self.import_file(filename)
@@ -46,8 +48,9 @@
                 b = np.fromfile(f, 'b', 1)
 
                 if (a[0] != 13) | (b[0] != 10):
-                    print "Warning: No end of line found after record. File could be corrupt"
-                channel = LicelFileChannel(current_channel_info, raw_data, self.duration(), use_id_as_name=self.use_id_as_name)
+                    logging.warning("No end of line found after record. File could be corrupt: %s" % filename)
+                channel = LicelFileChannel(current_channel_info, raw_data, self.duration(),
+                                           use_id_as_name=self.use_id_as_name)
 
                 channel_name = channel.channel_name
 
@@ -198,7 +201,7 @@
 
     def import_file(self, filename):
         if filename in self.files:
-            print "File has been imported already:" + filename
+            logging.warning("File has been imported already: %s" % filename)
         else:
             current_file = LicelFile(filename, use_id_as_name=self.use_id_as_name)
             self.raw_info[current_file.filename] = current_file.raw_info
@@ -269,9 +272,9 @@
     list2 = f2.split()
 
     if len(list1) != len(list2):
-        print "Warning: Combining lists of different lengths."
-        print "List 1: %s" % list1
-        print "List 2: %s" % list2
+        logging.debug("Channel parameter list has different length from licel specifications.")
+        logging.debug("List 1: %s" % list1)
+        logging.debug("List 2: %s" % list2)
     combined = zip(list2, list1)
     combined = dict(combined)
     return combined

--- a/atmospheric_lidar/rali_netcdf_parameters.py	Sun Feb 26 22:30:13 2017 +0200
+++ b/atmospheric_lidar/rali_netcdf_parameters.py	Sun Feb 26 22:30:27 2017 +0200
@@ -4,7 +4,8 @@
  'Molecular_Calc': 0,
  'Latitude_degrees_north': 44.348,
  'Longitude_degrees_east': 26.029,
- 'Altitude_meter_asl': 93.0} # This should be float
+ 'Altitude_meter_asl': 93.0,
+ 'Call sign': 'bu', } # This should be float
 
 channel_parameters = \
 {'01064.o_an': {'channel_ID': 89,

--- a/atmospheric_lidar/scripts/licel2scc.py	Sun Feb 26 22:30:13 2017 +0200
+++ b/atmospheric_lidar/scripts/licel2scc.py	Sun Feb 26 22:30:27 2017 +0200
@@ -1,10 +1,12 @@
 """ Command line tool to convert Licel binary files to SCC NetCDF format.
 """
+import argparse
+import glob
+import importlib
+import logging
 import os
 import sys
-import glob
-import argparse
-import importlib
+
 
 from ..licel import LicelLidarMeasurement
 
@@ -56,7 +58,8 @@
 
     The file should contain python code."""
     if not os.path.isfile(settings_path):
-        raise IOError("The provided settings path does not correspond to a file.")
+        logging.error("The provided settings path does not correspond to a file.")
+        sys.exit(1)
 
     dirname, basename = os.path.split(settings_path)
     sys.path.append(dirname)
@@ -68,10 +71,11 @@
 
 def main():
     # Define the command line argument
-    parser = argparse.ArgumentParser(description="A program to convert LICEL binary files to the SCC NetCDF format.")
+    parser = argparse.ArgumentParser(description="A program to convert Licel binary files to the SCC NetCDF format.")
     parser.add_argument("parameter_file", help="The path to a parameter file linking licel and SCC channels.")
     parser.add_argument("directory", nargs='?', help="Directory containing licel files (default '.')", default='.')
-    parser.add_argument("search_string", nargs='?', help="Search string for files in directory (default '*.*')", default="*.*")
+    parser.add_argument("search_string", nargs='?', help="Search string for files in directory (default '*.*')",
+                        default="*.*")
     parser.add_argument("-i", '--id_as_name',
                         help="Use transient digitizer ids as channel names, instead of descriptive names",
                         action="store_true")
@@ -85,24 +89,42 @@
     parser.add_argument("-p", "--pressure", type=float,
                         help="The pressure (in hPa) at lidar level, required if using US Standard atmosphere",
                         default="1020")
+    # Verbosity settings from http://stackoverflow.com/a/20663028
+    parser.add_argument('-d', '--debug', help="Print dubuging information.", action="store_const",
+                        dest="loglevel", const=logging.DEBUG, default=logging.INFO,
+                        )
+    parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const",
+                        dest="loglevel", const=logging.WARNING
+                        )
+
     args = parser.parse_args()
 
+    # Get the logger with the appropriate level
+    logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel)
+    logger = logging.getLogger(__name__)
+
+    #coloredlogs.install(fmt='%(levelname)s: %(message)s', level=args.loglevel)
+
     # Get a list of files to convert
     search_str = os.path.join(args.directory, args.search_string)
     files = glob.glob(search_str)
 
     if files:
         # Read the files
-        print "Reading {0} files from {1}".format(len(files), args.directory)
+        logger.info("Reading {0} files from {1}".format(len(files), args.directory))
         CustomLidarMeasurement = create_custom_class(args.parameter_file, args.id_as_name, args.temperature,
                                                      args.pressure)
         measurement = CustomLidarMeasurement(files)
-        measurement = measurement.subset_by_scc_channels()
-        
+        try:
+            measurement = measurement.subset_by_scc_channels()
+        except ValueError as err:
+            logging.error(err)
+            sys.exit(1)
         # Save the netcdf
-        print "Saving netcdf."
+        logger.info("Saving netcdf")
         measurement.set_measurement_id(args.measurement_id, args.measurement_number)
         measurement.save_as_netcdf()
-        print "Created file ", measurement.scc_filename
+        logger.info("Created file %s" % measurement.scc_filename)
     else:
-        print "No files found when searching for ", search_str
+        logger.error("No files found when searching for %s." % search_str)
+        sys.exit(1)
\ No newline at end of file

--- a/readme.rst	Sun Feb 26 22:30:13 2017 +0200
+++ b/readme.rst	Sun Feb 26 22:30:27 2017 +0200
@@ -1,82 +1,143 @@
-Basic instructions
-==================
+Overview
+========
 
-These classes can be used to handle lidar input data. They are still in a very initial stage. There are many features probably not working, but they work for some specific tasks. They work with Licel input files (and also with the Raymetrics modified format).
+This package provides utilities to handle raw (atmospheric) lidar input data.
+The main format supported are Licel binary files (including the Raymetrics modified format).
+
+The package provides a single command line tool, called licel2scc that can convert Licel binary files to the
+EARLINET's Single Calculus Chain NetCDF format.
 
 
-Set up
-------
+Using it as a Licel to SCC converter
+====================================
 
 Parameter file
-~~~~~~~~~~~~~~
-Before using the classes you need to setup some channel parameters, that are used when converting the lidar data to Single Calculus Chain format. 
+--------------
+Before converting Licel binary to SCC format, you need to create a file linking Licel channels to SCC channels.
+
+As an example, you can start by changing the file “cf_netcdf_parameters.py” that describe such
+parameters for the Clermont Ferrand  lidar.
+
+Command line interface
+----------------------
+The usage of the  ``licel2scc`` program is described bellow::
+
+    usage: licel2scc [-h] [-i] [-m MEASUREMENT_ID] [-n MEASUREMENT_NUMBER]
+                     [-t TEMPERATURE] [-p PRESSURE] [-d] [-s]
+                     parameter_file [directory] [search_string]
+
+    A program to convert LICEL binary files to the SCC NetCDF format.
+
+    positional arguments:
+      parameter_file        The path to a parameter file linking licel and SCC
+                            channels.
+      directory             Directory containing licel files (default '.')
+      search_string         Search string for files in directory (default '*.*')
 
-All the parameters are read from an external file stored in the same folder as the code. You can start by changing the file “cf_netcdf_parameters.py” that describe such parameters for the Clermont Ferrand  lidar. 
+    optional arguments:
+      -h, --help            show this help message and exit
+      -i, --id_as_name      Use transient digitizer ids as channel names, instead
+                            of descriptive names
+      -m MEASUREMENT_ID, --measurement_id MEASUREMENT_ID
+                            The new measurement id
+      -n MEASUREMENT_NUMBER, --measurement_number MEASUREMENT_NUMBER
+                            The measurement number for the date from 00 to 99.
+                            Used if no id is provided
+      -t TEMPERATURE, --temperature TEMPERATURE
+                            The temperature (in C) at lidar level, required if
+                            using US Standard atmosphere
+      -p PRESSURE, --pressure PRESSURE
+                            The pressure (in hPa) at lidar level, required if
+                            using US Standard atmosphere
+      -d, --debug           Print dubuging information.
+      -s, --silent          Show only warning and error messages.
 
+
+Usage in python code
+--------------------
 System class
 ~~~~~~~~~~~~
-The next thing you need to create a class that describes you system. This is very simple if your lidar data are in the Licel format, as you only need to specify the external file with the extra SCC parameters. You can use as an example the file “cf_raymetrics.py”:
+To read data from a system, you need create a class that describes you system.
+This is very simple if your lidar data are in the Licel format, as you only need to specify
+the external file with the extra SCC parameters. You can use as an example the file ``cf_netcdf_parameters.py``:
    
-    .. code:: python
-       
-       from licel import LicelLidarMeasurement
-       import cf_netcdf_parameters
-   
-       class CfLidarMeasurement(LicelLidarMeasurement):
-           extra_netcdf_parameters = cf_netcdf_parameters
+.. code-block:: python
+
+   from licel import LicelLidarMeasurement
+   import cf_netcdf_parameters
 
+   class CfLidarMeasurement(LicelLidarMeasurement):
+       extra_netcdf_parameters = cf_netcdf_parameters
+
+This code assumes that the ``cf_netcdf_parameters.py`` is in your python path.
 
 Using the class
----------------
+~~~~~~~~~~~~~~~
 
-Once you have made the above setup you can start using it. The best way to understand how it works is through an interactive shell (I suggest [ipython](http://ipython.org/)).  In the following example I use the cf_raymetrics setup:
+Once you have made the above setup you can start using it. The best way to understand how
+it works is through an interactive shell (I suggest [ipython](http://ipython.org/)).
+In the following example I use the cf_raymetrics setup:
    
-    .. code:: python
-    
-       import glob # This is needed to read a list of filenames
-       from lidar import cf_raymetrics #If you have saved the files in a directrory called “lidar”
+.. code-block:: python
 
-       # Go to the folder where you files are stored
-       cd /path/to/lidar/files
+   import glob  # This is needed to read a list of filenames
+   import cf_lidar
 
-       # Read the filenames
-       files  = glob.glob("*") # The * reads all the files in the folder.
+   # Go to the folder where you files are stored
+   cd /path/to/lidar/files
 
-       #Read the files
-       my_measurement = cf_raymetrics.CfLidarMeasurement(files)
+   # Read the filenames
+   files  = glob.glob("*") # The * reads all the files in the folder.
 
-       #Now the data have been read, and you have a measurement object to work with:
-       # See what channels are present
-       print my_measurement.channels
+   # Read the files
+   my_measurement = cf_lidar.CfLidarMeasurement(files)
 
-       # Quicklooks of all the channels
-       my_measurements.plot()
+   # Now the data have been read, and you have a measurement object to work with:
+   # See what channels are present
+   print(my_measurement.channels)
 
+   # Quicklooks of all the channels
+   my_measurements.plot()
 
 Converting to SCC format
-------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~
 
 There are some extra info you need to put in before converting to SCC format, "Measurement_ID", "Temperature", "Pressure":
    
-    .. code:: python
+.. code-block:: python
 
-       my_measurement.info["Measurement_ID"] = "20101229op00"
-       my_measurement.info["Temperature"] = "14"
-       my_measurement.info["Pressure"] = "1010"
+   my_measurement.info["Measurement_ID"] = "20101229op00"
+   my_measurement.info["Temperature"] = "14"
+   my_measurement.info["Pressure"] = "1010"
 
 You can use standard values of temperature and pressure by just calling:
    
-    .. code:: python
-    
-       my_measurement.get_PT() 
+.. code-block:: python
+
+    my_measurement.get_PT()
+
+You can specify the standard values by overriding your system's ``get_PT`` method:
+
+.. code-block:: python
 
-The standard values can be changed in generic.py. Search the  get_PT method and change of what is appropriate for your station. If you have an external source of temperature and pressure information (a meteorological station) you can automate this by overriding the get_PT method in your system"s class (in our example in the cf_raymetrics.py file).
+   from licel import LicelLidarMeasurement
+   import cf_netcdf_parameters
+
+   class CfLidarMeasurement(LicelLidarMeasurement):
+       extra_netcdf_parameters = cf_netcdf_parameters
+
+       def get_PT():
+           self.info['Temperature'] = 25.0
+           self.info['Pressure'] = 1020.0
+
+If you have an external source of temperature and pressure information (a meteorological station) you can automate
+this by reading the appropriate code in the ``get_PT`` method .
 
 
 After you have used this extra input, you save the file using this command:
 
-    .. code:: python
-       
-       my_measurement.save_as_netcdf("filename")
+.. code-block:: python
 
-where you change the filename to the filename you want to use.
\ No newline at end of file
+   my_measurement.save_as_netcdf("filename")
+
+where you change the output filename to the filename you want to use.
\ No newline at end of file

--- a/setup.py	Sun Feb 26 22:30:13 2017 +0200
+++ b/setup.py	Sun Feb 26 22:30:27 2017 +0200
@@ -7,7 +7,7 @@
 
 # Read the long description from the readme file
 with open("readme.rst", "rb") as f:
-    long_descr = f.read().decode("utf-8")
+    long_description = f.read().decode("utf-8")
 
 
 # Read the version parameters from the __init__.py file. In this way
@@ -30,11 +30,11 @@
 
 
 # Run setup
-setup(name='atmospheric_lidar_reader',
+setup(name='atmospheric_lidar',
       packages=['atmospheric_lidar', 'atmospheric_lidar.scripts'],
       version=find_version("atmospheric_lidar", "__init__.py"),
-      description='Classes for reading raw atmospheric lidar data.',
-      long_description=long_descr,
+      description='Package for reading raw atmospheric lidar data.',
+      long_description=long_description,
       url='https://bitbucket.org/iannis_b/atmospheric-lidar/',
       author='Ioannis Binietoglou',
       author_email='ioannis@inoe.ro',
@@ -48,11 +48,10 @@
       ],
       keywords='lidar aerosol licel SCC',
       install_requires=[
+          "netCDF4",
           "numpy",
           "matplotlib",
           "sphinx",
-          "pytest",
-          "netcdf4",
       ],
       entry_points={
           'console_scripts': ['licel2scc = atmospheric_lidar.scripts.licel2scc:main',