Mercurial > public > atmospheric_lidar / changeset

--- a/atmospheric_lidar/scripts/licel2scc.py	Fri Feb 16 08:43:23 2018 +0200
+++ b/atmospheric_lidar/scripts/licel2scc.py	Fri Feb 16 14:39:54 2018 +0200
@@ -74,11 +74,79 @@
     return settings
 
 
+def get_cloud_free_files(CustomLidarMeasurement, files, args):
+    logger.warning("Starting cloud mask procedure. This is an experimental feature.")
+
+    try:
+        from cloudmask import cloudmask  # Import here until we setup a proper installation procedure
+    except ImportError:
+        logger.error("Cloud mask module could not be loaded. Please install manually.")
+        sys.exit(1)
+
+    measurement = CustomLidarMeasurement(files)
+    channel = measurement.channels[args.cloudmask_channel]
+    cloud_mask = cloudmask.CloudMaskRaw(channel)
+    idxs = cloud_mask.cloud_free_periods(args.cloudfree_period, args.cloud_search_height)
+
+    if len(idxs) == 0:  # If no cloud-free period found
+        logger.info('No cloud free period found. Nothing converted.')
+        sys.exit(1)
+
+    logger.info("{0} cloud free period(s) found.".format(len(idxs)))
+    file_list = []
+    for idx_min, idx_max in idxs:
+        current_files = measurement.files[idx_min:idx_max]
+        file_list.append(current_files)
+
+    return file_list
+
+
+def get_corrected_measurement_id(args, n):
+    """ Correct the provided measurement id, in case of multiple cloud-free periods. """
+    if args.measurement_id is not None:
+        order = float(args.measurement_id[-2:])
+        new_no = order + n
+        measurement_id = args.measurement_id[:-2] + str(new_no)
+        measurement_no = args.measurement_number  # The same
+    else:
+        measurement_no =  str(float(args.measurement_number) + n).zfill(2)
+        measurement_id = None
+
+    return measurement_id, measurement_no
+
+
+def convert_to_scc(CustomLidarMeasurement, files, dark_pattern, measurement_id, measurement_number):
+    """ Convert files to SCC. """
+    measurement = CustomLidarMeasurement(files)
+    # Get a list of files containing dark measurements
+    if dark_pattern != "":
+        dark_files = glob.glob(dark_pattern)
+
+        if dark_files:
+            logger.debug("Using %s as dark measurements files!" % ', '.join(dark_files))
+            measurement.dark_measurement = CustomLidarMeasurement(dark_files)
+        else:
+            logger.warning(
+                'No dark measurement files found when searching for %s. Will not use any dark measurements.' % dark_pattern)
+    try:
+        measurement = measurement.subset_by_scc_channels()
+    except ValueError as err:
+        logger.error(err)
+        sys.exit(1)
+
+    # Save the netcdf
+    logger.info("Saving netcdf")
+    measurement.set_measurement_id(measurement_id, measurement_number)
+    measurement.save_as_SCC_netcdf()
+    logger.info("Created file %s" % measurement.scc_filename)
+
+
 def main():
     # Define the command line argument
     parser = argparse.ArgumentParser(description="A program to convert Licel binary files to the SCC NetCDF format.")
     parser.add_argument("parameter_file", help="The path to a parameter file linking licel and SCC channels.")
-    parser.add_argument("files", help="Location of licel files. Use relative path and filename wildcards. (default './*.*')",
+    parser.add_argument("files",
+                        help="Location of licel files. Use relative path and filename wildcards. (default './*.*')",
                         default="./*.*")
     parser.add_argument("-i", '--id_as_name',
                         help="Use transient digitizer ids as channel names, instead of descriptive names",
@@ -93,12 +161,23 @@
     parser.add_argument("-p", "--pressure", type=float,
                         help="The pressure (in hPa) at lidar level, required if using US Standard atmosphere",
                         default="1020")
-    parser.add_argument('-D', '--dark_measurements', help="Location of files containing dark measurements. Use relative path and filename wildcars, see 'files' parameter for example.",
+    parser.add_argument('-D', '--dark_measurements',
+                        help="Location of files containing dark measurements. Use relative path and filename wildcars, see 'files' parameter for example.",
                         default="", dest="dark_files"
-                       )
+                        )
     parser.add_argument('--licel_timezone', help="String describing the timezone according to the tz database.",
                         default="UTC", dest="licel_timezone",
-                       )
+                        )
+    parser.add_argument('--cloudmask', help="Experimental feature to automatically cloud mask measurements",
+                        default=False, action='store_true',
+                        )
+    parser.add_argument('--cloudmask_channel', help="Name of channel to apply the cloud mask.")
+    parser.add_argument('--cloudfree_period', type=float, help="Duration (in min) of cloud-free periods",
+                        default="30",
+                        )
+    parser.add_argument('--cloud_search_height', type=float, help="Maximum altitude (in m) to check for clouds.",
+                        default="12000",
+                        )
     # Verbosity settings from http://stackoverflow.com/a/20663028
     parser.add_argument('-d', '--debug', help="Print dubuging information.", action="store_const",
                         dest="loglevel", const=logging.DEBUG, default=logging.INFO,
@@ -133,27 +212,12 @@
     logger.info("Found {0} files matching {1}".format(len(files), os.path.abspath(args.files)))
     CustomLidarMeasurement = create_custom_class(args.parameter_file, args.id_as_name, args.temperature,
                                                  args.pressure, args.licel_timezone)
-    measurement = CustomLidarMeasurement(files)
 
-    # Get a list of files containing dark measurements
-    if args.dark_files != "":
-        dark_files = glob.glob(args.dark_files)
-
-        if dark_files:
-            logger.debug("Using %s as dark measurements files!" % ', '.join(dark_files))
-            measurement.dark_measurement = CustomLidarMeasurement(dark_files)
-        else:
-            logger.warning('No dark measurement files found when searching for %s. Will not use any dark measurements.' % args.dark_files)
+    if args.cloudmask:
+        file_lists = get_cloud_free_files(CustomLidarMeasurement, files, args)
 
-    try:
-        measurement = measurement.subset_by_scc_channels()
-    except ValueError as err:
-        logger.error(err)
-        sys.exit(1)
-
-    # Save the netcdf
-    logger.info("Saving netcdf")
-    measurement.set_measurement_id(args.measurement_id, args.measurement_number)
-    measurement.save_as_SCC_netcdf()
-    logger.info("Created file %s" % measurement.scc_filename)
-
+        for n, files in enumerate(file_lists):
+            measurement_id, measurement_no = get_corrected_measurement_id(args, n)
+            convert_to_scc(CustomLidarMeasurement, files, args.dark_files, measurement_id, measurement_no)
+    else:
+        convert_to_scc(CustomLidarMeasurement, files, args.dark_files, args.measurement_id, args.measurement_number)