# HG changeset patch # User Iannis # Date 1518784794 -7200 # Node ID a1788b77f33b9fe59b085122f5c283770e34dd1c # Parent 0c7d3ef1dd34a1ffa0ca4a981b31bf7a64003ab7 Attempt to add cloudmasking options. diff -r 0c7d3ef1dd34 -r a1788b77f33b atmospheric_lidar/scripts/licel2scc.py --- a/atmospheric_lidar/scripts/licel2scc.py Fri Feb 16 08:43:23 2018 +0200 +++ b/atmospheric_lidar/scripts/licel2scc.py Fri Feb 16 14:39:54 2018 +0200 @@ -74,11 +74,79 @@ return settings +def get_cloud_free_files(CustomLidarMeasurement, files, args): + logger.warning("Starting cloud mask procedure. This is an experimental feature.") + + try: + from cloudmask import cloudmask # Import here until we setup a proper installation procedure + except ImportError: + logger.error("Cloud mask module could not be loaded. Please install manually.") + sys.exit(1) + + measurement = CustomLidarMeasurement(files) + channel = measurement.channels[args.cloudmask_channel] + cloud_mask = cloudmask.CloudMaskRaw(channel) + idxs = cloud_mask.cloud_free_periods(args.cloudfree_period, args.cloud_search_height) + + if len(idxs) == 0: # If no cloud-free period found + logger.info('No cloud free period found. Nothing converted.') + sys.exit(1) + + logger.info("{0} cloud free period(s) found.".format(len(idxs))) + file_list = [] + for idx_min, idx_max in idxs: + current_files = measurement.files[idx_min:idx_max] + file_list.append(current_files) + + return file_list + + +def get_corrected_measurement_id(args, n): + """ Correct the provided measurement id, in case of multiple cloud-free periods. """ + if args.measurement_id is not None: + order = float(args.measurement_id[-2:]) + new_no = order + n + measurement_id = args.measurement_id[:-2] + str(new_no) + measurement_no = args.measurement_number # The same + else: + measurement_no = str(float(args.measurement_number) + n).zfill(2) + measurement_id = None + + return measurement_id, measurement_no + + +def convert_to_scc(CustomLidarMeasurement, files, dark_pattern, measurement_id, measurement_number): + """ Convert files to SCC. """ + measurement = CustomLidarMeasurement(files) + # Get a list of files containing dark measurements + if dark_pattern != "": + dark_files = glob.glob(dark_pattern) + + if dark_files: + logger.debug("Using %s as dark measurements files!" % ', '.join(dark_files)) + measurement.dark_measurement = CustomLidarMeasurement(dark_files) + else: + logger.warning( + 'No dark measurement files found when searching for %s. Will not use any dark measurements.' % dark_pattern) + try: + measurement = measurement.subset_by_scc_channels() + except ValueError as err: + logger.error(err) + sys.exit(1) + + # Save the netcdf + logger.info("Saving netcdf") + measurement.set_measurement_id(measurement_id, measurement_number) + measurement.save_as_SCC_netcdf() + logger.info("Created file %s" % measurement.scc_filename) + + def main(): # Define the command line argument parser = argparse.ArgumentParser(description="A program to convert Licel binary files to the SCC NetCDF format.") parser.add_argument("parameter_file", help="The path to a parameter file linking licel and SCC channels.") - parser.add_argument("files", help="Location of licel files. Use relative path and filename wildcards. (default './*.*')", + parser.add_argument("files", + help="Location of licel files. Use relative path and filename wildcards. (default './*.*')", default="./*.*") parser.add_argument("-i", '--id_as_name', help="Use transient digitizer ids as channel names, instead of descriptive names", @@ -93,12 +161,23 @@ parser.add_argument("-p", "--pressure", type=float, help="The pressure (in hPa) at lidar level, required if using US Standard atmosphere", default="1020") - parser.add_argument('-D', '--dark_measurements', help="Location of files containing dark measurements. Use relative path and filename wildcars, see 'files' parameter for example.", + parser.add_argument('-D', '--dark_measurements', + help="Location of files containing dark measurements. Use relative path and filename wildcars, see 'files' parameter for example.", default="", dest="dark_files" - ) + ) parser.add_argument('--licel_timezone', help="String describing the timezone according to the tz database.", default="UTC", dest="licel_timezone", - ) + ) + parser.add_argument('--cloudmask', help="Experimental feature to automatically cloud mask measurements", + default=False, action='store_true', + ) + parser.add_argument('--cloudmask_channel', help="Name of channel to apply the cloud mask.") + parser.add_argument('--cloudfree_period', type=float, help="Duration (in min) of cloud-free periods", + default="30", + ) + parser.add_argument('--cloud_search_height', type=float, help="Maximum altitude (in m) to check for clouds.", + default="12000", + ) # Verbosity settings from http://stackoverflow.com/a/20663028 parser.add_argument('-d', '--debug', help="Print dubuging information.", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO, @@ -133,27 +212,12 @@ logger.info("Found {0} files matching {1}".format(len(files), os.path.abspath(args.files))) CustomLidarMeasurement = create_custom_class(args.parameter_file, args.id_as_name, args.temperature, args.pressure, args.licel_timezone) - measurement = CustomLidarMeasurement(files) - # Get a list of files containing dark measurements - if args.dark_files != "": - dark_files = glob.glob(args.dark_files) - - if dark_files: - logger.debug("Using %s as dark measurements files!" % ', '.join(dark_files)) - measurement.dark_measurement = CustomLidarMeasurement(dark_files) - else: - logger.warning('No dark measurement files found when searching for %s. Will not use any dark measurements.' % args.dark_files) + if args.cloudmask: + file_lists = get_cloud_free_files(CustomLidarMeasurement, files, args) - try: - measurement = measurement.subset_by_scc_channels() - except ValueError as err: - logger.error(err) - sys.exit(1) - - # Save the netcdf - logger.info("Saving netcdf") - measurement.set_measurement_id(args.measurement_id, args.measurement_number) - measurement.save_as_SCC_netcdf() - logger.info("Created file %s" % measurement.scc_filename) - + for n, files in enumerate(file_lists): + measurement_id, measurement_no = get_corrected_measurement_id(args, n) + convert_to_scc(CustomLidarMeasurement, files, args.dark_files, measurement_id, measurement_no) + else: + convert_to_scc(CustomLidarMeasurement, files, args.dark_files, args.measurement_id, args.measurement_number)