Mercurial > public > atmospheric_lidar / file comparison
comparison: atmospheric_lidar/scripts/licel2scc.py
atmospheric_lidar/scripts/licel2scc.py
- changeset 124
- a1788b77f33b
- parent 121
- 4522191fe936
- child 126
- 4984793c500a
equal
deleted
inserted
replaced
| 123:0c7d3ef1dd34 | 124:a1788b77f33b |
|---|---|
| 72 module_name, _ = os.path.splitext(basename) | 72 module_name, _ = os.path.splitext(basename) |
| 73 settings = importlib.import_module(module_name) | 73 settings = importlib.import_module(module_name) |
| 74 return settings | 74 return settings |
| 75 | 75 |
| 76 | 76 |
| 77 def get_cloud_free_files(CustomLidarMeasurement, files, args): | |
| 78 logger.warning("Starting cloud mask procedure. This is an experimental feature.") | |
| 79 | |
| 80 try: | |
| 81 from cloudmask import cloudmask # Import here until we setup a proper installation procedure | |
| 82 except ImportError: | |
| 83 logger.error("Cloud mask module could not be loaded. Please install manually.") | |
| 84 sys.exit(1) | |
| 85 | |
| 86 measurement = CustomLidarMeasurement(files) | |
| 87 channel = measurement.channels[args.cloudmask_channel] | |
| 88 cloud_mask = cloudmask.CloudMaskRaw(channel) | |
| 89 idxs = cloud_mask.cloud_free_periods(args.cloudfree_period, args.cloud_search_height) | |
| 90 | |
| 91 if len(idxs) == 0: # If no cloud-free period found | |
| 92 logger.info('No cloud free period found. Nothing converted.') | |
| 93 sys.exit(1) | |
| 94 | |
| 95 logger.info("{0} cloud free period(s) found.".format(len(idxs))) | |
| 96 file_list = [] | |
| 97 for idx_min, idx_max in idxs: | |
| 98 current_files = measurement.files[idx_min:idx_max] | |
| 99 file_list.append(current_files) | |
| 100 | |
| 101 return file_list | |
| 102 | |
| 103 | |
| 104 def get_corrected_measurement_id(args, n): | |
| 105 """ Correct the provided measurement id, in case of multiple cloud-free periods. """ | |
| 106 if args.measurement_id is not None: | |
| 107 order = float(args.measurement_id[-2:]) | |
| 108 new_no = order + n | |
| 109 measurement_id = args.measurement_id[:-2] + str(new_no) | |
| 110 measurement_no = args.measurement_number # The same | |
| 111 else: | |
| 112 measurement_no = str(float(args.measurement_number) + n).zfill(2) | |
| 113 measurement_id = None | |
| 114 | |
| 115 return measurement_id, measurement_no | |
| 116 | |
| 117 | |
| 118 def convert_to_scc(CustomLidarMeasurement, files, dark_pattern, measurement_id, measurement_number): | |
| 119 """ Convert files to SCC. """ | |
| 120 measurement = CustomLidarMeasurement(files) | |
| 121 # Get a list of files containing dark measurements | |
| 122 if dark_pattern != "": | |
| 123 dark_files = glob.glob(dark_pattern) | |
| 124 | |
| 125 if dark_files: | |
| 126 logger.debug("Using %s as dark measurements files!" % ', '.join(dark_files)) | |
| 127 measurement.dark_measurement = CustomLidarMeasurement(dark_files) | |
| 128 else: | |
| 129 logger.warning( | |
| 130 'No dark measurement files found when searching for %s. Will not use any dark measurements.' % dark_pattern) | |
| 131 try: | |
| 132 measurement = measurement.subset_by_scc_channels() | |
| 133 except ValueError as err: | |
| 134 logger.error(err) | |
| 135 sys.exit(1) | |
| 136 | |
| 137 # Save the netcdf | |
| 138 logger.info("Saving netcdf") | |
| 139 measurement.set_measurement_id(measurement_id, measurement_number) | |
| 140 measurement.save_as_SCC_netcdf() | |
| 141 logger.info("Created file %s" % measurement.scc_filename) | |
| 142 | |
| 143 | |
| 77 def main(): | 144 def main(): |
| 78 # Define the command line argument | 145 # Define the command line argument |
| 79 parser = argparse.ArgumentParser(description="A program to convert Licel binary files to the SCC NetCDF format.") | 146 parser = argparse.ArgumentParser(description="A program to convert Licel binary files to the SCC NetCDF format.") |
| 80 parser.add_argument("parameter_file", help="The path to a parameter file linking licel and SCC channels.") | 147 parser.add_argument("parameter_file", help="The path to a parameter file linking licel and SCC channels.") |
| 81 parser.add_argument("files", help="Location of licel files. Use relative path and filename wildcards. (default './*.*')", | 148 parser.add_argument("files", |
| 149 help="Location of licel files. Use relative path and filename wildcards. (default './*.*')", | |
| 82 default="./*.*") | 150 default="./*.*") |
| 83 parser.add_argument("-i", '--id_as_name', | 151 parser.add_argument("-i", '--id_as_name', |
| 84 help="Use transient digitizer ids as channel names, instead of descriptive names", | 152 help="Use transient digitizer ids as channel names, instead of descriptive names", |
| 85 action="store_true") | 153 action="store_true") |
| 86 parser.add_argument("-m", "--measurement_id", help="The new measurement id", default=None) | 154 parser.add_argument("-m", "--measurement_id", help="The new measurement id", default=None) |
| 91 help="The temperature (in C) at lidar level, required if using US Standard atmosphere", | 159 help="The temperature (in C) at lidar level, required if using US Standard atmosphere", |
| 92 default="25") | 160 default="25") |
| 93 parser.add_argument("-p", "--pressure", type=float, | 161 parser.add_argument("-p", "--pressure", type=float, |
| 94 help="The pressure (in hPa) at lidar level, required if using US Standard atmosphere", | 162 help="The pressure (in hPa) at lidar level, required if using US Standard atmosphere", |
| 95 default="1020") | 163 default="1020") |
| 96 parser.add_argument('-D', '--dark_measurements', help="Location of files containing dark measurements. Use relative path and filename wildcars, see 'files' parameter for example.", | 164 parser.add_argument('-D', '--dark_measurements', |
| 165 help="Location of files containing dark measurements. Use relative path and filename wildcars, see 'files' parameter for example.", | |
| 97 default="", dest="dark_files" | 166 default="", dest="dark_files" |
| 98 ) | 167 ) |
| 99 parser.add_argument('--licel_timezone', help="String describing the timezone according to the tz database.", | 168 parser.add_argument('--licel_timezone', help="String describing the timezone according to the tz database.", |
| 100 default="UTC", dest="licel_timezone", | 169 default="UTC", dest="licel_timezone", |
| 101 ) | 170 ) |
| 171 parser.add_argument('--cloudmask', help="Experimental feature to automatically cloud mask measurements", | |
| 172 default=False, action='store_true', | |
| 173 ) | |
| 174 parser.add_argument('--cloudmask_channel', help="Name of channel to apply the cloud mask.") | |
| 175 parser.add_argument('--cloudfree_period', type=float, help="Duration (in min) of cloud-free periods", | |
| 176 default="30", | |
| 177 ) | |
| 178 parser.add_argument('--cloud_search_height', type=float, help="Maximum altitude (in m) to check for clouds.", | |
| 179 default="12000", | |
| 180 ) | |
| 102 # Verbosity settings from http://stackoverflow.com/a/20663028 | 181 # Verbosity settings from http://stackoverflow.com/a/20663028 |
| 103 parser.add_argument('-d', '--debug', help="Print dubuging information.", action="store_const", | 182 parser.add_argument('-d', '--debug', help="Print dubuging information.", action="store_const", |
| 104 dest="loglevel", const=logging.DEBUG, default=logging.INFO, | 183 dest="loglevel", const=logging.DEBUG, default=logging.INFO, |
| 105 ) | 184 ) |
| 106 parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const", | 185 parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const", |
| 131 | 210 |
| 132 # If everything OK, proceed | 211 # If everything OK, proceed |
| 133 logger.info("Found {0} files matching {1}".format(len(files), os.path.abspath(args.files))) | 212 logger.info("Found {0} files matching {1}".format(len(files), os.path.abspath(args.files))) |
| 134 CustomLidarMeasurement = create_custom_class(args.parameter_file, args.id_as_name, args.temperature, | 213 CustomLidarMeasurement = create_custom_class(args.parameter_file, args.id_as_name, args.temperature, |
| 135 args.pressure, args.licel_timezone) | 214 args.pressure, args.licel_timezone) |
| 136 measurement = CustomLidarMeasurement(files) | 215 |
| 137 | 216 if args.cloudmask: |
| 138 # Get a list of files containing dark measurements | 217 file_lists = get_cloud_free_files(CustomLidarMeasurement, files, args) |
| 139 if args.dark_files != "": | 218 |
| 140 dark_files = glob.glob(args.dark_files) | 219 for n, files in enumerate(file_lists): |
| 141 | 220 measurement_id, measurement_no = get_corrected_measurement_id(args, n) |
| 142 if dark_files: | 221 convert_to_scc(CustomLidarMeasurement, files, args.dark_files, measurement_id, measurement_no) |
| 143 logger.debug("Using %s as dark measurements files!" % ', '.join(dark_files)) | 222 else: |
| 144 measurement.dark_measurement = CustomLidarMeasurement(dark_files) | 223 convert_to_scc(CustomLidarMeasurement, files, args.dark_files, args.measurement_id, args.measurement_number) |
| 145 else: | |
| 146 logger.warning('No dark measurement files found when searching for %s. Will not use any dark measurements.' % args.dark_files) | |
| 147 | |
| 148 try: | |
| 149 measurement = measurement.subset_by_scc_channels() | |
| 150 except ValueError as err: | |
| 151 logger.error(err) | |
| 152 sys.exit(1) | |
| 153 | |
| 154 # Save the netcdf | |
| 155 logger.info("Saving netcdf") | |
| 156 measurement.set_measurement_id(args.measurement_id, args.measurement_number) | |
| 157 measurement.save_as_SCC_netcdf() | |
| 158 logger.info("Created file %s" % measurement.scc_filename) | |
| 159 |
