Mercurial > public > atmospheric_lidar / file comparison
comparison: atmospheric_lidar/scripts/licel2qa.py
atmospheric_lidar/scripts/licel2qa.py
- changeset 156
- 1e18b2a416ad
- parent 155
- 4a596849c721
- child 157
- 03b470b0a05f
equal
deleted
inserted
replaced
| 155:4a596849c721 | 156:1e18b2a416ad |
|---|---|
| 1 """ Command line tool to convert Licel binary files to EARLINET telecover files. | |
| 2 """ | |
| 3 import argparse | |
| 4 import glob | |
| 5 import importlib | |
| 6 import logging | |
| 7 import os | |
| 8 import sys | |
| 9 | |
| 10 from matplotlib import pyplot as plt | |
| 11 import yaml | |
| 12 | |
| 13 from ..licel import LicelLidarMeasurement | |
| 14 from ..__init__ import __version__ | |
| 15 | |
| 16 logger = logging.getLogger(__name__) | |
| 17 | |
| 18 | |
| 19 class TelecoverMeasurement(LicelLidarMeasurement): | |
| 20 | |
| 21 def __init__(self, file_list, use_id_as_name, licel_timezone, telecover_settings): | |
| 22 self.telecover_settings = telecover_settings | |
| 23 super(TelecoverMeasurement, self).__init__(file_list, use_id_as_name, licel_timezone=licel_timezone) | |
| 24 | |
| 25 | |
| 26 def create_custom_class(custom_netcdf_parameter_path, use_id_as_name=False, temperature=25., pressure=1020., | |
| 27 licel_timezone='UTC'): | |
| 28 """ This funtion creates a custom LicelLidarMeasurement subclass, | |
| 29 based on the input provided by the users. | |
| 30 | |
| 31 Parameters | |
| 32 ---------- | |
| 33 custom_netcdf_parameter_path : str | |
| 34 The path to the custom channels parameters. | |
| 35 use_id_as_name : bool | |
| 36 Defines if channels names are descriptive or transient digitizer IDs. | |
| 37 temperature : float | |
| 38 The ground temperature in degrees C (default 25.0). | |
| 39 pressure : float | |
| 40 The ground pressure in hPa (default: 1020.0). | |
| 41 licel_timezone : str | |
| 42 String describing the timezone according to the tz database. | |
| 43 | |
| 44 Returns | |
| 45 ------- | |
| 46 CustomLidarMeasurement: | |
| 47 A custom sub-class of LicelLidarMeasurement | |
| 48 """ | |
| 49 logger.debug('Reading parameter files: %s' % custom_netcdf_parameter_path) | |
| 50 custom_netcdf_parameters = read_settings_file(custom_netcdf_parameter_path) | |
| 51 | |
| 52 class CustomLidarMeasurement(LicelLidarMeasurement): | |
| 53 extra_netcdf_parameters = custom_netcdf_parameters | |
| 54 | |
| 55 def __init__(self, file_list=None): | |
| 56 super(CustomLidarMeasurement, self).__init__(file_list, use_id_as_name, licel_timezone=licel_timezone) | |
| 57 | |
| 58 def set_PT(self): | |
| 59 ''' Sets the pressure and temperature at station level. This is used if molecular_calc parameter is | |
| 60 set to 0 (i.e. use US Standard atmosphere). | |
| 61 | |
| 62 The results are stored in the info dictionary. | |
| 63 ''' | |
| 64 | |
| 65 self.info['Temperature'] = temperature | |
| 66 self.info['Pressure'] = pressure | |
| 67 | |
| 68 return CustomLidarMeasurement | |
| 69 | |
| 70 | |
| 71 def read_settings_file(settings_path): | |
| 72 """ Read the settings file. | |
| 73 | |
| 74 The file should contain python code.""" | |
| 75 if not os.path.isfile(settings_path): | |
| 76 logging.error("The provided settings path does not correspond to a file.") | |
| 77 sys.exit(1) | |
| 78 | |
| 79 dirname, basename = os.path.split(settings_path) | |
| 80 sys.path.append(dirname) | |
| 81 | |
| 82 module_name, _ = os.path.splitext(basename) | |
| 83 settings = importlib.import_module(module_name) | |
| 84 return settings | |
| 85 | |
| 86 | |
| 87 def read_cloudmask_settings_file(settings_file_path): | |
| 88 """ Read the configuration file. | |
| 89 | |
| 90 The file should be in YAML syntax.""" | |
| 91 | |
| 92 if not os.path.isfile(settings_file_path): | |
| 93 logging.error("Wrong path for cloudmask settings file (%s)" % settings_file_path) | |
| 94 sys.exit(1) | |
| 95 | |
| 96 with open(settings_file_path) as yaml_file: | |
| 97 try: | |
| 98 settings = yaml.load(yaml_file) | |
| 99 logging.debug("Read cloudmask settings file(%s)" % settings_file_path) | |
| 100 except: | |
| 101 logging.error("Could not parse YAML file (%s)" % settings_file_path) | |
| 102 sys.exit(1) | |
| 103 | |
| 104 return settings | |
| 105 | |
| 106 | |
| 107 def get_cloud_free_files(LidarMeasurementClass, files, settings): | |
| 108 """ Find cloud free periods in the given files. | |
| 109 | |
| 110 Depending on the provided settings, it could create plots of cloud mask and | |
| 111 selected cloud-free periods. | |
| 112 | |
| 113 Parameters | |
| 114 ---------- | |
| 115 LidarMeasurementClass : class | |
| 116 Class used to read the files. | |
| 117 files : list | |
| 118 A list of raw licel file paths. | |
| 119 settings : dict | |
| 120 A dictionary of cloud masking settings. | |
| 121 | |
| 122 Returns | |
| 123 ------- | |
| 124 file_list : list of lists | |
| 125 A list of lists containing paths to cloud-free files. | |
| 126 """ | |
| 127 logger.warning("Starting cloud mask procedure. This is an experimental feature.") | |
| 128 | |
| 129 try: | |
| 130 from cloudmask import cloudmask # Import here until we setup a proper installation procedure | |
| 131 except ImportError: | |
| 132 logger.error("Cloud mask module could not be loaded. Please install manually.") | |
| 133 sys.exit(1) | |
| 134 | |
| 135 measurement = LidarMeasurementClass(files) | |
| 136 channel = measurement.channels[settings['channel']] | |
| 137 cloud_mask = cloudmask.CloudMaskRaw(channel) | |
| 138 | |
| 139 idxs = cloud_mask.cloud_free_periods(settings['cloudfree_period_min'], | |
| 140 settings['file_duration_max'], | |
| 141 settings['max_cloud_height']) | |
| 142 | |
| 143 logger.debug('Cloud free indices: {0}'.format(idxs)) | |
| 144 | |
| 145 if len(idxs) == 0: # If no cloud-free period found | |
| 146 logger.info('No cloud free period found. Nothing converted.') | |
| 147 sys.exit(1) | |
| 148 | |
| 149 logger.info("{0} cloud free period(s) found.".format(len(idxs))) | |
| 150 | |
| 151 if settings['plot']: | |
| 152 # Plot cloud free periods | |
| 153 cloudfree_filename = "cloudfree_{0}_{1}_{2}.png".format(channel.wavelength, | |
| 154 channel.start_time.strftime('%Y%m%d_%H%M%S'), | |
| 155 channel.stop_time.strftime('%Y%m%d_%H%M%S')) | |
| 156 cloudfree_path = os.path.join(settings['plot_directory'], cloudfree_filename) | |
| 157 fig, _ = cloud_mask.plot_cloudfree(idxs) | |
| 158 | |
| 159 plt.savefig(cloudfree_path) | |
| 160 plt.close() | |
| 161 | |
| 162 # Plot cloud mask | |
| 163 cloudmask_filename = "cloudmask_{0}_{1}_{2}.png".format(channel.wavelength, | |
| 164 channel.start_time.strftime('%Y%m%d_%H%M%S'), | |
| 165 channel.stop_time.strftime('%Y%m%d_%H%M%S')) | |
| 166 cloudmask_path = os.path.join(settings['plot_directory'], cloudmask_filename) | |
| 167 | |
| 168 fig, _ = cloud_mask.plot_mask() | |
| 169 | |
| 170 plt.savefig(cloudmask_path) | |
| 171 plt.close() | |
| 172 | |
| 173 file_list = [] | |
| 174 for idx_min, idx_max in idxs: | |
| 175 current_files = measurement.files[idx_min:idx_max] | |
| 176 file_list.append(current_files) | |
| 177 | |
| 178 return file_list | |
| 179 | |
| 180 | |
| 181 def get_corrected_measurement_id(args, n): | |
| 182 """ Correct the provided measurement id, in case of multiple cloud-free periods. """ | |
| 183 if args.measurement_id is not None: | |
| 184 order = int(args.measurement_id[-2:]) | |
| 185 new_no = order + n | |
| 186 measurement_id = args.measurement_id[:-2] + str(new_no) | |
| 187 measurement_no = args.measurement_number # The same | |
| 188 else: | |
| 189 measurement_no = str(int(args.measurement_number) + n).zfill(2) | |
| 190 measurement_id = None | |
| 191 | |
| 192 return measurement_id, measurement_no | |
| 193 | |
| 194 | |
| 195 def convert_to_scc(CustomLidarMeasurement, files, dark_pattern, measurement_id, measurement_number): | |
| 196 """ Convert files to SCC. """ | |
| 197 measurement = CustomLidarMeasurement(files) | |
| 198 # Get a list of files containing dark measurements | |
| 199 if dark_pattern != "": | |
| 200 dark_files = glob.glob(dark_pattern) | |
| 201 | |
| 202 if dark_files: | |
| 203 logger.debug("Using %s as dark measurements files!" % ', '.join(dark_files)) | |
| 204 measurement.dark_measurement = CustomLidarMeasurement(dark_files) | |
| 205 else: | |
| 206 logger.warning( | |
| 207 'No dark measurement files found when searching for %s. Will not use any dark measurements.' % dark_pattern) | |
| 208 try: | |
| 209 measurement = measurement.subset_by_scc_channels() | |
| 210 except ValueError as err: | |
| 211 logger.error(err) | |
| 212 sys.exit(1) | |
| 213 | |
| 214 # Save the netcdf | |
| 215 logger.info("Saving netcdf") | |
| 216 measurement.set_measurement_id(measurement_id, measurement_number) | |
| 217 measurement.save_as_SCC_netcdf() | |
| 218 logger.info("Created file %s" % measurement.scc_filename) | |
| 219 | |
| 220 | |
| 221 def main(): | |
| 222 # Define the command line argument | |
| 223 parser = argparse.ArgumentParser( | |
| 224 description="A program to convert Licel binary files to EARLIENT telecover ASCII format") | |
| 225 parser.add_argument("settings_file", help="The path to a parameter YAML.") | |
| 226 parser.add_argument("files", | |
| 227 help="Location of licel files. Use relative path and filename wildcards. (default './*.*')", | |
| 228 default="./*.*") | |
| 229 parser.add_argument("-i", '--id_as_name', | |
| 230 help="Use transient digitizer ids as channel names, instead of descriptive names", | |
| 231 action="store_true") | |
| 232 parser.add_argument("-t", "--temperature", type=float, | |
| 233 help="The temperature (in C) at lidar level, required if using US Standard atmosphere", | |
| 234 default="25") | |
| 235 parser.add_argument("-p", "--pressure", type=float, | |
| 236 help="The pressure (in hPa) at lidar level, required if using US Standard atmosphere", | |
| 237 default="1020") | |
| 238 parser.add_argument('-D', '--dark_measurements', | |
| 239 help="Location of files containing dark measurements. Use relative path and filename wildcars, see 'files' parameter for example.", | |
| 240 default="", dest="dark_files" | |
| 241 ) | |
| 242 parser.add_argument('--licel_timezone', help="String describing the timezone according to the tz database.", | |
| 243 default="UTC", dest="licel_timezone", | |
| 244 ) | |
| 245 | |
| 246 # Verbosity settings from http://stackoverflow.com/a/20663028 | |
| 247 parser.add_argument('-d', '--debug', help="Print dubuging information.", action="store_const", | |
| 248 dest="loglevel", const=logging.DEBUG, default=logging.INFO, | |
| 249 ) | |
| 250 parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const", | |
| 251 dest="loglevel", const=logging.WARNING | |
| 252 ) | |
| 253 parser.add_argument('--version', help="Show current version.", action='store_true') | |
| 254 | |
| 255 args = parser.parse_args() | |
| 256 | |
| 257 # Get the logger with the appropriate level | |
| 258 logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel) | |
| 259 logger = logging.getLogger(__name__) | |
| 260 | |
| 261 # Check for version | |
| 262 if args.version: | |
| 263 print("Version: %s" % __version__) | |
| 264 sys.exit(0) | |
| 265 | |
| 266 # Get a list of files to process | |
| 267 files = glob.glob(args.files) | |
| 268 | |
| 269 # If not files found, exit | |
| 270 if len(files) == 0: | |
| 271 logger.error("No files found when searching for %s." % args.files) | |
| 272 sys.exit(1) | |
| 273 | |
| 274 # If everything OK, proceed | |
| 275 logger.info("Found {0} files matching {1}".format(len(files), os.path.abspath(args.files))) | |
| 276 | |
| 277 CustomLidarMeasurement = create_custom_class(args.parameter_file, args.id_as_name, args.temperature, | |
| 278 args.pressure, args.licel_timezone) | |
| 279 | |
| 280 convert_to_telecover(CustomLidarMeasurement, files, args.dark_files, args.measurement_id, args.measurement_number) |
