Mercurial > public > atmospheric_lidar / file comparison
comparison: atmospheric_lidar/scripts/licel2scc.py
atmospheric_lidar/scripts/licel2scc.py
- changeset 128
- 168da625489b
- parent 126
- 4984793c500a
- child 142
- b1cac5351db6
equal
deleted
inserted
replaced
| 127:6051a9bae77b | 128:168da625489b |
|---|---|
| 4 import glob | 4 import glob |
| 5 import importlib | 5 import importlib |
| 6 import logging | 6 import logging |
| 7 import os | 7 import os |
| 8 import sys | 8 import sys |
| 9 | |
| 10 from matplotlib import pyplot as plt | |
| 11 import yaml | |
| 9 | 12 |
| 10 from ..licel import LicelLidarMeasurement | 13 from ..licel import LicelLidarMeasurement |
| 11 from ..__init__ import __version__ | 14 from ..__init__ import __version__ |
| 12 | 15 |
| 13 logger = logging.getLogger(__name__) | 16 logger = logging.getLogger(__name__) |
| 72 module_name, _ = os.path.splitext(basename) | 75 module_name, _ = os.path.splitext(basename) |
| 73 settings = importlib.import_module(module_name) | 76 settings = importlib.import_module(module_name) |
| 74 return settings | 77 return settings |
| 75 | 78 |
| 76 | 79 |
| 77 def get_cloud_free_files(CustomLidarMeasurement, files, args): | 80 def read_cloudmask_settings_file(settings_file_path): |
| 81 """ Read the configuration file. | |
| 82 | |
| 83 The file should be in YAML syntax.""" | |
| 84 | |
| 85 if not os.path.isfile(settings_file_path): | |
| 86 logging.error("Wrong path for cloudmask settings file (%s)" % settings_file_path) | |
| 87 sys.exit(1) | |
| 88 | |
| 89 with open(settings_file_path) as yaml_file: | |
| 90 try: | |
| 91 settings = yaml.load(yaml_file) | |
| 92 logging.debug("Read cloudmask settings file(%s)" % settings_file_path) | |
| 93 except: | |
| 94 logging.error("Could not parse YAML file (%s)" % settings_file_path) | |
| 95 sys.exit(1) | |
| 96 | |
| 97 return settings | |
| 98 | |
| 99 | |
| 100 def get_cloud_free_files(CustomLidarMeasurement, files, settings): | |
| 78 logger.warning("Starting cloud mask procedure. This is an experimental feature.") | 101 logger.warning("Starting cloud mask procedure. This is an experimental feature.") |
| 79 | 102 |
| 80 try: | 103 try: |
| 81 from cloudmask import cloudmask # Import here until we setup a proper installation procedure | 104 from cloudmask import cloudmask # Import here until we setup a proper installation procedure |
| 82 except ImportError: | 105 except ImportError: |
| 83 logger.error("Cloud mask module could not be loaded. Please install manually.") | 106 logger.error("Cloud mask module could not be loaded. Please install manually.") |
| 84 sys.exit(1) | 107 sys.exit(1) |
| 85 | 108 |
| 86 measurement = CustomLidarMeasurement(files) | 109 measurement = CustomLidarMeasurement(files) |
| 87 channel = measurement.channels[args.cloudmask_channel] | 110 channel = measurement.channels[settings['channel']] |
| 88 cloud_mask = cloudmask.CloudMaskRaw(channel) | 111 cloud_mask = cloudmask.CloudMaskRaw(channel) |
| 89 idxs = cloud_mask.cloud_free_periods(args.cloudfree_period, args.cloud_search_height) | 112 |
| 113 idxs = cloud_mask.cloud_free_periods(settings['cloudfree_period_min'], | |
| 114 settings['file_duration_max'], | |
| 115 settings['max_cloud_height']) | |
| 116 | |
| 117 logger.debug('Cloud free indices: {0}'.format(idxs)) | |
| 90 | 118 |
| 91 if len(idxs) == 0: # If no cloud-free period found | 119 if len(idxs) == 0: # If no cloud-free period found |
| 92 logger.info('No cloud free period found. Nothing converted.') | 120 logger.info('No cloud free period found. Nothing converted.') |
| 93 sys.exit(1) | 121 sys.exit(1) |
| 94 | 122 |
| 95 logger.info("{0} cloud free period(s) found.".format(len(idxs))) | 123 logger.info("{0} cloud free period(s) found.".format(len(idxs))) |
| 124 | |
| 125 if settings['plot']: | |
| 126 output_filename = "cloudfree_{0}_{1}_{2}.png".format(channel.wavelength, | |
| 127 channel.start_time.strftime('%Y%m%d_%H%M%S'), | |
| 128 channel.stop_time.strftime('%Y%m%d_%H%M%S')) | |
| 129 output_path = os.path.join(settings['plot_directory'], output_filename) | |
| 130 fig, _ = cloud_mask.plot_cloudfree(idxs) | |
| 131 | |
| 132 plt.savefig(output_path) | |
| 133 | |
| 96 file_list = [] | 134 file_list = [] |
| 97 for idx_min, idx_max in idxs: | 135 for idx_min, idx_max in idxs: |
| 98 current_files = measurement.files[idx_min:idx_max] | 136 current_files = measurement.files[idx_min:idx_max] |
| 99 file_list.append(current_files) | 137 file_list.append(current_files) |
| 100 | 138 |
| 166 default="", dest="dark_files" | 204 default="", dest="dark_files" |
| 167 ) | 205 ) |
| 168 parser.add_argument('--licel_timezone', help="String describing the timezone according to the tz database.", | 206 parser.add_argument('--licel_timezone', help="String describing the timezone according to the tz database.", |
| 169 default="UTC", dest="licel_timezone", | 207 default="UTC", dest="licel_timezone", |
| 170 ) | 208 ) |
| 171 parser.add_argument('--cloudmask', help="Experimental feature to automatically cloud mask measurements", | 209 parser.add_argument('--cloudmask_settings', help="Experimental feature to automatically cloud mask measurements", |
| 172 default=False, action='store_true', | 210 default="") |
| 173 ) | 211 |
| 174 parser.add_argument('--cloudmask_channel', help="Name of channel to apply the cloud mask.") | |
| 175 parser.add_argument('--cloudfree_period', type=float, help="Duration (in min) of cloud-free periods", | |
| 176 default="30", | |
| 177 ) | |
| 178 parser.add_argument('--cloud_search_height', type=float, help="Maximum altitude (in m) to check for clouds.", | |
| 179 default="12000", | |
| 180 ) | |
| 181 # Verbosity settings from http://stackoverflow.com/a/20663028 | 212 # Verbosity settings from http://stackoverflow.com/a/20663028 |
| 182 parser.add_argument('-d', '--debug', help="Print dubuging information.", action="store_const", | 213 parser.add_argument('-d', '--debug', help="Print dubuging information.", action="store_const", |
| 183 dest="loglevel", const=logging.DEBUG, default=logging.INFO, | 214 dest="loglevel", const=logging.DEBUG, default=logging.INFO, |
| 184 ) | 215 ) |
| 185 parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const", | 216 parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const", |
| 211 # If everything OK, proceed | 242 # If everything OK, proceed |
| 212 logger.info("Found {0} files matching {1}".format(len(files), os.path.abspath(args.files))) | 243 logger.info("Found {0} files matching {1}".format(len(files), os.path.abspath(args.files))) |
| 213 CustomLidarMeasurement = create_custom_class(args.parameter_file, args.id_as_name, args.temperature, | 244 CustomLidarMeasurement = create_custom_class(args.parameter_file, args.id_as_name, args.temperature, |
| 214 args.pressure, args.licel_timezone) | 245 args.pressure, args.licel_timezone) |
| 215 | 246 |
| 216 if args.cloudmask: | 247 if args.cloudmask_settings: |
| 217 file_lists = get_cloud_free_files(CustomLidarMeasurement, files, args) | 248 cloudmask_settings = read_cloudmask_settings_file(args.cloudmask_settings) |
| 249 | |
| 250 file_lists = get_cloud_free_files(CustomLidarMeasurement, files, cloudmask_settings) | |
| 218 | 251 |
| 219 for n, files in enumerate(file_lists): | 252 for n, files in enumerate(file_lists): |
| 220 measurement_id, measurement_no = get_corrected_measurement_id(args, n) | 253 measurement_id, measurement_no = get_corrected_measurement_id(args, n) |
| 221 convert_to_scc(CustomLidarMeasurement, files, args.dark_files, measurement_id, measurement_no) | 254 convert_to_scc(CustomLidarMeasurement, files, args.dark_files, measurement_id, measurement_no) |
| 222 else: | 255 else: |
