Thu, 19 Dec 2019 18:02:57 +0200
Checking if ancillary files are already on the DB before starting upload.
victor@7 | 1 | import requests |
moritz@29 | 2 | |
ioannis@17 | 3 | try: |
ioannis@17 | 4 | import urllib.parse as urlparse # Python 3 |
ioannis@17 | 5 | except ImportError: |
ioannis@17 | 6 | import urlparse # Python 2 |
victor@7 | 7 | |
victor@7 | 8 | import argparse |
madrouin@24 | 9 | import datetime |
madrouin@24 | 10 | import logging |
victor@7 | 11 | import os |
victor@7 | 12 | import re |
ioannis@32 | 13 | from io import BytesIO |
madrouin@24 | 14 | import sys |
victor@7 | 15 | import time |
madrouin@24 | 16 | import urlparse |
victor@7 | 17 | from zipfile import ZipFile |
madrouin@24 | 18 | |
i@14 | 19 | import yaml |
victor@7 | 20 | |
madrouin@24 | 21 | requests.packages.urllib3.disable_warnings() |
i@14 | 22 | logger = logging.getLogger(__name__) |
victor@7 | 23 | |
victor@7 | 24 | # The regex to find the measurement id from the measurement page |
victor@7 | 25 | # This should be read from the uploaded file, but would require an extra NetCDF module. |
i@36 | 26 | regex = "<h3>Measurement (?P<measurement_id>.{12,15}) <small>" # {12, 15} to handle both old- and new-style measurement ids. |
victor@7 | 27 | |
victor@7 | 28 | |
victor@7 | 29 | class SCC: |
madrouin@24 | 30 | """A simple class that will attempt to upload a file on the SCC server. |
i@14 | 31 | |
victor@7 | 32 | The uploading is done by simulating a normal browser session. In the current |
madrouin@20 | 33 | version no check is performed, and no feedback is given if the upload |
madrouin@20 | 34 | was successful. If everything is setup correctly, it will work. |
victor@7 | 35 | """ |
victor@7 | 36 | |
i@14 | 37 | def __init__(self, auth, output_dir, base_url): |
moritz@29 | 38 | |
victor@7 | 39 | self.auth = auth |
victor@7 | 40 | self.output_dir = output_dir |
i@14 | 41 | self.base_url = base_url |
victor@7 | 42 | self.session = requests.Session() |
moritz@29 | 43 | self.session.auth = auth |
moritz@29 | 44 | self.session.verify = False |
i@31 | 45 | |
i@14 | 46 | self.login_url = urlparse.urljoin(self.base_url, 'accounts/login/') |
i@31 | 47 | self.logout_url = urlparse.urljoin(self.base_url, 'accounts/logout/') |
i@31 | 48 | self.list_measurements_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/') |
i@31 | 49 | |
i@14 | 50 | self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/') |
ioannis@34 | 51 | self.download_hirelpp_pattern = urlparse.urljoin(self.base_url, |
ioannis@34 | 52 | 'data_processing/measurements/{0}/download-hirelpp/') |
ioannis@34 | 53 | self.download_cloudmask_pattern = urlparse.urljoin(self.base_url, |
ioannis@34 | 54 | 'data_processing/measurements/{0}/download-cloudmask/') |
ioannis@34 | 55 | |
moritz@29 | 56 | self.download_preprocessed_pattern = urlparse.urljoin(self.base_url, |
moritz@29 | 57 | 'data_processing/measurements/{0}/download-preprocessed/') |
moritz@29 | 58 | self.download_optical_pattern = urlparse.urljoin(self.base_url, |
moritz@29 | 59 | 'data_processing/measurements/{0}/download-optical/') |
moritz@29 | 60 | self.download_graph_pattern = urlparse.urljoin(self.base_url, |
moritz@29 | 61 | 'data_processing/measurements/{0}/download-plots/') |
ioannis@34 | 62 | self.download_elic_pattern = urlparse.urljoin(self.base_url, |
ioannis@34 | 63 | 'data_processing/measurements/{0}/download-elic/') |
i@14 | 64 | self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/') |
i@31 | 65 | |
i@14 | 66 | self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/') |
i@31 | 67 | self.api_measurement_pattern = urlparse.urljoin(self.api_base_url, 'measurements/{0}/') |
i@31 | 68 | self.api_measurements_url = urlparse.urljoin(self.api_base_url, 'measurements') |
i@38 | 69 | self.api_sounding_search_pattern = urlparse.urljoin(self.api_base_url, 'sounding_files/?filename={0}') |
i@38 | 70 | self.api_lidarratio_search_pattern = urlparse.urljoin(self.api_base_url, 'lidarratio_files/?filename={0}') |
i@38 | 71 | self.api_overlap_search_pattern = urlparse.urljoin(self.api_base_url, 'overlap_files/?filename={0}') |
i@14 | 72 | |
i@14 | 73 | def login(self, credentials): |
i@31 | 74 | """ Login to SCC. """ |
victor@7 | 75 | logger.debug("Attempting to login to SCC, username %s." % credentials[0]) |
moritz@29 | 76 | login_credentials = {'username': credentials[0], |
moritz@29 | 77 | 'password': credentials[1]} |
victor@7 | 78 | |
i@14 | 79 | logger.debug("Accessing login page at %s." % self.login_url) |
victor@7 | 80 | |
victor@7 | 81 | # Get upload form |
moritz@29 | 82 | login_page = self.session.get(self.login_url) |
victor@7 | 83 | |
i@31 | 84 | if not login_page.ok: |
i@31 | 85 | raise self.PageNotAccessibleError('Could not access login pages. Status code %s' % login_page.status_code) |
victor@7 | 86 | |
i@31 | 87 | logger.debug("Submitting credentials.") |
victor@7 | 88 | # Submit the login data |
i@14 | 89 | login_submit = self.session.post(self.login_url, |
moritz@29 | 90 | data=login_credentials, |
victor@7 | 91 | headers={'X-CSRFToken': login_page.cookies['csrftoken'], |
moritz@29 | 92 | 'referer': self.login_url}) |
victor@7 | 93 | return login_submit |
victor@7 | 94 | |
victor@7 | 95 | def logout(self): |
i@31 | 96 | """ Logout from SCC """ |
i@31 | 97 | return self.session.get(self.logout_url, stream=True) |
victor@7 | 98 | |
ioannis@32 | 99 | def upload_file(self, filename, system_id, rs_filename=None, ov_filename=None, lr_filename=None): |
madrouin@20 | 100 | """ Upload a filename for processing with a specific system. If the |
victor@7 | 101 | upload is successful, it returns the measurement id. """ |
victor@7 | 102 | # Get submit page |
moritz@29 | 103 | upload_page = self.session.get(self.upload_url) |
victor@7 | 104 | |
victor@7 | 105 | # Submit the data |
victor@7 | 106 | upload_data = {'system': system_id} |
victor@7 | 107 | files = {'data': open(filename, 'rb')} |
victor@7 | 108 | |
madrouin@20 | 109 | if rs_filename is not None: |
i@38 | 110 | ancillary_file, _ = self.get_ancillary(rs_filename, 'sounding') |
i@38 | 111 | |
i@38 | 112 | if ancillary_file.already_on_scc: |
i@38 | 113 | logger.warning("Sounding file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(ancillary_file)) |
i@38 | 114 | else: |
i@38 | 115 | logger.debug('Adding sounding file %s' % rs_filename) |
i@38 | 116 | files['sounding_file'] = open(rs_filename, 'rb') |
madrouin@20 | 117 | |
ioannis@32 | 118 | if ov_filename is not None: |
i@38 | 119 | ancillary_file, _ = self.get_ancillary(ov_filename, 'overlap') |
i@38 | 120 | |
i@38 | 121 | if ancillary_file.already_on_scc: |
i@38 | 122 | logger.warning("Overlap file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(ancillary_file)) |
i@38 | 123 | else: |
i@38 | 124 | logger.debug('Adding overlap file %s' % ov_filename) |
i@38 | 125 | files['overlap_file'] = open(ov_filename, 'rb') |
i@31 | 126 | |
i@31 | 127 | if lr_filename is not None: |
i@38 | 128 | ancillary_file, _ = self.get_ancillary(lr_filename, 'lidarratio') |
i@38 | 129 | |
i@38 | 130 | if ancillary_file.already_on_scc: |
i@38 | 131 | logger.warning( |
i@38 | 132 | "Lidar ratio file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(ancillary_file)) |
i@38 | 133 | else: |
i@38 | 134 | logger.debug('Adding lidar ratio file %s' % lr_filename) |
i@38 | 135 | files['lidar_ratio_file'] = open(lr_filename, 'rb') |
i@31 | 136 | |
ioannis@32 | 137 | logger.info("Uploading of file(s) %s started." % filename) |
victor@7 | 138 | |
i@14 | 139 | upload_submit = self.session.post(self.upload_url, |
victor@7 | 140 | data=upload_data, |
victor@7 | 141 | files=files, |
victor@7 | 142 | headers={'X-CSRFToken': upload_page.cookies['csrftoken'], |
moritz@29 | 143 | 'referer': self.upload_url}) |
victor@7 | 144 | |
victor@7 | 145 | if upload_submit.status_code != 200: |
i@14 | 146 | logger.warning("Connection error. Status code: %s" % upload_submit.status_code) |
victor@7 | 147 | return False |
victor@7 | 148 | |
victor@7 | 149 | # Check if there was a redirect to a new page. |
i@14 | 150 | if upload_submit.url == self.upload_url: |
victor@7 | 151 | measurement_id = False |
ioannis@32 | 152 | logger.error("Uploaded file(s) rejected! Try to upload manually to see the error.") |
victor@7 | 153 | else: |
victor@7 | 154 | measurement_id = re.findall(regex, upload_submit.text)[0] |
madrouin@23 | 155 | logger.info("Successfully uploaded measurement with id %s." % measurement_id) |
victor@7 | 156 | |
victor@7 | 157 | return measurement_id |
victor@7 | 158 | |
victor@7 | 159 | def download_files(self, measurement_id, subdir, download_url): |
victor@7 | 160 | """ Downloads some files from the download_url to the specified |
madrouin@20 | 161 | subdir. This method is used to download preprocessed file, optical |
victor@7 | 162 | files etc. |
victor@7 | 163 | """ |
ioannis@32 | 164 | # TODO: Make downloading more robust (e.g. in case that files do not exist on server). |
victor@7 | 165 | # Get the file |
moritz@29 | 166 | request = self.session.get(download_url, stream=True) |
moritz@29 | 167 | |
moritz@29 | 168 | if not request.ok: |
moritz@29 | 169 | raise Exception("Could not download files for measurement '%s'" % measurement_id) |
victor@7 | 170 | |
victor@7 | 171 | # Create the dir if it does not exist |
victor@7 | 172 | local_dir = os.path.join(self.output_dir, measurement_id, subdir) |
victor@7 | 173 | if not os.path.exists(local_dir): |
victor@7 | 174 | os.makedirs(local_dir) |
victor@7 | 175 | |
victor@7 | 176 | # Save the file by chunk, needed if the file is big. |
ioannis@32 | 177 | memory_file = BytesIO() |
victor@7 | 178 | |
victor@7 | 179 | for chunk in request.iter_content(chunk_size=1024): |
victor@7 | 180 | if chunk: # filter out keep-alive new chunks |
victor@7 | 181 | memory_file.write(chunk) |
victor@7 | 182 | memory_file.flush() |
victor@7 | 183 | |
victor@7 | 184 | zip_file = ZipFile(memory_file) |
victor@7 | 185 | |
victor@7 | 186 | for ziped_name in zip_file.namelist(): |
victor@7 | 187 | basename = os.path.basename(ziped_name) |
victor@7 | 188 | |
victor@7 | 189 | local_file = os.path.join(local_dir, basename) |
victor@7 | 190 | |
victor@7 | 191 | with open(local_file, 'wb') as f: |
victor@7 | 192 | f.write(zip_file.read(ziped_name)) |
victor@7 | 193 | |
ioannis@34 | 194 | def download_hirelpp(self, measurement_id): |
ioannis@34 | 195 | """ Download HiRElPP files for the measurement id. """ |
ioannis@34 | 196 | # Construct the download url |
ioannis@34 | 197 | download_url = self.download_hirelpp_pattern.format(measurement_id) |
ioannis@34 | 198 | self.download_files(measurement_id, 'hirelpp', download_url) |
ioannis@34 | 199 | |
ioannis@34 | 200 | def download_cloudmask(self, measurement_id): |
ioannis@34 | 201 | """ Download preprocessed files for the measurement id. """ |
ioannis@34 | 202 | # Construct the download url |
ioannis@34 | 203 | download_url = self.download_cloudmask_pattern.format(measurement_id) |
ioannis@34 | 204 | self.download_files(measurement_id, 'cloudmask', download_url) |
ioannis@34 | 205 | |
victor@7 | 206 | def download_preprocessed(self, measurement_id): |
victor@7 | 207 | """ Download preprocessed files for the measurement id. """ |
victor@7 | 208 | # Construct the download url |
i@14 | 209 | download_url = self.download_preprocessed_pattern.format(measurement_id) |
victor@7 | 210 | self.download_files(measurement_id, 'scc_preprocessed', download_url) |
victor@7 | 211 | |
victor@7 | 212 | def download_optical(self, measurement_id): |
victor@7 | 213 | """ Download optical files for the measurement id. """ |
victor@7 | 214 | # Construct the download url |
moritz@29 | 215 | download_url = self.download_optical_pattern.format(measurement_id) |
victor@7 | 216 | self.download_files(measurement_id, 'scc_optical', download_url) |
victor@7 | 217 | |
victor@7 | 218 | def download_graphs(self, measurement_id): |
victor@7 | 219 | """ Download profile graphs for the measurement id. """ |
victor@7 | 220 | # Construct the download url |
i@14 | 221 | download_url = self.download_graph_pattern.format(measurement_id) |
victor@7 | 222 | self.download_files(measurement_id, 'scc_plots', download_url) |
victor@7 | 223 | |
ioannis@34 | 224 | def download_elic(self, measurement_id): |
ioannis@34 | 225 | """ Download profile graphs for the measurement id. """ |
ioannis@34 | 226 | # Construct the download url |
ioannis@34 | 227 | download_url = self.download_elic_pattern.format(measurement_id) |
ioannis@34 | 228 | self.download_files(measurement_id, 'elic', download_url) |
ioannis@32 | 229 | |
victor@7 | 230 | def rerun_processing(self, measurement_id, monitor=True): |
madrouin@26 | 231 | measurement, status = self.get_measurement(measurement_id) |
victor@7 | 232 | |
victor@7 | 233 | if measurement: |
moritz@29 | 234 | request = self.session.get(measurement.rerun_processing_url, stream=True) |
victor@7 | 235 | |
victor@7 | 236 | if request.status_code != 200: |
i@14 | 237 | logger.error( |
i@14 | 238 | "Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code)) |
victor@7 | 239 | return |
victor@7 | 240 | |
victor@7 | 241 | if monitor: |
victor@7 | 242 | self.monitor_processing(measurement_id) |
victor@7 | 243 | |
victor@7 | 244 | def rerun_all(self, measurement_id, monitor=True): |
victor@7 | 245 | logger.debug("Started rerun_all procedure.") |
victor@7 | 246 | |
victor@7 | 247 | logger.debug("Getting measurement %s" % measurement_id) |
madrouin@26 | 248 | measurement, status = self.get_measurement(measurement_id) |
victor@7 | 249 | |
victor@7 | 250 | if measurement: |
victor@7 | 251 | logger.debug("Attempting to rerun all processing through %s." % measurement.rerun_all_url) |
victor@7 | 252 | |
moritz@29 | 253 | request = self.session.get(measurement.rerun_all_url, stream=True) |
victor@7 | 254 | |
victor@7 | 255 | if request.status_code != 200: |
victor@7 | 256 | logger.error("Could not rerun pre processing for %s. Status code: %s" % |
victor@7 | 257 | (measurement_id, request.status_code)) |
victor@7 | 258 | return |
victor@7 | 259 | |
victor@7 | 260 | if monitor: |
victor@7 | 261 | self.monitor_processing(measurement_id) |
victor@7 | 262 | |
ioannis@32 | 263 | def process(self, filename, system_id, monitor, rs_filename=None, lr_filename=None, ov_filename=None): |
victor@7 | 264 | """ Upload a file for processing and wait for the processing to finish. |
victor@7 | 265 | If the processing is successful, it will download all produced files. |
victor@7 | 266 | """ |
victor@7 | 267 | logger.info("--- Processing started on %s. ---" % datetime.datetime.now()) |
victor@7 | 268 | # Upload file |
madrouin@26 | 269 | logger.info("--- Uploading file") |
ioannis@32 | 270 | measurement_id = self.upload_file(filename, system_id, |
ioannis@32 | 271 | rs_filename=rs_filename, |
ioannis@32 | 272 | lr_filename=lr_filename, |
ioannis@32 | 273 | ov_filename=ov_filename) |
victor@7 | 274 | |
ioannis@32 | 275 | if measurement_id and monitor: |
ioannis@32 | 276 | logger.info("--- Monitoring processing") |
moritz@29 | 277 | return self.monitor_processing(measurement_id) |
ioannis@32 | 278 | |
moritz@29 | 279 | return None |
victor@7 | 280 | |
victor@7 | 281 | def monitor_processing(self, measurement_id): |
victor@7 | 282 | """ Monitor the processing progress of a measurement id""" |
victor@7 | 283 | |
madrouin@24 | 284 | # try to deal with error 404 |
madrouin@24 | 285 | error_count = 0 |
madrouin@24 | 286 | error_max = 6 |
madrouin@26 | 287 | time_sleep = 10 |
madrouin@24 | 288 | |
madrouin@25 | 289 | # try to wait for measurement to appear in API |
madrouin@24 | 290 | measurement = None |
ioannis@32 | 291 | logger.info("Looking for measurement %s in SCC", measurement_id) |
madrouin@26 | 292 | while error_count < error_max: |
madrouin@26 | 293 | time.sleep(time_sleep) |
madrouin@26 | 294 | measurement, status = self.get_measurement(measurement_id) |
madrouin@26 | 295 | if status != 200 and error_count < error_max: |
ioannis@32 | 296 | logger.error("Measurement not found. waiting %ds", time_sleep) |
madrouin@24 | 297 | error_count += 1 |
madrouin@26 | 298 | else: |
madrouin@26 | 299 | break |
madrouin@24 | 300 | |
madrouin@24 | 301 | if error_count == error_max: |
ioannis@32 | 302 | logger.critical("Measurement %s doesn't seem to exist", measurement_id) |
madrouin@26 | 303 | sys.exit(1) |
madrouin@26 | 304 | |
ioannis@32 | 305 | logger.info('Measurement %s found', measurement_id) |
madrouin@24 | 306 | |
victor@7 | 307 | if measurement is not None: |
victor@7 | 308 | while measurement.is_running: |
ioannis@34 | 309 | logger.info("Measurement is being processed. Please wait.") |
victor@7 | 310 | time.sleep(10) |
madrouin@26 | 311 | measurement, status = self.get_measurement(measurement_id) |
madrouin@24 | 312 | |
ioannis@34 | 313 | logger.info("Measurement processing finished.") |
ioannis@34 | 314 | if measurement.hirelpp == 127: |
ioannis@34 | 315 | logger.info("Downloading hirelpp files.") |
ioannis@34 | 316 | self.download_hirelpp(measurement_id) |
ioannis@34 | 317 | if measurement.cloudmask == 127: |
ioannis@34 | 318 | logger.info("Downloading cloudmask files.") |
ioannis@34 | 319 | self.download_cloudmask(measurement_id) |
ioannis@34 | 320 | if measurement.elpp == 127: |
victor@7 | 321 | logger.info("Downloading preprocessed files.") |
victor@7 | 322 | self.download_preprocessed(measurement_id) |
ioannis@34 | 323 | if measurement.elda == 127: |
victor@7 | 324 | logger.info("Downloading optical files.") |
victor@7 | 325 | self.download_optical(measurement_id) |
victor@7 | 326 | logger.info("Downloading graphs.") |
victor@7 | 327 | self.download_graphs(measurement_id) |
ioannis@34 | 328 | if measurement.elic == 127: |
ioannis@34 | 329 | logger.info("Downloading preprocessed files.") |
ioannis@34 | 330 | self.download_elic(measurement_id) |
victor@7 | 331 | logger.info("--- Processing finished. ---") |
victor@7 | 332 | return measurement |
victor@7 | 333 | |
victor@7 | 334 | def get_measurement(self, measurement_id): |
i@31 | 335 | measurement_url = self.api_measurement_pattern.format(measurement_id) |
i@31 | 336 | logger.debug("Measurement API URL: %s" % measurement_url) |
victor@7 | 337 | |
moritz@29 | 338 | response = self.session.get(measurement_url) |
victor@7 | 339 | |
moritz@29 | 340 | if not response.ok: |
i@14 | 341 | logger.error('Could not access API. Status code %s.' % response.status_code) |
madrouin@26 | 342 | return None, response.status_code |
i@14 | 343 | |
victor@7 | 344 | response_dict = response.json() |
victor@7 | 345 | |
victor@7 | 346 | if response_dict: |
moritz@29 | 347 | measurement = Measurement(self.base_url, response_dict) |
madrouin@26 | 348 | return measurement, response.status_code |
victor@7 | 349 | else: |
victor@7 | 350 | logger.error("No measurement with id %s found on the SCC." % measurement_id) |
madrouin@26 | 351 | return None, response.status_code |
victor@7 | 352 | |
victor@7 | 353 | def delete_measurement(self, measurement_id): |
victor@7 | 354 | """ Deletes a measurement with the provided measurement id. The user |
madrouin@20 | 355 | should have the appropriate permissions. |
madrouin@20 | 356 | |
victor@7 | 357 | The procedures is performed directly through the web interface and |
victor@7 | 358 | NOT through the API. |
victor@7 | 359 | """ |
victor@7 | 360 | # Get the measurement object |
i@31 | 361 | measurement, _ = self.get_measurement(measurement_id) |
victor@7 | 362 | |
victor@7 | 363 | # Check that it exists |
victor@7 | 364 | if measurement is None: |
victor@7 | 365 | logger.warning("Nothing to delete.") |
victor@7 | 366 | return None |
victor@7 | 367 | |
victor@7 | 368 | # Go the the page confirming the deletion |
moritz@29 | 369 | delete_url = self.delete_measurement_pattern.format(measurement_id) |
victor@7 | 370 | |
moritz@29 | 371 | confirm_page = self.session.get(delete_url) |
victor@7 | 372 | |
victor@7 | 373 | # Check that the page opened properly |
victor@7 | 374 | if confirm_page.status_code != 200: |
victor@7 | 375 | logger.warning("Could not open delete page. Status: {0}".format(confirm_page.status_code)) |
victor@7 | 376 | return None |
victor@7 | 377 | |
victor@7 | 378 | # Delete the measurement |
victor@7 | 379 | delete_page = self.session.post(delete_url, |
victor@7 | 380 | data={'post': 'yes'}, |
victor@7 | 381 | headers={'X-CSRFToken': confirm_page.cookies['csrftoken'], |
victor@7 | 382 | 'referer': delete_url} |
victor@7 | 383 | ) |
i@31 | 384 | if not delete_page.ok: |
victor@7 | 385 | logger.warning("Something went wrong. Delete page status: {0}".format( |
victor@7 | 386 | delete_page.status_code)) |
victor@7 | 387 | return None |
victor@7 | 388 | |
victor@7 | 389 | logger.info("Deleted measurement {0}".format(measurement_id)) |
victor@7 | 390 | return True |
victor@7 | 391 | |
victor@7 | 392 | def available_measurements(self): |
victor@7 | 393 | """ Get a list of available measurement on the SCC. """ |
i@31 | 394 | response = self.session.get(self.api_measurements_url) |
victor@7 | 395 | response_dict = response.json() |
victor@7 | 396 | |
victor@7 | 397 | if response_dict: |
victor@7 | 398 | measurement_list = response_dict['objects'] |
i@14 | 399 | measurements = [Measurement(self.base_url, measurement_dict) for measurement_dict in measurement_list] |
victor@7 | 400 | logger.info("Found %s measurements on the SCC." % len(measurements)) |
victor@7 | 401 | else: |
victor@7 | 402 | logger.warning("No response received from the SCC when asked for available measurements.") |
i@31 | 403 | measurements = None |
victor@7 | 404 | |
victor@7 | 405 | return measurements |
victor@7 | 406 | |
moritz@29 | 407 | def list_measurements(self, station=None, system=None, start=None, stop=None, upload_status=None, |
moritz@29 | 408 | processing_status=None, optical_processing=None): |
moritz@29 | 409 | |
i@31 | 410 | # TODO: Change this to work through the API |
i@31 | 411 | |
moritz@29 | 412 | # Need to set to empty string if not specified, we won't get any results |
moritz@29 | 413 | params = { |
moritz@29 | 414 | "station": station if station is not None else "", |
moritz@29 | 415 | "system": system if system is not None else "", |
moritz@29 | 416 | "stop": stop if stop is not None else "", |
moritz@29 | 417 | "start": start if start is not None else "", |
moritz@29 | 418 | "upload_status": upload_status if upload_status is not None else "", |
moritz@29 | 419 | "preprocessing_status": processing_status if processing_status is not None else "", |
moritz@29 | 420 | "optical_processing_status": optical_processing if optical_processing is not None else "" |
moritz@29 | 421 | } |
i@31 | 422 | |
i@31 | 423 | response_txt = self.session.get(self.list_measurements_url, params=params).text |
moritz@29 | 424 | tbl_rgx = re.compile(r'<table id="measurements">(.*?)</table>', re.DOTALL) |
moritz@29 | 425 | entry_rgx = re.compile(r'<tr>(.*?)</tr>', re.DOTALL) |
moritz@29 | 426 | measurement_rgx = re.compile( |
moritz@29 | 427 | r'.*?<td><a[^>]*>(\w+)</a>.*?<td>.*?<td>([\w-]+ [\w:]+)</td>.*<td data-order="([-]?\d+),([-]?\d+),([-]?\d+)".*', |
moritz@29 | 428 | re.DOTALL) |
i@31 | 429 | matches = tbl_rgx.findall(response_txt) |
moritz@29 | 430 | if len(matches) != 1: |
moritz@29 | 431 | return [] |
moritz@29 | 432 | |
moritz@29 | 433 | ret = [] |
moritz@29 | 434 | for entry in entry_rgx.finditer(matches[0]): |
moritz@29 | 435 | m = measurement_rgx.match(entry.string[entry.start(0):entry.end(0)]) |
moritz@29 | 436 | if m: |
moritz@29 | 437 | name, date, upload, preproc, optical = m.groups() |
moritz@29 | 438 | ret.append( |
moritz@29 | 439 | Measurement(self.base_url, {"id": name, "upload": int(upload), "pre_processing": int(preproc), |
moritz@29 | 440 | "processing": int(optical)})) |
moritz@29 | 441 | |
moritz@29 | 442 | return ret |
moritz@29 | 443 | |
i@31 | 444 | def measurement_id_for_date(self, t1, call_sign, base_number=0): |
victor@7 | 445 | """ Give the first available measurement id on the SCC for the specific |
madrouin@20 | 446 | date. |
victor@7 | 447 | """ |
victor@7 | 448 | date_str = t1.strftime('%Y%m%d') |
i@31 | 449 | base_id = "%s%s" % (date_str, call_sign) |
i@31 | 450 | search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % base_id) |
victor@7 | 451 | |
moritz@29 | 452 | response = self.session.get(search_url) |
victor@7 | 453 | |
victor@7 | 454 | response_dict = response.json() |
victor@7 | 455 | |
victor@7 | 456 | measurement_id = None |
victor@7 | 457 | |
victor@7 | 458 | if response_dict: |
victor@7 | 459 | measurement_list = response_dict['objects'] |
i@31 | 460 | |
i@31 | 461 | if len(measurement_list) == 100: |
i@31 | 462 | raise ValueError('No available measurement id found.') |
i@31 | 463 | |
victor@7 | 464 | existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list] |
victor@7 | 465 | |
victor@7 | 466 | measurement_number = base_number |
i@31 | 467 | measurement_id = "%s%02i" % (base_id, measurement_number) |
victor@7 | 468 | |
victor@7 | 469 | while measurement_id in existing_ids: |
victor@7 | 470 | measurement_number = measurement_number + 1 |
i@31 | 471 | measurement_id = "%s%02i" % (base_id, measurement_number) |
victor@7 | 472 | |
victor@7 | 473 | return measurement_id |
victor@7 | 474 | |
i@38 | 475 | def get_ancillary(self, filename, file_type): |
i@38 | 476 | """ |
i@38 | 477 | Try to get the ancillary file data from the SCC API. |
i@38 | 478 | |
i@38 | 479 | The result will always be an API object. If the file does not exist, the .exists property is set to False. |
i@38 | 480 | |
i@38 | 481 | Parameters |
i@38 | 482 | ---------- |
i@38 | 483 | filename : str |
i@38 | 484 | Filename of the uploaded file. |
i@38 | 485 | file_type : str |
i@38 | 486 | Type of ancillary file. One of 'sounding', 'overlap', 'lidarratio'. |
i@38 | 487 | |
i@38 | 488 | Returns |
i@38 | 489 | : AncillaryFile |
i@38 | 490 | The api object. |
i@38 | 491 | """ |
i@38 | 492 | assert file_type in ['sounding', 'overlap', 'lidarratio'] |
i@38 | 493 | |
i@38 | 494 | if file_type == 'sounding': |
i@38 | 495 | file_url = self.api_sounding_search_pattern.format(filename) |
i@38 | 496 | elif file_type == 'overlap': |
i@38 | 497 | file_url = self.api_overlap_search_pattern.format(filename) |
i@38 | 498 | else: |
i@38 | 499 | file_url = self.api_lidarratio_search_pattern.format(filename) |
i@38 | 500 | |
i@38 | 501 | response = self.session.get(file_url) |
i@38 | 502 | |
i@38 | 503 | if not response.ok: |
i@38 | 504 | logger.error('Could not access API. Status code %s.' % response.status_code) |
i@38 | 505 | return None, response.status_code |
i@38 | 506 | |
i@38 | 507 | response_dict = response.json() |
i@38 | 508 | object_list = response_dict['objects'] |
i@38 | 509 | |
i@38 | 510 | logger.debug("Ancillary file JSON: {0}".format(object_list)) |
i@38 | 511 | |
i@38 | 512 | if object_list: |
i@38 | 513 | ancillary_file = AncillaryFile(self.api_base_url, object_list[0]) # Assume only one file is returned |
i@38 | 514 | else: |
i@38 | 515 | ancillary_file = AncillaryFile(self.api_base_url, None) # Create an empty object |
i@38 | 516 | |
i@38 | 517 | return ancillary_file, response.status_code |
i@38 | 518 | |
i@31 | 519 | class PageNotAccessibleError(RuntimeError): |
i@31 | 520 | pass |
i@31 | 521 | |
victor@7 | 522 | |
ioannis@17 | 523 | class ApiObject(object): |
victor@7 | 524 | """ A generic class object. """ |
victor@7 | 525 | |
i@14 | 526 | def __init__(self, base_url, dict_response): |
i@14 | 527 | self.base_url = base_url |
victor@7 | 528 | |
victor@7 | 529 | if dict_response: |
victor@7 | 530 | # Add the dictionary key value pairs as object properties |
victor@7 | 531 | for key, value in dict_response.items(): |
ioannis@17 | 532 | # logger.debug('Setting key {0} to value {1}'.format(key, value)) |
ioannis@17 | 533 | try: |
ioannis@17 | 534 | setattr(self, key, value) |
ioannis@17 | 535 | except: |
ioannis@17 | 536 | logger.warning('Could not set attribute {0} to value {1}'.format(key, value)) |
victor@7 | 537 | self.exists = True |
victor@7 | 538 | else: |
victor@7 | 539 | self.exists = False |
victor@7 | 540 | |
victor@7 | 541 | |
victor@7 | 542 | class Measurement(ApiObject): |
victor@7 | 543 | """ This class represents the measurement object as returned in the SCC API. |
victor@7 | 544 | """ |
victor@7 | 545 | |
victor@7 | 546 | @property |
victor@7 | 547 | def rerun_processing_url(self): |
ioannis@34 | 548 | url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-elda/') |
i@14 | 549 | return url_pattern.format(self.id) |
victor@7 | 550 | |
victor@7 | 551 | @property |
victor@7 | 552 | def rerun_all_url(self): |
i@14 | 553 | ulr_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-all/') |
i@14 | 554 | return ulr_pattern.format(self.id) |
victor@7 | 555 | |
victor@7 | 556 | def __str__(self): |
victor@7 | 557 | return "%s: %s, %s, %s" % (self.id, |
victor@7 | 558 | self.upload, |
victor@7 | 559 | self.pre_processing, |
victor@7 | 560 | self.processing) |
victor@7 | 561 | |
victor@7 | 562 | |
i@38 | 563 | class AncillaryFile(ApiObject): |
i@38 | 564 | """ This class represents the ancilalry file object as returned in the SCC API. |
i@38 | 565 | """ |
i@38 | 566 | @property |
i@38 | 567 | def already_on_scc(self): |
i@38 | 568 | if self.exists is False: |
i@38 | 569 | return False |
i@38 | 570 | |
i@38 | 571 | return not self.status == 'missing' |
i@38 | 572 | |
i@38 | 573 | def __str__(self): |
i@38 | 574 | return "%s: %s, %s" % (self.id, |
i@38 | 575 | self.filename, |
i@38 | 576 | self.status) |
i@38 | 577 | |
i@38 | 578 | |
ioannis@32 | 579 | def process_file(filename, system_id, settings, monitor=True, rs_filename=None, lr_filename=None, ov_filename=None): |
ioannis@32 | 580 | """ Shortcut function to process a file to the SCC. """ |
ioannis@32 | 581 | logger.info("Processing file %s, using system %s" % (filename, system_id)) |
victor@7 | 582 | |
i@14 | 583 | scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) |
i@14 | 584 | scc.login(settings['website_credentials']) |
ioannis@32 | 585 | measurement = scc.process(filename, system_id, |
ioannis@32 | 586 | monitor=monitor, |
ioannis@32 | 587 | rs_filename=rs_filename, |
ioannis@32 | 588 | lr_filename=lr_filename, |
ioannis@32 | 589 | ov_filename=ov_filename) |
victor@7 | 590 | scc.logout() |
victor@7 | 591 | return measurement |
victor@7 | 592 | |
victor@7 | 593 | |
ioannis@32 | 594 | def delete_measurements(measurement_ids, settings): |
moritz@29 | 595 | """ Shortcut function to delete measurements from the SCC. """ |
moritz@29 | 596 | scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) |
moritz@29 | 597 | scc.login(settings['website_credentials']) |
moritz@29 | 598 | for m_id in measurement_ids: |
moritz@29 | 599 | logger.info("Deleting %s" % m_id) |
moritz@29 | 600 | scc.delete_measurement(m_id) |
moritz@29 | 601 | scc.logout() |
moritz@29 | 602 | |
moritz@29 | 603 | |
moritz@29 | 604 | def rerun_all(measurement_ids, monitor, settings): |
moritz@29 | 605 | """ Shortcut function to rerun measurements from the SCC. """ |
i@14 | 606 | |
i@14 | 607 | scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) |
i@14 | 608 | scc.login(settings['website_credentials']) |
moritz@29 | 609 | for m_id in measurement_ids: |
moritz@29 | 610 | logger.info("Rerunning all products for %s" % m_id) |
moritz@29 | 611 | scc.rerun_all(m_id, monitor) |
victor@7 | 612 | scc.logout() |
victor@7 | 613 | |
victor@7 | 614 | |
moritz@29 | 615 | def rerun_processing(measurement_ids, monitor, settings): |
victor@7 | 616 | """ Shortcut function to delete a measurement from the SCC. """ |
i@14 | 617 | |
i@14 | 618 | scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) |
i@14 | 619 | scc.login(settings['website_credentials']) |
moritz@29 | 620 | for m_id in measurement_ids: |
moritz@29 | 621 | logger.info("Rerunning (optical) processing for %s" % m_id) |
moritz@29 | 622 | scc.rerun_processing(m_id, monitor) |
moritz@29 | 623 | scc.logout() |
moritz@29 | 624 | |
moritz@29 | 625 | |
moritz@29 | 626 | def list_measurements(settings, station=None, system=None, start=None, stop=None, upload_status=None, |
moritz@29 | 627 | preprocessing_status=None, |
moritz@29 | 628 | optical_processing=None): |
moritz@29 | 629 | """List all available measurements""" |
moritz@29 | 630 | scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) |
moritz@29 | 631 | scc.login(settings['website_credentials']) |
moritz@29 | 632 | ret = scc.list_measurements(station=station, system=system, start=start, stop=stop, upload_status=upload_status, |
moritz@29 | 633 | processing_status=preprocessing_status, optical_processing=optical_processing) |
moritz@29 | 634 | for entry in ret: |
moritz@29 | 635 | print("%s" % entry.id) |
victor@7 | 636 | scc.logout() |
victor@7 | 637 | |
victor@7 | 638 | |
moritz@29 | 639 | def download_measurements(measurement_ids, download_preproc, download_optical, download_graph, settings): |
moritz@29 | 640 | """Download all measurements for the specified IDs""" |
i@14 | 641 | scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) |
i@14 | 642 | scc.login(settings['website_credentials']) |
moritz@29 | 643 | for m_id in measurement_ids: |
moritz@29 | 644 | if download_preproc: |
moritz@29 | 645 | logger.info("Downloading preprocessed files for '%s'" % m_id) |
moritz@29 | 646 | scc.download_preprocessed(m_id) |
moritz@29 | 647 | logger.info("Complete") |
moritz@29 | 648 | if download_optical: |
moritz@29 | 649 | logger.info("Downloading optical files for '%s'" % m_id) |
moritz@29 | 650 | scc.download_optical(m_id) |
moritz@29 | 651 | logger.info("Complete") |
moritz@29 | 652 | if download_graph: |
moritz@29 | 653 | logger.info("Downloading profile graph files for '%s'" % m_id) |
moritz@29 | 654 | scc.download_graphs(m_id) |
moritz@29 | 655 | logger.info("Complete") |
i@14 | 656 | |
i@14 | 657 | |
moritz@29 | 658 | def settings_from_path(config_file_path): |
i@14 | 659 | """ Read the configuration file. |
i@14 | 660 | |
i@14 | 661 | The file should be in YAML syntax.""" |
i@14 | 662 | |
i@14 | 663 | if not os.path.isfile(config_file_path): |
moritz@29 | 664 | raise argparse.ArgumentTypeError("Wrong path for configuration file (%s)" % config_file_path) |
i@14 | 665 | |
i@14 | 666 | with open(config_file_path) as yaml_file: |
i@14 | 667 | try: |
i@14 | 668 | settings = yaml.safe_load(yaml_file) |
i@14 | 669 | logger.debug("Read settings file(%s)" % config_file_path) |
moritz@29 | 670 | except Exception: |
moritz@29 | 671 | raise argparse.ArgumentTypeError("Could not parse YAML file (%s)" % config_file_path) |
i@14 | 672 | |
i@14 | 673 | # YAML limitation: does not read tuples |
i@14 | 674 | settings['basic_credentials'] = tuple(settings['basic_credentials']) |
i@14 | 675 | settings['website_credentials'] = tuple(settings['website_credentials']) |
i@14 | 676 | return settings |
i@14 | 677 | |
i@14 | 678 | |
moritz@29 | 679 | # Setup for command specific parsers |
moritz@29 | 680 | def setup_delete(parser): |
moritz@29 | 681 | def delete_from_args(parsed): |
ioannis@32 | 682 | delete_measurements(parsed.IDs, parsed.config) |
moritz@29 | 683 | |
moritz@29 | 684 | parser.add_argument("IDs", nargs="+", help="measurement IDs to delete.") |
moritz@29 | 685 | parser.set_defaults(execute=delete_from_args) |
moritz@29 | 686 | |
moritz@29 | 687 | |
moritz@29 | 688 | def setup_rerun_all(parser): |
moritz@29 | 689 | def rerun_all_from_args(parsed): |
moritz@29 | 690 | rerun_all(parsed.IDs, parsed.process, parsed.config) |
moritz@29 | 691 | |
moritz@29 | 692 | parser.add_argument("IDs", nargs="+", help="Measurement IDs to rerun.") |
moritz@29 | 693 | parser.add_argument("-p", "--process", help="Wait for the results of the processing.", |
moritz@29 | 694 | action="store_true") |
moritz@29 | 695 | parser.set_defaults(execute=rerun_all_from_args) |
moritz@29 | 696 | |
moritz@29 | 697 | |
moritz@29 | 698 | def setup_rerun_processing(parser): |
moritz@29 | 699 | def rerun_processing_from_args(parsed): |
moritz@29 | 700 | rerun_processing(parsed.IDs, parsed.process, parsed.config) |
moritz@29 | 701 | |
moritz@29 | 702 | parser.add_argument("IDs", nargs="+", help="Measurement IDs to rerun the processing on.") |
moritz@29 | 703 | parser.add_argument("-p", "--process", help="Wait for the results of the processing.", |
moritz@29 | 704 | action="store_true") |
moritz@29 | 705 | parser.set_defaults(execute=rerun_processing_from_args) |
moritz@29 | 706 | |
moritz@29 | 707 | |
moritz@29 | 708 | def setup_process_file(parser): |
ioannis@32 | 709 | """ Upload and monitor processing progress.""" |
moritz@29 | 710 | def process_file_from_args(parsed): |
ioannis@32 | 711 | process_file(parsed.filename, parsed.system, parsed.config, monitor=True, |
ioannis@32 | 712 | rs_filename=parsed.radiosounding, |
ioannis@32 | 713 | ov_filename=parsed.overlap, |
ioannis@32 | 714 | lr_filename=parsed.lidarratio) |
moritz@29 | 715 | |
moritz@29 | 716 | parser.add_argument("filename", help="Measurement file name or path.") |
moritz@29 | 717 | parser.add_argument("system", help="Processing system id.") |
i@30 | 718 | parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path") |
ioannis@32 | 719 | parser.add_argument("--overlap", default=None, help="Overlap file name or path") |
ioannis@32 | 720 | parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path") |
ioannis@32 | 721 | |
moritz@29 | 722 | parser.set_defaults(execute=process_file_from_args) |
moritz@29 | 723 | |
moritz@29 | 724 | |
moritz@29 | 725 | def setup_upload_file(parser): |
ioannis@32 | 726 | """ Upload but do not monitor processing progress. """ |
moritz@29 | 727 | def upload_file_from_args(parsed): |
ioannis@32 | 728 | process_file(parsed.filename, parsed.system, parsed.config, monitor=False, |
ioannis@32 | 729 | rs_filename=parsed.radiosounding, |
ioannis@32 | 730 | ov_filename=parsed.overlap, |
ioannis@32 | 731 | lr_filename=parsed.lidarratio) |
moritz@29 | 732 | |
moritz@29 | 733 | parser.add_argument("filename", help="Measurement file name or path.") |
moritz@29 | 734 | parser.add_argument("system", help="Processing system id.") |
i@30 | 735 | parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path") |
ioannis@32 | 736 | parser.add_argument("--overlap", default=None, help="Overlap file name or path") |
ioannis@32 | 737 | parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path") |
i@30 | 738 | |
moritz@29 | 739 | parser.set_defaults(execute=upload_file_from_args) |
moritz@29 | 740 | |
moritz@29 | 741 | |
moritz@29 | 742 | def setup_list_measurements(parser): |
moritz@29 | 743 | def list_measurements_from_args(parsed): |
moritz@29 | 744 | list_measurements(parsed.config, station=parsed.station, system=parsed.system, start=parsed.start, |
moritz@29 | 745 | stop=parsed.stop, |
moritz@29 | 746 | upload_status=parsed.upload_status, preprocessing_status=parsed.preprocessing_status, |
moritz@29 | 747 | optical_processing=parsed.optical_processing_status) |
moritz@29 | 748 | |
moritz@29 | 749 | def status(arg): |
moritz@29 | 750 | if -127 <= int(arg) <= 127: |
moritz@29 | 751 | return arg |
moritz@29 | 752 | else: |
moritz@29 | 753 | raise argparse.ArgumentTypeError("Status must be between -127 and 127") |
moritz@29 | 754 | |
moritz@29 | 755 | def date(arg): |
moritz@29 | 756 | if re.match(r'\d{4}-\d{2}-\d{2}', arg): |
moritz@29 | 757 | return arg |
moritz@29 | 758 | else: |
moritz@29 | 759 | raise argparse.ArgumentTypeError("Date must be in format 'YYYY-MM-DD'") |
moritz@29 | 760 | |
moritz@29 | 761 | parser.add_argument("--station", help="Filter for only the selected station") |
moritz@29 | 762 | parser.add_argument("--system", help="Filter for only the selected station") |
moritz@29 | 763 | parser.add_argument("--start", help="Filter for only the selected station", type=date) |
moritz@29 | 764 | parser.add_argument("--stop", help="Filter for only the selected station", type=date) |
moritz@29 | 765 | parser.add_argument("--upload-status", help="Filter for only the selected station", type=status) |
moritz@29 | 766 | parser.add_argument("--preprocessing-status", help="Filter for only the selected station", type=status) |
moritz@29 | 767 | parser.add_argument("--optical-processing-status", help="Filter for only the selected station", type=status) |
moritz@29 | 768 | parser.set_defaults(execute=list_measurements_from_args) |
moritz@29 | 769 | |
moritz@29 | 770 | |
moritz@29 | 771 | def setup_download_measurements(parser): |
moritz@29 | 772 | def download_measurements_from_args(parsed): |
moritz@29 | 773 | preproc = parsed.download_preprocessed |
moritz@29 | 774 | optical = parsed.download_optical |
moritz@29 | 775 | graphs = parsed.download_profile_graphs |
moritz@29 | 776 | if not preproc and not graphs: |
moritz@29 | 777 | optical = True |
moritz@29 | 778 | download_measurements(parsed.IDs, preproc, optical, graphs, parsed.config) |
moritz@29 | 779 | |
moritz@29 | 780 | parser.add_argument("IDs", help="Measurement IDs that should be downloaded.", nargs="+") |
moritz@29 | 781 | parser.add_argument("--download-preprocessed", action="store_true", help="Download preprocessed files.") |
moritz@29 | 782 | parser.add_argument("--download-optical", action="store_true", |
moritz@29 | 783 | help="Download optical files (default if no other download is used).") |
moritz@29 | 784 | parser.add_argument("--download-profile-graphs", action="store_true", help="Download profile graph files.") |
moritz@29 | 785 | parser.set_defaults(execute=download_measurements_from_args) |
moritz@29 | 786 | |
moritz@29 | 787 | |
victor@7 | 788 | def main(): |
victor@7 | 789 | # Define the command line arguments. |
victor@7 | 790 | parser = argparse.ArgumentParser() |
moritz@29 | 791 | subparsers = parser.add_subparsers() |
moritz@29 | 792 | |
moritz@29 | 793 | delete_parser = subparsers.add_parser("delete", help="Deletes a measurement.") |
ioannis@32 | 794 | rerun_all_parser = subparsers.add_parser("rerun-all", help="Reprocess a measurement on the SCC.") |
moritz@29 | 795 | rerun_processing_parser = subparsers.add_parser("rerun-processing", |
i@36 | 796 | help="Rerun processing routines for a measurement.") |
i@36 | 797 | process_file_parser = subparsers.add_parser("process-file", help="Upload a file and download processing results.") |
moritz@29 | 798 | upload_file_parser = subparsers.add_parser("upload-file", help="Upload a file.") |
ioannis@32 | 799 | list_parser = subparsers.add_parser("list", help="List measurements registered on the SCC.") |
moritz@29 | 800 | download_parser = subparsers.add_parser("download", help="Download selected measurements.") |
moritz@29 | 801 | |
moritz@29 | 802 | setup_delete(delete_parser) |
moritz@29 | 803 | setup_rerun_all(rerun_all_parser) |
moritz@29 | 804 | setup_rerun_processing(rerun_processing_parser) |
moritz@29 | 805 | setup_process_file(process_file_parser) |
moritz@29 | 806 | setup_upload_file(upload_file_parser) |
moritz@29 | 807 | setup_list_measurements(list_parser) |
moritz@29 | 808 | setup_download_measurements(download_parser) |
victor@7 | 809 | |
victor@7 | 810 | # Verbosity settings from http://stackoverflow.com/a/20663028 |
victor@7 | 811 | parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const", |
victor@7 | 812 | dest="loglevel", const=logging.DEBUG, default=logging.INFO, |
victor@7 | 813 | ) |
victor@7 | 814 | parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const", |
victor@7 | 815 | dest="loglevel", const=logging.WARNING |
victor@7 | 816 | ) |
victor@7 | 817 | |
ioannis@32 | 818 | # Setup default config location |
moritz@29 | 819 | home = os.path.expanduser("~") |
moritz@29 | 820 | default_config_location = os.path.abspath(os.path.join(home, ".scc_access.yaml")) |
moritz@29 | 821 | parser.add_argument("-c", "--config", help="Path to the config file.", type=settings_from_path, |
moritz@29 | 822 | default=default_config_location) |
moritz@29 | 823 | |
victor@7 | 824 | args = parser.parse_args() |
victor@7 | 825 | |
victor@7 | 826 | # Get the logger with the appropriate level |
victor@7 | 827 | logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel) |
victor@7 | 828 | |
moritz@29 | 829 | # Dispatch to appropriate function |
moritz@29 | 830 | args.execute(args) |
moritz@29 | 831 | |
i@14 | 832 | |
moritz@29 | 833 | # When running through terminal |
moritz@29 | 834 | if __name__ == '__main__': |
moritz@29 | 835 | main() |