Mon, 24 May 2021 11:43:31 +0300
Slightly improved documentation.
ioannis@67 | 1 | """ This is a script that allows interaction with the SCC through the command line. |
ioannis@67 | 2 | |
ioannis@67 | 3 | It is based on the requests module for accessing the server. |
ioannis@67 | 4 | |
ioannis@67 | 5 | Most of the interactions are done through the web interface, i.e. by mimicking user interaction with the |
ioannis@67 | 6 | SCC website (i.e. user login, data submission, etc.). In few cases, the SCC API is also used. |
ioannis@67 | 7 | |
ioannis@67 | 8 | Most of the functionality is included in the SCC class. The class is used to login into the SCC website and automate |
ioannis@67 | 9 | interaction with the site (i.e. upload a file, get measurement status, etc.). |
ioannis@67 | 10 | |
ioannis@67 | 11 | Two other classes (Measurement, AncillaryFile) are used in some cases to handle the output of the SCC API. |
ioannis@67 | 12 | |
ioannis@67 | 13 | Several shortcut functions are defined to perform specific tasks using the SCC class (e.g. process_file, delete_measurements etc). |
ioannis@67 | 14 | """ |
ioannis@43 | 15 | import sys |
ioannis@43 | 16 | |
victor@7 | 17 | import requests |
victor@7 | 18 | |
ioannis@43 | 19 | try: |
ioannis@43 | 20 | import urllib.parse as urlparse # Python 3 |
ioannis@43 | 21 | except ImportError: |
ioannis@17 | 22 | import urlparse # Python 2 |
ioannis@43 | 23 | |
victor@7 | 24 | import argparse |
madrouin@24 | 25 | import datetime |
madrouin@24 | 26 | import logging |
victor@7 | 27 | import os |
victor@7 | 28 | import re |
ioannis@32 | 29 | from io import BytesIO |
ioannis@45 | 30 | |
victor@7 | 31 | import time |
i@40 | 32 | |
victor@7 | 33 | from zipfile import ZipFile |
madrouin@24 | 34 | |
i@14 | 35 | import yaml |
victor@7 | 36 | |
ioannis@43 | 37 | import netCDF4 as netcdf |
ioannis@43 | 38 | |
ioannis@43 | 39 | requests.packages.urllib3.disable_warnings() |
i@14 | 40 | logger = logging.getLogger(__name__) |
victor@7 | 41 | |
victor@7 | 42 | # The regex to find the measurement id from the measurement page |
victor@7 | 43 | # This should be read from the uploaded file, but would require an extra NetCDF module. |
i@36 | 44 | regex = "<h3>Measurement (?P<measurement_id>.{12,15}) <small>" # {12, 15} to handle both old- and new-style measurement ids. |
victor@7 | 45 | |
victor@7 | 46 | |
victor@7 | 47 | class SCC: |
ioannis@67 | 48 | """A class that will attempt to interact SCC server. |
i@14 | 49 | |
ioannis@67 | 50 | Most interactions are by simulating a normal browser session. In the current |
ioannis@67 | 51 | version few checks are performed before upload a file, and no feedback is given in case the upload |
ioannis@67 | 52 | fails. |
victor@7 | 53 | """ |
victor@7 | 54 | |
i@14 | 55 | def __init__(self, auth, output_dir, base_url): |
moritz@29 | 56 | |
victor@7 | 57 | self.auth = auth |
victor@7 | 58 | self.output_dir = output_dir |
i@14 | 59 | self.base_url = base_url |
victor@7 | 60 | self.session = requests.Session() |
moritz@29 | 61 | self.session.auth = auth |
moritz@29 | 62 | self.session.verify = False |
victor@7 | 63 | |
ioannis@67 | 64 | # Setup SCC server URLS for later use |
i@14 | 65 | self.login_url = urlparse.urljoin(self.base_url, 'accounts/login/') |
i@31 | 66 | self.logout_url = urlparse.urljoin(self.base_url, 'accounts/logout/') |
i@31 | 67 | self.list_measurements_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/') |
i@31 | 68 | |
i@14 | 69 | self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/') |
ioannis@54 | 70 | self.measurement_page_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/') |
ioannis@43 | 71 | self.download_hirelpp_pattern = urlparse.urljoin(self.base_url, |
ioannis@67 | 72 | 'data_processing/measurements/{0}/download-hirelpp/') |
ioannis@43 | 73 | self.download_cloudmask_pattern = urlparse.urljoin(self.base_url, |
ioannis@67 | 74 | 'data_processing/measurements/{0}/download-cloudmask/') |
ioannis@34 | 75 | |
ioannis@43 | 76 | self.download_elpp_pattern = urlparse.urljoin(self.base_url, |
ioannis@67 | 77 | 'data_processing/measurements/{0}/download-preprocessed/') |
ioannis@43 | 78 | self.download_elda_pattern = urlparse.urljoin(self.base_url, |
ioannis@67 | 79 | 'data_processing/measurements/{0}/download-optical/') |
ioannis@45 | 80 | self.download_plots_pattern = urlparse.urljoin(self.base_url, |
moritz@29 | 81 | 'data_processing/measurements/{0}/download-plots/') |
ioannis@43 | 82 | self.download_elic_pattern = urlparse.urljoin(self.base_url, |
ioannis@67 | 83 | 'data_processing/measurements/{0}/download-elic/') |
i@14 | 84 | self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/') |
i@31 | 85 | |
ioannis@67 | 86 | # Setup API URLs for later use |
i@14 | 87 | self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/') |
i@31 | 88 | self.api_measurement_pattern = urlparse.urljoin(self.api_base_url, 'measurements/{0}/') |
i@31 | 89 | self.api_measurements_url = urlparse.urljoin(self.api_base_url, 'measurements') |
i@38 | 90 | self.api_sounding_search_pattern = urlparse.urljoin(self.api_base_url, 'sounding_files/?filename={0}') |
i@38 | 91 | self.api_lidarratio_search_pattern = urlparse.urljoin(self.api_base_url, 'lidarratio_files/?filename={0}') |
i@38 | 92 | self.api_overlap_search_pattern = urlparse.urljoin(self.api_base_url, 'overlap_files/?filename={0}') |
ioannis@43 | 93 | |
i@14 | 94 | def login(self, credentials): |
ioannis@67 | 95 | """ Login to the SCC. |
ioannis@67 | 96 | |
ioannis@67 | 97 | Parameters |
ioannis@67 | 98 | ---------- |
ioannis@67 | 99 | credentials : tuple or list |
ioannis@67 | 100 | A list or tuple in the form (username, password). |
ioannis@67 | 101 | """ |
victor@7 | 102 | logger.debug("Attempting to login to SCC, username %s." % credentials[0]) |
moritz@29 | 103 | login_credentials = {'username': credentials[0], |
moritz@29 | 104 | 'password': credentials[1]} |
victor@7 | 105 | |
i@14 | 106 | logger.debug("Accessing login page at %s." % self.login_url) |
victor@7 | 107 | |
ioannis@67 | 108 | # Get login form |
moritz@29 | 109 | login_page = self.session.get(self.login_url) |
i@14 | 110 | |
i@31 | 111 | if not login_page.ok: |
i@31 | 112 | raise self.PageNotAccessibleError('Could not access login pages. Status code %s' % login_page.status_code) |
victor@7 | 113 | |
i@31 | 114 | logger.debug("Submitting credentials.") |
victor@7 | 115 | # Submit the login data |
i@14 | 116 | login_submit = self.session.post(self.login_url, |
moritz@29 | 117 | data=login_credentials, |
victor@7 | 118 | headers={'X-CSRFToken': login_page.cookies['csrftoken'], |
moritz@29 | 119 | 'referer': self.login_url}) |
victor@7 | 120 | return login_submit |
victor@7 | 121 | |
victor@7 | 122 | def logout(self): |
ioannis@67 | 123 | """ Logout from the SCC """ |
i@31 | 124 | return self.session.get(self.logout_url, stream=True) |
victor@7 | 125 | |
ioannis@67 | 126 | def upload_file(self, filename, system_id, force_upload, delete_related, delay=0, rs_filename=None, |
ioannis@67 | 127 | ov_filename=None, lr_filename=None): |
ioannis@67 | 128 | """ Upload a file for processing. |
ioannis@67 | 129 | |
ioannis@67 | 130 | If the upload is successful, it returns the measurement id. |
ioannis@67 | 131 | |
ioannis@67 | 132 | |
ioannis@67 | 133 | Parameters |
ioannis@67 | 134 | ---------- |
ioannis@67 | 135 | filename : str |
ioannis@67 | 136 | File path of the file to upload |
ioannis@67 | 137 | system_id : int |
ioannis@67 | 138 | System id to be used in the processing |
ioannis@67 | 139 | force_upload : bool |
ioannis@67 | 140 | If True, if a measurement with the same ID is found on the server, it will be first deleted and the |
ioannis@67 | 141 | file current file will be uploaded. If False, the file will not be uploaded if the measurement ID is |
ioannis@67 | 142 | already present on the SCC server. |
ioannis@67 | 143 | delete_related : bool |
ioannis@67 | 144 | Answer to delete related question when deleting existing measurements from the SCC. |
ioannis@67 | 145 | rs_filename, ov_filename, lr_filename : str |
ioannis@67 | 146 | Ancillary files pahts to be uploaded. |
ioannis@67 | 147 | """ |
ioannis@67 | 148 | # Get the measurement ID from the netcdf file |
ioannis@43 | 149 | measurement_id = self.measurement_id_from_file(filename) |
ioannis@43 | 150 | |
ioannis@67 | 151 | # Handle possible existing measurements with the same ID on the SCC server. |
ioannis@43 | 152 | logger.debug('Checking if a measurement with the same id already exists on the SCC server.') |
ioannis@50 | 153 | existing_measurement, _ = self.get_measurement(measurement_id) |
ioannis@43 | 154 | |
ioannis@43 | 155 | if existing_measurement: |
ioannis@43 | 156 | if force_upload: |
ioannis@43 | 157 | logger.info( |
ioannis@43 | 158 | "Measurement with id {} already exists on the SCC. Trying to delete it...".format(measurement_id)) |
ioannis@43 | 159 | self.delete_measurement(measurement_id, delete_related) |
ioannis@43 | 160 | else: |
ioannis@43 | 161 | logger.error( |
ioannis@43 | 162 | "Measurement with id {} already exists on the SCC. Use --force_upload flag to overwrite it.".format( |
ioannis@43 | 163 | measurement_id)) |
ioannis@67 | 164 | # TODO: Implement handling at the proper place. Exiting here does not allow the SCC class to be |
ioannis@67 | 165 | # used by external programs. Instead an exception should be raised. |
ioannis@43 | 166 | sys.exit(1) |
ioannis@43 | 167 | |
ioannis@67 | 168 | # Upload the file(s) |
ioannis@67 | 169 | |
victor@7 | 170 | # Get submit page |
moritz@29 | 171 | upload_page = self.session.get(self.upload_url) |
victor@7 | 172 | |
victor@7 | 173 | # Submit the data |
ioannis@54 | 174 | upload_data = {'system': system_id, |
ioannis@54 | 175 | 'delay': delay} |
ioannis@54 | 176 | |
ioannis@54 | 177 | logger.debug("Submitted processing parameters - System: {}, Delay: {}".format(system_id, delay)) |
ioannis@54 | 178 | |
victor@7 | 179 | files = {'data': open(filename, 'rb')} |
victor@7 | 180 | |
ioannis@67 | 181 | # Add ancillary files to be uploaded |
madrouin@20 | 182 | if rs_filename is not None: |
i@38 | 183 | ancillary_file, _ = self.get_ancillary(rs_filename, 'sounding') |
i@38 | 184 | |
i@38 | 185 | if ancillary_file.already_on_scc: |
ioannis@67 | 186 | logger.warning( |
ioannis@67 | 187 | "Sounding file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(ancillary_file)) |
i@38 | 188 | else: |
i@38 | 189 | logger.debug('Adding sounding file %s' % rs_filename) |
i@38 | 190 | files['sounding_file'] = open(rs_filename, 'rb') |
madrouin@20 | 191 | |
ioannis@32 | 192 | if ov_filename is not None: |
i@38 | 193 | ancillary_file, _ = self.get_ancillary(ov_filename, 'overlap') |
i@38 | 194 | |
i@38 | 195 | if ancillary_file.already_on_scc: |
ioannis@67 | 196 | logger.warning( |
ioannis@67 | 197 | "Overlap file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(ancillary_file)) |
i@38 | 198 | else: |
i@38 | 199 | logger.debug('Adding overlap file %s' % ov_filename) |
i@38 | 200 | files['overlap_file'] = open(ov_filename, 'rb') |
i@31 | 201 | |
i@31 | 202 | if lr_filename is not None: |
i@38 | 203 | ancillary_file, _ = self.get_ancillary(lr_filename, 'lidarratio') |
i@38 | 204 | |
i@38 | 205 | if ancillary_file.already_on_scc: |
i@38 | 206 | logger.warning( |
ioannis@67 | 207 | "Lidar ratio file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format( |
ioannis@67 | 208 | ancillary_file)) |
i@38 | 209 | else: |
i@38 | 210 | logger.debug('Adding lidar ratio file %s' % lr_filename) |
i@38 | 211 | files['lidar_ratio_file'] = open(lr_filename, 'rb') |
i@31 | 212 | |
ioannis@67 | 213 | # Upload the files |
i@14 | 214 | logger.info("Uploading of file %s started." % filename) |
victor@7 | 215 | |
i@14 | 216 | upload_submit = self.session.post(self.upload_url, |
victor@7 | 217 | data=upload_data, |
victor@7 | 218 | files=files, |
victor@7 | 219 | headers={'X-CSRFToken': upload_page.cookies['csrftoken'], |
moritz@29 | 220 | 'referer': self.upload_url}) |
victor@7 | 221 | |
victor@7 | 222 | if upload_submit.status_code != 200: |
i@14 | 223 | logger.warning("Connection error. Status code: %s" % upload_submit.status_code) |
victor@7 | 224 | return False |
victor@7 | 225 | |
ioannis@67 | 226 | # Check if there was a redirect to a new page. If not, something went wrong |
i@14 | 227 | if upload_submit.url == self.upload_url: |
victor@7 | 228 | measurement_id = False |
ioannis@32 | 229 | logger.error("Uploaded file(s) rejected! Try to upload manually to see the error.") |
victor@7 | 230 | else: |
ioannis@67 | 231 | # TODO: Check if this is needed. This was used when the measurement ID was not read from the input file. |
ioannis@67 | 232 | measurement_id = re.findall(regex, upload_submit.text)[0] # Get the measurement ID from the output page |
ioannis@43 | 233 | logger.info("Successfully uploaded measurement with id %s." % measurement_id) |
ioannis@67 | 234 | logger.info("You can monitor the processing progress online: {}".format( |
ioannis@67 | 235 | self.measurement_page_pattern.format(measurement_id))) |
ioannis@43 | 236 | return measurement_id |
ioannis@43 | 237 | |
ioannis@43 | 238 | @staticmethod |
ioannis@43 | 239 | def measurement_id_from_file(filename): |
ioannis@67 | 240 | """ Get the measurement id from the input file. |
ioannis@67 | 241 | |
ioannis@67 | 242 | Parameters |
ioannis@67 | 243 | ---------- |
ioannis@67 | 244 | filename : str |
ioannis@67 | 245 | File path of the input file. |
ioannis@67 | 246 | """ |
ioannis@43 | 247 | |
ioannis@43 | 248 | if not os.path.isfile(filename): |
ioannis@43 | 249 | logger.error("File {} does not exist.".format(filename)) |
ioannis@43 | 250 | sys.exit(1) |
ioannis@43 | 251 | |
ioannis@43 | 252 | with netcdf.Dataset(filename) as f: |
ioannis@43 | 253 | try: |
ioannis@43 | 254 | measurement_id = f.Measurement_ID |
ioannis@43 | 255 | except AttributeError: |
ioannis@43 | 256 | logger.error( |
ioannis@43 | 257 | "Input file {} does not contain a Measurement_ID global attribute. Wrong file format?".format( |
ioannis@43 | 258 | filename)) |
ioannis@43 | 259 | sys.exit(1) |
victor@7 | 260 | |
victor@7 | 261 | return measurement_id |
victor@7 | 262 | |
victor@7 | 263 | def download_files(self, measurement_id, subdir, download_url): |
victor@7 | 264 | """ Downloads some files from the download_url to the specified |
ioannis@67 | 265 | subdir. |
ioannis@67 | 266 | |
ioannis@67 | 267 | This is a general method used to download preprocessed file, optical |
ioannis@67 | 268 | files by other, file-specific, methods. |
victor@7 | 269 | """ |
ioannis@32 | 270 | # TODO: Make downloading more robust (e.g. in case that files do not exist on server). |
victor@7 | 271 | # Get the file |
moritz@29 | 272 | request = self.session.get(download_url, stream=True) |
moritz@29 | 273 | |
moritz@29 | 274 | if not request.ok: |
moritz@29 | 275 | raise Exception("Could not download files for measurement '%s'" % measurement_id) |
victor@7 | 276 | |
victor@7 | 277 | # Create the dir if it does not exist |
victor@7 | 278 | local_dir = os.path.join(self.output_dir, measurement_id, subdir) |
victor@7 | 279 | if not os.path.exists(local_dir): |
victor@7 | 280 | os.makedirs(local_dir) |
victor@7 | 281 | |
victor@7 | 282 | # Save the file by chunk, needed if the file is big. |
ioannis@43 | 283 | memory_file = BytesIO() |
victor@7 | 284 | |
victor@7 | 285 | for chunk in request.iter_content(chunk_size=1024): |
victor@7 | 286 | if chunk: # filter out keep-alive new chunks |
victor@7 | 287 | memory_file.write(chunk) |
victor@7 | 288 | memory_file.flush() |
victor@7 | 289 | |
victor@7 | 290 | zip_file = ZipFile(memory_file) |
victor@7 | 291 | |
victor@7 | 292 | for ziped_name in zip_file.namelist(): |
victor@7 | 293 | basename = os.path.basename(ziped_name) |
victor@7 | 294 | |
victor@7 | 295 | local_file = os.path.join(local_dir, basename) |
victor@7 | 296 | |
victor@7 | 297 | with open(local_file, 'wb') as f: |
victor@7 | 298 | f.write(zip_file.read(ziped_name)) |
victor@7 | 299 | |
ioannis@43 | 300 | def download_hirelpp(self, measurement_id): |
ioannis@43 | 301 | """ Download hirelpp files for the measurement id. """ |
ioannis@43 | 302 | # Construct the download url |
ioannis@43 | 303 | download_url = self.download_hirelpp_pattern.format(measurement_id) |
ioannis@43 | 304 | try: |
ioannis@50 | 305 | self.download_files(measurement_id, 'hirelpp', download_url) |
ioannis@43 | 306 | except Exception as e: |
ioannis@43 | 307 | logger.error("Could not download HiRElPP files. Error message: {}".format(e)) |
ioannis@43 | 308 | logger.debug('Download exception:', exc_info=True) |
ioannis@43 | 309 | |
ioannis@43 | 310 | def download_cloudmask(self, measurement_id): |
ioannis@43 | 311 | """ Download cloudmask files for the measurement id. """ |
ioannis@43 | 312 | # Construct the download url |
ioannis@43 | 313 | download_url = self.download_cloudmask_pattern.format(measurement_id) |
ioannis@43 | 314 | try: |
ioannis@50 | 315 | self.download_files(measurement_id, 'cloudscreen', download_url) |
ioannis@43 | 316 | except Exception as e: |
ioannis@43 | 317 | logger.error("Could not download cloudscreen files. Error message: {}".format(e)) |
ioannis@43 | 318 | logger.debug('Download exception:', exc_info=True) |
ioannis@43 | 319 | |
ioannis@43 | 320 | def download_elpp(self, measurement_id): |
victor@7 | 321 | """ Download preprocessed files for the measurement id. """ |
victor@7 | 322 | # Construct the download url |
ioannis@43 | 323 | download_url = self.download_elpp_pattern.format(measurement_id) |
ioannis@43 | 324 | try: |
ioannis@50 | 325 | self.download_files(measurement_id, 'elpp', download_url) |
ioannis@43 | 326 | except Exception as e: |
ioannis@43 | 327 | logger.error("Could not download ElPP files. Error message: {}".format(e)) |
ioannis@43 | 328 | logger.debug('Download exception:', exc_info=True) |
victor@7 | 329 | |
ioannis@43 | 330 | def download_elda(self, measurement_id): |
victor@7 | 331 | """ Download optical files for the measurement id. """ |
victor@7 | 332 | # Construct the download url |
ioannis@43 | 333 | download_url = self.download_elda_pattern.format(measurement_id) |
ioannis@43 | 334 | try: |
ioannis@50 | 335 | self.download_files(measurement_id, 'elda', download_url) |
ioannis@43 | 336 | except Exception as e: |
ioannis@43 | 337 | logger.error("Could not download ELDA files. Error message: {}".format(e)) |
ioannis@43 | 338 | logger.debug('Download exception:', exc_info=True) |
victor@7 | 339 | |
ioannis@43 | 340 | def download_plots(self, measurement_id): |
victor@7 | 341 | """ Download profile graphs for the measurement id. """ |
victor@7 | 342 | # Construct the download url |
ioannis@45 | 343 | download_url = self.download_plots_pattern.format(measurement_id) |
ioannis@43 | 344 | try: |
ioannis@50 | 345 | self.download_files(measurement_id, 'elda_plots', download_url) |
ioannis@43 | 346 | except Exception as e: |
ioannis@43 | 347 | logger.error("Could not download ELDA plots. Error message: {}".format(e)) |
ioannis@43 | 348 | logger.debug('Download exception:', exc_info=True) |
victor@7 | 349 | |
ioannis@43 | 350 | def download_elic(self, measurement_id): |
ioannis@43 | 351 | """ Download ELIC files for the measurement id. """ |
ioannis@43 | 352 | # Construct the download url |
ioannis@43 | 353 | download_url = self.download_elic_pattern.format(measurement_id) |
ioannis@43 | 354 | try: |
ioannis@50 | 355 | self.download_files(measurement_id, 'elic', download_url) |
ioannis@43 | 356 | except Exception as e: |
ioannis@43 | 357 | logger.error("Could not download ELIC files. Error message: {}".format(e)) |
ioannis@43 | 358 | logger.debug('Download exception:', exc_info=True) |
ioannis@43 | 359 | |
ioannis@43 | 360 | def download_eldec(self, measurement_id): |
ioannis@43 | 361 | """ Download ELDEC files for the measurement id. """ |
ioannis@43 | 362 | # Construct the download url |
ioannis@43 | 363 | download_url = self.download_elda_pattern.format(measurement_id) # ELDA patter is used for now |
ioannis@43 | 364 | try: |
ioannis@50 | 365 | self.download_files(measurement_id, 'eldec', download_url) |
ioannis@43 | 366 | except Exception as e: |
ioannis@43 | 367 | logger.error("Could not download EDELC files. Error message: {}".format(e)) |
ioannis@43 | 368 | logger.debug('Download exception:', exc_info=True) |
ioannis@43 | 369 | |
ioannis@43 | 370 | def rerun_elpp(self, measurement_id, monitor=True): |
ioannis@45 | 371 | logger.debug("Started rerun_elpp procedure.") |
ioannis@45 | 372 | |
ioannis@45 | 373 | logger.debug("Getting measurement %s" % measurement_id) |
madrouin@26 | 374 | measurement, status = self.get_measurement(measurement_id) |
victor@7 | 375 | |
victor@7 | 376 | if measurement: |
ioannis@45 | 377 | logger.debug("Attempting to rerun ElPP through %s." % measurement.rerun_all_url) |
ioannis@45 | 378 | request = self.session.get(measurement.rerun_elpp_url, stream=True) |
victor@7 | 379 | |
victor@7 | 380 | if request.status_code != 200: |
i@14 | 381 | logger.error( |
i@14 | 382 | "Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code)) |
ioannis@45 | 383 | else: |
ioannis@45 | 384 | logger.info("Rerun-elpp command submitted successfully for id {}.".format(measurement_id)) |
victor@7 | 385 | |
victor@7 | 386 | if monitor: |
victor@7 | 387 | self.monitor_processing(measurement_id) |
victor@7 | 388 | |
victor@7 | 389 | def rerun_all(self, measurement_id, monitor=True): |
victor@7 | 390 | logger.debug("Started rerun_all procedure.") |
victor@7 | 391 | |
victor@7 | 392 | logger.debug("Getting measurement %s" % measurement_id) |
madrouin@26 | 393 | measurement, status = self.get_measurement(measurement_id) |
victor@7 | 394 | |
victor@7 | 395 | if measurement: |
victor@7 | 396 | logger.debug("Attempting to rerun all processing through %s." % measurement.rerun_all_url) |
victor@7 | 397 | |
moritz@29 | 398 | request = self.session.get(measurement.rerun_all_url, stream=True) |
victor@7 | 399 | |
victor@7 | 400 | if request.status_code != 200: |
victor@7 | 401 | logger.error("Could not rerun pre processing for %s. Status code: %s" % |
victor@7 | 402 | (measurement_id, request.status_code)) |
ioannis@45 | 403 | else: |
ioannis@45 | 404 | logger.info("Rerun-all command submitted successfully for id {}.".format(measurement_id)) |
victor@7 | 405 | |
victor@7 | 406 | if monitor: |
victor@7 | 407 | self.monitor_processing(measurement_id) |
victor@7 | 408 | |
ioannis@67 | 409 | def process(self, filename, system_id, monitor, force_upload, delete_related, delay=0, rs_filename=None, |
ioannis@67 | 410 | lr_filename=None, ov_filename=None): |
victor@7 | 411 | """ Upload a file for processing and wait for the processing to finish. |
victor@7 | 412 | If the processing is successful, it will download all produced files. |
victor@7 | 413 | """ |
victor@7 | 414 | logger.info("--- Processing started on %s. ---" % datetime.datetime.now()) |
victor@7 | 415 | # Upload file |
ioannis@50 | 416 | logger.info("Uploading file.") |
ioannis@45 | 417 | measurement_id = self.upload_file(filename, system_id, force_upload, delete_related, |
ioannis@54 | 418 | delay=delay, |
ioannis@32 | 419 | rs_filename=rs_filename, |
ioannis@32 | 420 | lr_filename=lr_filename, |
ioannis@32 | 421 | ov_filename=ov_filename) |
ioannis@43 | 422 | |
ioannis@54 | 423 | if monitor and (delay > 0): |
ioannis@54 | 424 | logger.warning("Will not start monitoring, since a delay was specified: {} hours.".format(delay)) |
ioannis@54 | 425 | return None |
ioannis@54 | 426 | |
ioannis@32 | 427 | if measurement_id and monitor: |
ioannis@54 | 428 | logger.info("Monitoring processing.") |
moritz@29 | 429 | return self.monitor_processing(measurement_id) |
victor@7 | 430 | |
moritz@29 | 431 | return None |
victor@7 | 432 | |
ioannis@61 | 433 | def monitor_processing(self, measurement_id, retry_max=2, time_sleep=2, exit_if_missing=True): |
victor@7 | 434 | """ Monitor the processing progress of a measurement id""" |
victor@7 | 435 | |
madrouin@24 | 436 | # try to deal with error 404 |
ioannis@61 | 437 | attempts_count = 0 |
ioannis@61 | 438 | max_attempts = retry_max + 1 |
madrouin@24 | 439 | |
ioannis@67 | 440 | # try to wait for measurement to appear in API. A user has reported that this does not happen immediately. |
madrouin@24 | 441 | measurement = None |
ioannis@50 | 442 | logger.info("Looking for measurement %s on the SCC.", measurement_id) |
ioannis@62 | 443 | |
ioannis@61 | 444 | while attempts_count < max_attempts: |
ioannis@61 | 445 | attempts_count += 1 |
madrouin@26 | 446 | measurement, status = self.get_measurement(measurement_id) |
ioannis@61 | 447 | if status != 200: |
ioannis@61 | 448 | logger.warning("Measurement not found.") |
ioannis@61 | 449 | if attempts_count < max_attempts: |
ioannis@61 | 450 | logger.warning("Waiting %ds.", time_sleep) |
ioannis@61 | 451 | time.sleep(time_sleep) |
madrouin@26 | 452 | else: |
madrouin@26 | 453 | break |
ioannis@62 | 454 | print("Measurement: {}".format(measurement)) |
madrouin@24 | 455 | |
ioannis@62 | 456 | if measurement is None: |
ioannis@61 | 457 | logger.error("Measurement %s doesn't seem to exist.", measurement_id) |
ioannis@61 | 458 | if exit_if_missing: |
ioannis@61 | 459 | sys.exit(1) |
ioannis@61 | 460 | else: |
ioannis@61 | 461 | return measurement |
madrouin@26 | 462 | |
ioannis@45 | 463 | logger.info('Measurement %s found.', measurement_id) |
ioannis@62 | 464 | while not measurement.has_finished: |
ioannis@62 | 465 | measurement.log_processing_status() |
ioannis@62 | 466 | time.sleep(10) |
ioannis@62 | 467 | measurement, status = self.get_measurement(measurement_id) |
madrouin@24 | 468 | |
ioannis@62 | 469 | logger.info("Measurement processing finished.") |
ioannis@62 | 470 | measurement.log_detailed_status() |
ioannis@53 | 471 | |
ioannis@62 | 472 | if measurement.hirelpp == 127: |
ioannis@62 | 473 | logger.info("Downloading HiRElPP files.") |
ioannis@62 | 474 | self.download_hirelpp(measurement_id) |
ioannis@62 | 475 | if measurement.cloudmask == 127: |
ioannis@62 | 476 | logger.info("Downloading cloud screening files.") |
ioannis@62 | 477 | self.download_cloudmask(measurement_id) |
ioannis@62 | 478 | if measurement.elpp == 127: |
ioannis@62 | 479 | logger.info("Downloading ElPP files.") |
ioannis@62 | 480 | self.download_elpp(measurement_id) |
ioannis@62 | 481 | if measurement.elda == 127: |
ioannis@62 | 482 | logger.info("Downloading ELDA files.") |
ioannis@62 | 483 | self.download_elda(measurement_id) |
ioannis@62 | 484 | logger.info("Downloading ELDA plots.") |
ioannis@62 | 485 | self.download_plots(measurement_id) |
ioannis@62 | 486 | if measurement.elic == 127: |
ioannis@62 | 487 | logger.info("Downloading ELIC files.") |
ioannis@62 | 488 | self.download_elic(measurement_id) |
ioannis@67 | 489 | if measurement.is_calibration and measurement.eldec == 0: |
ioannis@62 | 490 | logger.info("Downloading ELDEC files.") |
ioannis@62 | 491 | self.download_eldec(measurement_id) |
ioannis@62 | 492 | logger.info("--- Processing finished. ---") |
ioannis@43 | 493 | |
victor@7 | 494 | return measurement |
victor@7 | 495 | |
ioannis@43 | 496 | def get_measurement(self, measurement_id): |
ioannis@67 | 497 | """ Get a measurement information from the SCC API. |
victor@7 | 498 | |
ioannis@67 | 499 | Parameters |
ioannis@67 | 500 | ---------- |
ioannis@67 | 501 | measurement_id : str |
ioannis@67 | 502 | The measurement ID to search. |
ioannis@67 | 503 | |
ioannis@67 | 504 | Returns |
ioannis@67 | 505 | ------- |
ioannis@67 | 506 | : Measurement object or None |
ioannis@67 | 507 | If the measurement is found, a Measurement object is returned. If not, it returns None |
ioannis@67 | 508 | """ |
ioannis@67 | 509 | # TODO: Consider to homogenize with get_ancillary method (i.e. always return a Measurement object). |
ioannis@67 | 510 | |
ioannis@67 | 511 | if measurement_id is None: # TODO: Is this still required? |
victor@7 | 512 | return None |
victor@7 | 513 | |
ioannis@67 | 514 | # Access the API |
i@31 | 515 | measurement_url = self.api_measurement_pattern.format(measurement_id) |
i@31 | 516 | logger.debug("Measurement API URL: %s" % measurement_url) |
victor@7 | 517 | |
moritz@29 | 518 | response = self.session.get(measurement_url) |
victor@7 | 519 | |
ioannis@43 | 520 | response_dict = None |
ioannis@45 | 521 | |
ioannis@43 | 522 | if response.status_code == 200: |
ioannis@43 | 523 | response_dict = response.json() |
ioannis@45 | 524 | elif response.status_code == 404: |
ioannis@43 | 525 | logger.info("No measurement with id %s found on the SCC." % measurement_id) |
ioannis@45 | 526 | else: |
i@14 | 527 | logger.error('Could not access API. Status code %s.' % response.status_code) |
victor@7 | 528 | |
ioannis@55 | 529 | # TODO: Implement better handling for status 401. |
ioannis@55 | 530 | |
victor@7 | 531 | if response_dict: |
ioannis@43 | 532 | measurement = Measurement(self.base_url, response_dict) |
victor@7 | 533 | else: |
ioannis@45 | 534 | measurement = None |
victor@7 | 535 | |
ioannis@45 | 536 | return measurement, response.status_code |
victor@7 | 537 | |
ioannis@45 | 538 | def delete_measurement(self, measurement_id, delete_related): |
victor@7 | 539 | """ Deletes a measurement with the provided measurement id. The user |
ioannis@67 | 540 | should have the appropriate permissions (i.e. access to the admin site). |
madrouin@20 | 541 | |
victor@7 | 542 | The procedures is performed directly through the web interface and |
victor@7 | 543 | NOT through the API. |
victor@7 | 544 | """ |
victor@7 | 545 | # Get the measurement object |
i@31 | 546 | measurement, _ = self.get_measurement(measurement_id) |
victor@7 | 547 | |
victor@7 | 548 | # Check that it exists |
victor@7 | 549 | if measurement is None: |
victor@7 | 550 | logger.warning("Nothing to delete.") |
victor@7 | 551 | return None |
victor@7 | 552 | |
victor@7 | 553 | # Go the the page confirming the deletion |
moritz@29 | 554 | delete_url = self.delete_measurement_pattern.format(measurement_id) |
ioannis@43 | 555 | |
moritz@29 | 556 | confirm_page = self.session.get(delete_url) |
victor@7 | 557 | |
victor@7 | 558 | # Check that the page opened properly |
victor@7 | 559 | if confirm_page.status_code != 200: |
victor@7 | 560 | logger.warning("Could not open delete page. Status: {0}".format(confirm_page.status_code)) |
victor@7 | 561 | return None |
victor@7 | 562 | |
ioannis@43 | 563 | # Get the delete related value |
ioannis@43 | 564 | if delete_related: |
ioannis@43 | 565 | delete_related_option = 'delete_related' |
ioannis@43 | 566 | else: |
ioannis@43 | 567 | delete_related_option = 'not_delete_related' |
ioannis@43 | 568 | |
victor@7 | 569 | # Delete the measurement |
victor@7 | 570 | delete_page = self.session.post(delete_url, |
ioannis@43 | 571 | data={'post': 'yes', |
ioannis@43 | 572 | 'select_delete_related_measurements': delete_related_option}, |
victor@7 | 573 | headers={'X-CSRFToken': confirm_page.cookies['csrftoken'], |
victor@7 | 574 | 'referer': delete_url} |
victor@7 | 575 | ) |
i@31 | 576 | if not delete_page.ok: |
victor@7 | 577 | logger.warning("Something went wrong. Delete page status: {0}".format( |
victor@7 | 578 | delete_page.status_code)) |
victor@7 | 579 | return None |
victor@7 | 580 | |
victor@7 | 581 | logger.info("Deleted measurement {0}".format(measurement_id)) |
victor@7 | 582 | return True |
victor@7 | 583 | |
victor@7 | 584 | def available_measurements(self): |
ioannis@67 | 585 | """ Get a list of available measurement on the SCC. |
ioannis@67 | 586 | |
ioannis@67 | 587 | The methods is currently not used, could be merged with list_measurements. |
ioannis@67 | 588 | """ |
i@31 | 589 | response = self.session.get(self.api_measurements_url) |
victor@7 | 590 | response_dict = response.json() |
victor@7 | 591 | |
victor@7 | 592 | if response_dict: |
victor@7 | 593 | measurement_list = response_dict['objects'] |
i@14 | 594 | measurements = [Measurement(self.base_url, measurement_dict) for measurement_dict in measurement_list] |
victor@7 | 595 | logger.info("Found %s measurements on the SCC." % len(measurements)) |
victor@7 | 596 | else: |
victor@7 | 597 | logger.warning("No response received from the SCC when asked for available measurements.") |
ioannis@43 | 598 | measurements = None |
victor@7 | 599 | |
victor@7 | 600 | return measurements |
victor@7 | 601 | |
ioannis@65 | 602 | def list_measurements(self, id_exact=None, id_startswith=None): |
ioannis@67 | 603 | """ Get the response text from the API. """ |
moritz@29 | 604 | |
ioannis@67 | 605 | # TODO: Add some error handling, e.g. as per available_measurements method |
i@31 | 606 | |
moritz@29 | 607 | # Need to set to empty string if not specified, we won't get any results |
ioannis@65 | 608 | params = {} |
i@31 | 609 | |
ioannis@65 | 610 | if id_exact is not None: |
ioannis@65 | 611 | params['id__exact'] = id_exact |
ioannis@65 | 612 | else: |
ioannis@65 | 613 | params['id__startswith'] = id_startswith |
moritz@29 | 614 | |
ioannis@65 | 615 | response_json = self.session.get(self.api_measurements_url, params=params).text |
moritz@29 | 616 | |
ioannis@65 | 617 | return response_json |
moritz@29 | 618 | |
ioannis@43 | 619 | def measurement_id_for_date(self, t1, call_sign, base_number=0): |
victor@7 | 620 | """ Give the first available measurement id on the SCC for the specific |
madrouin@20 | 621 | date. |
victor@7 | 622 | """ |
ioannis@67 | 623 | # TODO: Check if this method needs updating to handle all measurement_ID formats. |
victor@7 | 624 | date_str = t1.strftime('%Y%m%d') |
i@31 | 625 | base_id = "%s%s" % (date_str, call_sign) |
i@31 | 626 | search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % base_id) |
victor@7 | 627 | |
moritz@29 | 628 | response = self.session.get(search_url) |
victor@7 | 629 | |
victor@7 | 630 | response_dict = response.json() |
victor@7 | 631 | |
victor@7 | 632 | measurement_id = None |
victor@7 | 633 | |
victor@7 | 634 | if response_dict: |
victor@7 | 635 | measurement_list = response_dict['objects'] |
i@31 | 636 | |
i@31 | 637 | if len(measurement_list) == 100: |
i@31 | 638 | raise ValueError('No available measurement id found.') |
i@31 | 639 | |
victor@7 | 640 | existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list] |
victor@7 | 641 | |
victor@7 | 642 | measurement_number = base_number |
i@31 | 643 | measurement_id = "%s%02i" % (base_id, measurement_number) |
victor@7 | 644 | |
victor@7 | 645 | while measurement_id in existing_ids: |
victor@7 | 646 | measurement_number = measurement_number + 1 |
i@31 | 647 | measurement_id = "%s%02i" % (base_id, measurement_number) |
victor@7 | 648 | |
victor@7 | 649 | return measurement_id |
victor@7 | 650 | |
i@40 | 651 | def get_ancillary(self, file_path, file_type): |
i@38 | 652 | """ |
i@38 | 653 | Try to get the ancillary file data from the SCC API. |
i@38 | 654 | |
i@38 | 655 | The result will always be an API object. If the file does not exist, the .exists property is set to False. |
i@38 | 656 | |
i@38 | 657 | Parameters |
i@38 | 658 | ---------- |
i@40 | 659 | file_path : str |
i@40 | 660 | Path of the uploaded file. |
i@38 | 661 | file_type : str |
i@38 | 662 | Type of ancillary file. One of 'sounding', 'overlap', 'lidarratio'. |
i@38 | 663 | |
i@38 | 664 | Returns |
i@38 | 665 | : AncillaryFile |
i@38 | 666 | The api object. |
i@38 | 667 | """ |
i@38 | 668 | assert file_type in ['sounding', 'overlap', 'lidarratio'] |
i@38 | 669 | |
i@40 | 670 | filename = os.path.basename(file_path) |
i@40 | 671 | |
i@38 | 672 | if file_type == 'sounding': |
i@38 | 673 | file_url = self.api_sounding_search_pattern.format(filename) |
i@38 | 674 | elif file_type == 'overlap': |
i@38 | 675 | file_url = self.api_overlap_search_pattern.format(filename) |
i@38 | 676 | else: |
i@38 | 677 | file_url = self.api_lidarratio_search_pattern.format(filename) |
i@38 | 678 | |
i@38 | 679 | response = self.session.get(file_url) |
i@38 | 680 | |
i@38 | 681 | if not response.ok: |
i@38 | 682 | logger.error('Could not access API. Status code %s.' % response.status_code) |
i@38 | 683 | return None, response.status_code |
i@38 | 684 | |
i@38 | 685 | response_dict = response.json() |
i@38 | 686 | object_list = response_dict['objects'] |
i@38 | 687 | |
i@38 | 688 | logger.debug("Ancillary file JSON: {0}".format(object_list)) |
i@38 | 689 | |
i@38 | 690 | if object_list: |
i@38 | 691 | ancillary_file = AncillaryFile(self.api_base_url, object_list[0]) # Assume only one file is returned |
i@38 | 692 | else: |
i@38 | 693 | ancillary_file = AncillaryFile(self.api_base_url, None) # Create an empty object |
i@38 | 694 | |
i@38 | 695 | return ancillary_file, response.status_code |
i@38 | 696 | |
ioannis@43 | 697 | def __enter__(self): |
ioannis@43 | 698 | return self |
victor@7 | 699 | |
ioannis@43 | 700 | def __exit__(self, *args): |
ioannis@43 | 701 | logger.debug("Closing SCC connection session.") |
ioannis@43 | 702 | self.session.close() |
ioannis@43 | 703 | |
i@31 | 704 | class PageNotAccessibleError(RuntimeError): |
i@31 | 705 | pass |
ioannis@43 | 706 | |
victor@7 | 707 | |
ioannis@17 | 708 | class ApiObject(object): |
victor@7 | 709 | """ A generic class object. """ |
victor@7 | 710 | |
i@14 | 711 | def __init__(self, base_url, dict_response): |
i@14 | 712 | self.base_url = base_url |
victor@7 | 713 | |
victor@7 | 714 | if dict_response: |
victor@7 | 715 | # Add the dictionary key value pairs as object properties |
victor@7 | 716 | for key, value in dict_response.items(): |
ioannis@17 | 717 | # logger.debug('Setting key {0} to value {1}'.format(key, value)) |
ioannis@17 | 718 | try: |
ioannis@17 | 719 | setattr(self, key, value) |
ioannis@17 | 720 | except: |
ioannis@17 | 721 | logger.warning('Could not set attribute {0} to value {1}'.format(key, value)) |
victor@7 | 722 | self.exists = True |
victor@7 | 723 | else: |
victor@7 | 724 | self.exists = False |
victor@7 | 725 | |
victor@7 | 726 | |
victor@7 | 727 | class Measurement(ApiObject): |
ioannis@43 | 728 | """ This class represents the measurement object as returned in the SCC API. |
ioannis@43 | 729 | """ |
victor@7 | 730 | |
i@14 | 731 | def __init__(self, base_url, dict_response): |
victor@7 | 732 | |
ioannis@45 | 733 | # Define expected attributes to assist debugging |
ioannis@53 | 734 | |
ioannis@53 | 735 | self.hirelpp = None |
ioannis@53 | 736 | self.hirelpp_exit_code = None |
ioannis@43 | 737 | self.cloudmask = None |
ioannis@53 | 738 | self.cloudmask_exit_code = None |
ioannis@53 | 739 | self.elpp = None |
ioannis@53 | 740 | self.elpp_exit_code = None |
ioannis@43 | 741 | self.elda = None |
ioannis@53 | 742 | self.elda_exit_code = None |
ioannis@43 | 743 | self.elic = None |
ioannis@53 | 744 | self.elic_exit_code = None |
ioannis@53 | 745 | self.eldec = None |
ioannis@53 | 746 | self.eldec_exit_code = None |
ioannis@53 | 747 | self.elquick = None |
ioannis@53 | 748 | self.elquick_exit_code = None |
ioannis@53 | 749 | |
ioannis@43 | 750 | self.id = None |
ioannis@43 | 751 | self.is_calibration = None |
ioannis@43 | 752 | self.is_running = None |
ioannis@53 | 753 | |
ioannis@43 | 754 | self.resource_uri = None |
ioannis@43 | 755 | self.start = None |
ioannis@43 | 756 | self.stop = None |
ioannis@43 | 757 | self.system = None |
ioannis@43 | 758 | self.upload = None |
ioannis@43 | 759 | |
ioannis@45 | 760 | super().__init__(base_url, dict_response) |
victor@7 | 761 | |
ioannis@57 | 762 | @property |
ioannis@57 | 763 | def has_finished(self): |
ioannis@57 | 764 | """ Temporary implementation for SCC version 5.2.0, until the API returns a flag indicating if the |
ioannis@57 | 765 | processing measurement has finished. """ |
ioannis@57 | 766 | if (self.is_running is False) and (self.hirelpp != 0 or self.elpp != 0): |
ioannis@57 | 767 | return True |
ioannis@57 | 768 | else: |
ioannis@57 | 769 | return False |
ioannis@57 | 770 | |
ioannis@53 | 771 | def log_processing_status(self): |
ioannis@53 | 772 | """ Log module status. """ |
ioannis@53 | 773 | logger.info("Measurement is being processed. Status: {}, {}, {}, {}, {}, {}). Please wait.".format( |
ioannis@53 | 774 | self.upload, |
ioannis@53 | 775 | self.hirelpp, |
ioannis@53 | 776 | self.cloudmask, |
ioannis@53 | 777 | self.elpp, |
ioannis@53 | 778 | self.elda, |
ioannis@53 | 779 | self.elic)) |
ioannis@53 | 780 | |
ioannis@53 | 781 | def log_detailed_status(self): |
ioannis@53 | 782 | """ Log module exit and status codes.""" |
ioannis@53 | 783 | logger.info("Measurement exit status:".format(self.id)) |
ioannis@53 | 784 | if self.is_calibration: |
ioannis@53 | 785 | self._log_module_status('ElPP', self.elpp, self.elpp_exit_code) |
ioannis@53 | 786 | self._log_module_status('ElDEC', self.eldec, self.eldec_exit_code) |
ioannis@53 | 787 | else: |
ioannis@53 | 788 | self._log_module_status('HiRElPP', self.hirelpp, self.hirelpp_exit_code) |
ioannis@53 | 789 | self._log_module_status('CloudScreen', self.cloudmask, self.cloudmask_exit_code) |
ioannis@53 | 790 | self._log_module_status('ElPP', self.elpp, self.elpp_exit_code) |
ioannis@53 | 791 | self._log_module_status('ELDA', self.elda, self.elda_exit_code) |
ioannis@53 | 792 | self._log_module_status('ELIC', self.elic, self.elic_exit_code) |
ioannis@53 | 793 | self._log_module_status('ELQuick', self.elquick, self.elquick_exit_code) |
ioannis@53 | 794 | |
ioannis@53 | 795 | def _log_module_status(self, name, status, exit_code): |
ioannis@53 | 796 | if exit_code: |
ioannis@54 | 797 | if exit_code['exit_code'] > 0: |
ioannis@54 | 798 | logger.warning("{0} exit code: {2[exit_code]} - {2[description]}".format(name, status, exit_code)) |
ioannis@54 | 799 | else: |
ioannis@54 | 800 | logger.info("{0} exit code: {2[exit_code]} - {2[description]}".format(name, status, exit_code)) |
ioannis@53 | 801 | else: |
ioannis@53 | 802 | logger.info("{0} exit code: {2}".format(name, status, exit_code)) |
ioannis@53 | 803 | |
ioannis@43 | 804 | @property |
ioannis@43 | 805 | def rerun_elda_url(self): |
ioannis@43 | 806 | url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-elda/') |
ioannis@43 | 807 | return url_pattern.format(self.id) |
victor@7 | 808 | |
victor@7 | 809 | @property |
ioannis@43 | 810 | def rerun_elpp_url(self): |
ioannis@43 | 811 | url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-elpp/') |
i@14 | 812 | return url_pattern.format(self.id) |
victor@7 | 813 | |
victor@7 | 814 | @property |
victor@7 | 815 | def rerun_all_url(self): |
i@14 | 816 | ulr_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-all/') |
i@14 | 817 | return ulr_pattern.format(self.id) |
victor@7 | 818 | |
victor@7 | 819 | def __str__(self): |
ioannis@43 | 820 | return "Measurement {}".format(self.id) |
victor@7 | 821 | |
victor@7 | 822 | |
i@38 | 823 | class AncillaryFile(ApiObject): |
i@38 | 824 | """ This class represents the ancilalry file object as returned in the SCC API. |
i@38 | 825 | """ |
ioannis@67 | 826 | |
i@38 | 827 | @property |
i@38 | 828 | def already_on_scc(self): |
i@38 | 829 | if self.exists is False: |
i@38 | 830 | return False |
i@38 | 831 | |
i@38 | 832 | return not self.status == 'missing' |
i@38 | 833 | |
i@38 | 834 | def __str__(self): |
i@38 | 835 | return "%s: %s, %s" % (self.id, |
i@38 | 836 | self.filename, |
i@38 | 837 | self.status) |
i@38 | 838 | |
i@38 | 839 | |
ioannis@67 | 840 | # Methods that use the SCC class to perform specific tasks. |
ioannis@45 | 841 | def process_file(filename, system_id, settings, force_upload, delete_related, |
ioannis@54 | 842 | delay=0, monitor=True, rs_filename=None, lr_filename=None, ov_filename=None): |
ioannis@32 | 843 | """ Shortcut function to process a file to the SCC. """ |
ioannis@32 | 844 | logger.info("Processing file %s, using system %s" % (filename, system_id)) |
victor@7 | 845 | |
ioannis@43 | 846 | with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: |
ioannis@43 | 847 | scc.login(settings['website_credentials']) |
ioannis@45 | 848 | measurement = scc.process(filename, system_id, |
ioannis@45 | 849 | force_upload=force_upload, |
ioannis@45 | 850 | delete_related=delete_related, |
ioannis@54 | 851 | delay=delay, |
ioannis@45 | 852 | monitor=monitor, |
ioannis@45 | 853 | rs_filename=rs_filename, |
ioannis@45 | 854 | lr_filename=lr_filename, |
ioannis@45 | 855 | ov_filename=ov_filename) |
ioannis@43 | 856 | scc.logout() |
victor@7 | 857 | return measurement |
victor@7 | 858 | |
victor@7 | 859 | |
ioannis@45 | 860 | def delete_measurements(measurement_ids, delete_related, settings): |
moritz@29 | 861 | """ Shortcut function to delete measurements from the SCC. """ |
ioannis@45 | 862 | with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: |
ioannis@45 | 863 | scc.login(settings['website_credentials']) |
ioannis@45 | 864 | for m_id in measurement_ids: |
ioannis@54 | 865 | logger.info("Deleting %s." % m_id) |
ioannis@45 | 866 | scc.delete_measurement(m_id, delete_related) |
ioannis@45 | 867 | scc.logout() |
moritz@29 | 868 | |
moritz@29 | 869 | |
moritz@29 | 870 | def rerun_all(measurement_ids, monitor, settings): |
moritz@29 | 871 | """ Shortcut function to rerun measurements from the SCC. """ |
i@14 | 872 | |
ioannis@43 | 873 | with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: |
ioannis@43 | 874 | scc.login(settings['website_credentials']) |
ioannis@45 | 875 | for m_id in measurement_ids: |
ioannis@54 | 876 | logger.info("Rerunning all products for %s." % m_id) |
ioannis@45 | 877 | scc.rerun_all(m_id, monitor) |
ioannis@43 | 878 | scc.logout() |
victor@7 | 879 | |
victor@7 | 880 | |
moritz@29 | 881 | def rerun_processing(measurement_ids, monitor, settings): |
victor@7 | 882 | """ Shortcut function to delete a measurement from the SCC. """ |
i@14 | 883 | |
ioannis@43 | 884 | with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: |
ioannis@43 | 885 | scc.login(settings['website_credentials']) |
ioannis@45 | 886 | for m_id in measurement_ids: |
ioannis@45 | 887 | logger.info("Rerunning (optical) processing for %s" % m_id) |
ioannis@45 | 888 | scc.rerun_elpp(m_id, monitor) |
ioannis@45 | 889 | scc.logout() |
moritz@29 | 890 | |
moritz@29 | 891 | |
ioannis@65 | 892 | def list_measurements(settings, id_exact=None, id_startswith=None): |
moritz@29 | 893 | """List all available measurements""" |
ioannis@45 | 894 | with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: |
ioannis@45 | 895 | scc.login(settings['website_credentials']) |
ioannis@65 | 896 | |
ioannis@65 | 897 | results_json = scc.list_measurements(id_exact=id_exact, id_startswith=id_startswith) |
ioannis@65 | 898 | print(results_json) |
ioannis@65 | 899 | |
ioannis@43 | 900 | scc.logout() |
victor@7 | 901 | |
victor@7 | 902 | |
ioannis@61 | 903 | def download_measurements(measurement_ids, max_retries, exit_if_missing, settings): |
moritz@29 | 904 | """Download all measurements for the specified IDs""" |
ioannis@43 | 905 | with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: |
ioannis@43 | 906 | scc.login(settings['website_credentials']) |
ioannis@45 | 907 | for m_id in measurement_ids: |
ioannis@61 | 908 | scc.monitor_processing(m_id, retry_max=max_retries, time_sleep=3, exit_if_missing=exit_if_missing) |
ioannis@61 | 909 | |
ioannis@43 | 910 | scc.logout() |
i@14 | 911 | |
i@14 | 912 | |
moritz@29 | 913 | def settings_from_path(config_file_path): |
i@14 | 914 | """ Read the configuration file. |
i@14 | 915 | |
i@14 | 916 | The file should be in YAML syntax.""" |
i@14 | 917 | |
i@14 | 918 | if not os.path.isfile(config_file_path): |
moritz@29 | 919 | raise argparse.ArgumentTypeError("Wrong path for configuration file (%s)" % config_file_path) |
i@14 | 920 | |
i@14 | 921 | with open(config_file_path) as yaml_file: |
i@14 | 922 | try: |
i@14 | 923 | settings = yaml.safe_load(yaml_file) |
i@14 | 924 | logger.debug("Read settings file(%s)" % config_file_path) |
moritz@29 | 925 | except Exception: |
moritz@29 | 926 | raise argparse.ArgumentTypeError("Could not parse YAML file (%s)" % config_file_path) |
i@14 | 927 | |
i@14 | 928 | # YAML limitation: does not read tuples |
i@14 | 929 | settings['basic_credentials'] = tuple(settings['basic_credentials']) |
i@14 | 930 | settings['website_credentials'] = tuple(settings['website_credentials']) |
i@14 | 931 | return settings |
i@14 | 932 | |
i@14 | 933 | |
moritz@29 | 934 | # Setup for command specific parsers |
moritz@29 | 935 | def setup_delete(parser): |
moritz@29 | 936 | def delete_from_args(parsed): |
ioannis@45 | 937 | delete_measurements(parsed.IDs, |
ioannis@45 | 938 | delete_related=False, |
ioannis@45 | 939 | settings=parsed.config) |
moritz@29 | 940 | |
moritz@29 | 941 | parser.add_argument("IDs", nargs="+", help="measurement IDs to delete.") |
moritz@29 | 942 | parser.set_defaults(execute=delete_from_args) |
moritz@29 | 943 | |
moritz@29 | 944 | |
moritz@29 | 945 | def setup_rerun_all(parser): |
moritz@29 | 946 | def rerun_all_from_args(parsed): |
moritz@29 | 947 | rerun_all(parsed.IDs, parsed.process, parsed.config) |
moritz@29 | 948 | |
moritz@29 | 949 | parser.add_argument("IDs", nargs="+", help="Measurement IDs to rerun.") |
moritz@29 | 950 | parser.add_argument("-p", "--process", help="Wait for the results of the processing.", |
moritz@29 | 951 | action="store_true") |
moritz@29 | 952 | parser.set_defaults(execute=rerun_all_from_args) |
moritz@29 | 953 | |
moritz@29 | 954 | |
ioannis@45 | 955 | def setup_rerun_elpp(parser): |
moritz@29 | 956 | def rerun_processing_from_args(parsed): |
moritz@29 | 957 | rerun_processing(parsed.IDs, parsed.process, parsed.config) |
moritz@29 | 958 | |
moritz@29 | 959 | parser.add_argument("IDs", nargs="+", help="Measurement IDs to rerun the processing on.") |
moritz@29 | 960 | parser.add_argument("-p", "--process", help="Wait for the results of the processing.", |
moritz@29 | 961 | action="store_true") |
moritz@29 | 962 | parser.set_defaults(execute=rerun_processing_from_args) |
moritz@29 | 963 | |
moritz@29 | 964 | |
ioannis@45 | 965 | def setup_upload_file(parser): |
ioannis@45 | 966 | """ Upload but do not monitor processing progress. """ |
ioannis@67 | 967 | |
ioannis@45 | 968 | def upload_file_from_args(parsed): |
ioannis@45 | 969 | process_file(parsed.filename, parsed.system, parsed.config, |
ioannis@54 | 970 | delay=parsed.delay, |
ioannis@45 | 971 | monitor=parsed.process, |
ioannis@45 | 972 | force_upload=parsed.force_upload, |
ioannis@45 | 973 | delete_related=False, # For now, use this as default |
ioannis@32 | 974 | rs_filename=parsed.radiosounding, |
ioannis@32 | 975 | ov_filename=parsed.overlap, |
ioannis@32 | 976 | lr_filename=parsed.lidarratio) |
moritz@29 | 977 | |
ioannis@54 | 978 | def delay(arg): |
ioannis@54 | 979 | try: |
ioannis@54 | 980 | int_arg = int(arg) |
ioannis@54 | 981 | except ValueError: |
ioannis@54 | 982 | raise argparse.ArgumentTypeError("Could not convert delay argument {} to integer.".format(arg)) |
ioannis@54 | 983 | |
ioannis@54 | 984 | if 0 <= int_arg <= 96: |
ioannis@54 | 985 | return int_arg |
ioannis@54 | 986 | else: |
ioannis@54 | 987 | raise argparse.ArgumentTypeError("Delay should be an integer between 0 and 96.") |
ioannis@54 | 988 | |
moritz@29 | 989 | parser.add_argument("filename", help="Measurement file name or path.") |
moritz@29 | 990 | parser.add_argument("system", help="Processing system id.") |
ioannis@54 | 991 | parser.add_argument("--delay", help="Delay processing by the specified number of hours (0 to 96).", |
ioannis@54 | 992 | default=0, type=delay) |
ioannis@45 | 993 | parser.add_argument("-p", "--process", help="Wait for the processing results.", |
ioannis@45 | 994 | action="store_true") |
ioannis@45 | 995 | parser.add_argument("--force_upload", help="If measurement ID exists on SCC, delete before uploading.", |
ioannis@45 | 996 | action="store_true") |
i@30 | 997 | parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path") |
ioannis@32 | 998 | parser.add_argument("--overlap", default=None, help="Overlap file name or path") |
ioannis@32 | 999 | parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path") |
i@30 | 1000 | |
moritz@29 | 1001 | parser.set_defaults(execute=upload_file_from_args) |
moritz@29 | 1002 | |
moritz@29 | 1003 | |
moritz@29 | 1004 | def setup_list_measurements(parser): |
moritz@29 | 1005 | def list_measurements_from_args(parsed): |
ioannis@65 | 1006 | list_measurements(parsed.config, id_exact=parsed.id_exact, id_startswith=parsed.id_startswith) |
moritz@29 | 1007 | |
ioannis@65 | 1008 | group = parser.add_mutually_exclusive_group() |
ioannis@65 | 1009 | group.add_argument("--id_exact", help="Exact measurement id.") |
ioannis@65 | 1010 | group.add_argument("--id_startswith", help="Initial part of measurement id.") |
moritz@29 | 1011 | |
moritz@29 | 1012 | parser.set_defaults(execute=list_measurements_from_args) |
moritz@29 | 1013 | |
moritz@29 | 1014 | |
moritz@29 | 1015 | def setup_download_measurements(parser): |
moritz@29 | 1016 | def download_measurements_from_args(parsed): |
ioannis@61 | 1017 | download_measurements(parsed.IDs, parsed.max_retries, parsed.ignore_errors, parsed.config) |
moritz@29 | 1018 | |
moritz@29 | 1019 | parser.add_argument("IDs", help="Measurement IDs that should be downloaded.", nargs="+") |
ioannis@67 | 1020 | parser.add_argument("--max_retries", help="Number of times to retry in cases of missing measurement id.", default=0, |
ioannis@67 | 1021 | type=int) |
ioannis@67 | 1022 | parser.add_argument("--ignore_errors", help="Ignore errors when downloading multiple measurements.", |
ioannis@67 | 1023 | action="store_false") |
moritz@29 | 1024 | parser.set_defaults(execute=download_measurements_from_args) |
moritz@29 | 1025 | |
moritz@29 | 1026 | |
victor@7 | 1027 | def main(): |
victor@7 | 1028 | # Define the command line arguments. |
victor@7 | 1029 | parser = argparse.ArgumentParser() |
moritz@29 | 1030 | subparsers = parser.add_subparsers() |
moritz@29 | 1031 | |
moritz@29 | 1032 | delete_parser = subparsers.add_parser("delete", help="Deletes a measurement.") |
ioannis@67 | 1033 | rerun_all_parser = subparsers.add_parser("rerun-all", |
ioannis@67 | 1034 | help="Rerun all processing steps for the provided measurement IDs.") |
ioannis@45 | 1035 | rerun_processing_parser = subparsers.add_parser("rerun-elpp", |
ioannis@45 | 1036 | help="Rerun low-resolution processing steps for the provided measurement ID.") |
ioannis@67 | 1037 | upload_file_parser = subparsers.add_parser("upload-file", |
ioannis@67 | 1038 | help="Submit a file and, optionally, download the output products.") |
ioannis@32 | 1039 | list_parser = subparsers.add_parser("list", help="List measurements registered on the SCC.") |
moritz@29 | 1040 | download_parser = subparsers.add_parser("download", help="Download selected measurements.") |
moritz@29 | 1041 | |
moritz@29 | 1042 | setup_delete(delete_parser) |
moritz@29 | 1043 | setup_rerun_all(rerun_all_parser) |
ioannis@45 | 1044 | setup_rerun_elpp(rerun_processing_parser) |
ioannis@45 | 1045 | |
moritz@29 | 1046 | setup_upload_file(upload_file_parser) |
moritz@29 | 1047 | setup_list_measurements(list_parser) |
moritz@29 | 1048 | setup_download_measurements(download_parser) |
victor@7 | 1049 | |
victor@7 | 1050 | # Verbosity settings from http://stackoverflow.com/a/20663028 |
victor@7 | 1051 | parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const", |
victor@7 | 1052 | dest="loglevel", const=logging.DEBUG, default=logging.INFO, |
victor@7 | 1053 | ) |
victor@7 | 1054 | parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const", |
victor@7 | 1055 | dest="loglevel", const=logging.WARNING |
victor@7 | 1056 | ) |
victor@7 | 1057 | |
ioannis@32 | 1058 | # Setup default config location |
moritz@29 | 1059 | home = os.path.expanduser("~") |
moritz@29 | 1060 | default_config_location = os.path.abspath(os.path.join(home, ".scc_access.yaml")) |
moritz@29 | 1061 | parser.add_argument("-c", "--config", help="Path to the config file.", type=settings_from_path, |
moritz@29 | 1062 | default=default_config_location) |
victor@7 | 1063 | |
victor@7 | 1064 | args = parser.parse_args() |
ioannis@43 | 1065 | |
victor@7 | 1066 | # Get the logger with the appropriate level |
victor@7 | 1067 | logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel) |
victor@7 | 1068 | |
moritz@29 | 1069 | # Dispatch to appropriate function |
moritz@29 | 1070 | args.execute(args) |
moritz@29 | 1071 | |
i@14 | 1072 | |
moritz@29 | 1073 | # When running through terminal |
moritz@29 | 1074 | if __name__ == '__main__': |
moritz@29 | 1075 | main() |