Mercurial > public > scc_access / file comparison
comparison: scc_access/scc_access.py
scc_access/scc_access.py
- changeset 31
- 020e80de1f64
- parent 30
- 4669876326d4
- child 32
- a7b7e0056fce
equal
deleted
inserted
replaced
| 30:4669876326d4 | 31:020e80de1f64 |
|---|---|
| 40 self.output_dir = output_dir | 40 self.output_dir = output_dir |
| 41 self.base_url = base_url | 41 self.base_url = base_url |
| 42 self.session = requests.Session() | 42 self.session = requests.Session() |
| 43 self.session.auth = auth | 43 self.session.auth = auth |
| 44 self.session.verify = False | 44 self.session.verify = False |
| 45 | |
| 45 self.login_url = urlparse.urljoin(self.base_url, 'accounts/login/') | 46 self.login_url = urlparse.urljoin(self.base_url, 'accounts/login/') |
| 47 self.logout_url = urlparse.urljoin(self.base_url, 'accounts/logout/') | |
| 48 self.list_measurements_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/') | |
| 49 | |
| 46 self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/') | 50 self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/') |
| 47 self.download_preprocessed_pattern = urlparse.urljoin(self.base_url, | 51 self.download_preprocessed_pattern = urlparse.urljoin(self.base_url, |
| 48 'data_processing/measurements/{0}/download-preprocessed/') | 52 'data_processing/measurements/{0}/download-preprocessed/') |
| 49 self.download_optical_pattern = urlparse.urljoin(self.base_url, | 53 self.download_optical_pattern = urlparse.urljoin(self.base_url, |
| 50 'data_processing/measurements/{0}/download-optical/') | 54 'data_processing/measurements/{0}/download-optical/') |
| 51 self.download_graph_pattern = urlparse.urljoin(self.base_url, | 55 self.download_graph_pattern = urlparse.urljoin(self.base_url, |
| 52 'data_processing/measurements/{0}/download-plots/') | 56 'data_processing/measurements/{0}/download-plots/') |
| 53 self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/') | 57 self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/') |
| 58 | |
| 54 self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/') | 59 self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/') |
| 55 self.list_measurements_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/') | 60 self.api_measurement_pattern = urlparse.urljoin(self.api_base_url, 'measurements/{0}/') |
| 61 self.api_measurements_url = urlparse.urljoin(self.api_base_url, 'measurements') | |
| 56 | 62 |
| 57 def login(self, credentials): | 63 def login(self, credentials): |
| 58 """ Login the the website. """ | 64 """ Login to SCC. """ |
| 59 logger.debug("Attempting to login to SCC, username %s." % credentials[0]) | 65 logger.debug("Attempting to login to SCC, username %s." % credentials[0]) |
| 60 login_credentials = {'username': credentials[0], | 66 login_credentials = {'username': credentials[0], |
| 61 'password': credentials[1]} | 67 'password': credentials[1]} |
| 62 | 68 |
| 63 logger.debug("Accessing login page at %s." % self.login_url) | 69 logger.debug("Accessing login page at %s." % self.login_url) |
| 64 | 70 |
| 65 # Get upload form | 71 # Get upload form |
| 66 login_page = self.session.get(self.login_url) | 72 login_page = self.session.get(self.login_url) |
| 67 | 73 |
| 68 # TODO: Do we need this? Mortiz removed it. | 74 if not login_page.ok: |
| 69 if login_page.status_code != 200: | 75 raise self.PageNotAccessibleError('Could not access login pages. Status code %s' % login_page.status_code) |
| 70 logger.error('Could not access login pages. Status code %s' % login_page.status_code) | 76 |
| 71 sys.exit(1) | 77 logger.debug("Submitting credentials.") |
| 72 | |
| 73 logger.debug("Submiting credentials.") | |
| 74 # Submit the login data | 78 # Submit the login data |
| 75 login_submit = self.session.post(self.login_url, | 79 login_submit = self.session.post(self.login_url, |
| 76 data=login_credentials, | 80 data=login_credentials, |
| 77 headers={'X-CSRFToken': login_page.cookies['csrftoken'], | 81 headers={'X-CSRFToken': login_page.cookies['csrftoken'], |
| 78 'referer': self.login_url}) | 82 'referer': self.login_url}) |
| 79 return login_submit | 83 return login_submit |
| 80 | 84 |
| 81 def logout(self): | 85 def logout(self): |
| 82 pass | 86 """ Logout from SCC """ |
| 83 | 87 return self.session.get(self.logout_url, stream=True) |
| 84 def upload_file(self, filename, system_id, rs_filename=None): | 88 |
| 89 def upload_file(self, filename, system_id, rs_filename=None, ol_filename=None, lr_filename=None): | |
| 85 """ Upload a filename for processing with a specific system. If the | 90 """ Upload a filename for processing with a specific system. If the |
| 86 upload is successful, it returns the measurement id. """ | 91 upload is successful, it returns the measurement id. """ |
| 87 # Get submit page | 92 # Get submit page |
| 88 upload_page = self.session.get(self.upload_url) | 93 upload_page = self.session.get(self.upload_url) |
| 89 | 94 |
| 90 # Submit the data | 95 # Submit the data |
| 91 upload_data = {'system': system_id} | 96 upload_data = {'system': system_id} |
| 92 files = {'data': open(filename, 'rb')} | 97 files = {'data': open(filename, 'rb')} |
| 93 | 98 |
| 94 if rs_filename is not None: | 99 if rs_filename is not None: |
| 100 logger.debug('Adding sounding file %s' % rs_filename) | |
| 95 files['sounding_file'] = open(rs_filename, 'rb') | 101 files['sounding_file'] = open(rs_filename, 'rb') |
| 102 | |
| 103 if ol_filename is not None: | |
| 104 logger.debug('Adding overlap file %s' % ol_filename) | |
| 105 files['overlap_file'] = open(ol_filename, 'rb') | |
| 106 | |
| 107 if lr_filename is not None: | |
| 108 logger.debug('Adding lidar ratio file %s' % lr_filename) | |
| 109 files['lidar_ratio_file'] = open(lr_filename, 'rb') | |
| 96 | 110 |
| 97 logger.info("Uploading of file %s started." % filename) | 111 logger.info("Uploading of file %s started." % filename) |
| 98 | 112 |
| 99 upload_submit = self.session.post(self.upload_url, | 113 upload_submit = self.session.post(self.upload_url, |
| 100 data=upload_data, | 114 data=upload_data, |
| 245 while measurement.is_running: | 259 while measurement.is_running: |
| 246 logger.info("Measurement is being processed (status: %s, %s, %s). Please wait.", measurement.upload, measurement.pre_processing, measurement.processing) | 260 logger.info("Measurement is being processed (status: %s, %s, %s). Please wait.", measurement.upload, measurement.pre_processing, measurement.processing) |
| 247 time.sleep(10) | 261 time.sleep(10) |
| 248 measurement, status = self.get_measurement(measurement_id) | 262 measurement, status = self.get_measurement(measurement_id) |
| 249 | 263 |
| 250 logger.info("Measurement processing finished (status: %s, %s, %s).",measurement.upload, measurement.pre_processing, measurement.processing) | 264 logger.info("Measurement processing finished (status: %s, %s, %s).", measurement.upload, measurement.pre_processing, measurement.processing) |
| 251 if measurement.pre_processing == 127: | 265 if measurement.pre_processing == 127: |
| 252 logger.info("Downloading preprocessed files.") | 266 logger.info("Downloading preprocessed files.") |
| 253 self.download_preprocessed(measurement_id) | 267 self.download_preprocessed(measurement_id) |
| 254 if measurement.processing == 127: | 268 if measurement.processing == 127: |
| 255 logger.info("Downloading optical files.") | 269 logger.info("Downloading optical files.") |
| 257 logger.info("Downloading graphs.") | 271 logger.info("Downloading graphs.") |
| 258 self.download_graphs(measurement_id) | 272 self.download_graphs(measurement_id) |
| 259 logger.info("--- Processing finished. ---") | 273 logger.info("--- Processing finished. ---") |
| 260 return measurement | 274 return measurement |
| 261 | 275 |
| 262 def get_status(self, measurement_id): | |
| 263 """ Get the processing status for a measurement id through the API. """ | |
| 264 measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__exact=%s' % measurement_id) | |
| 265 | |
| 266 response = self.session.get(measurement_url) | |
| 267 | |
| 268 response_dict = response.json() | |
| 269 | |
| 270 if response_dict['objects']: | |
| 271 measurement_list = response_dict['objects'] | |
| 272 measurement = Measurement(self.base_url, measurement_list[0]) | |
| 273 return measurement.upload, measurement.pre_processing, measurement.processing | |
| 274 else: | |
| 275 logger.error("No measurement with id %s found on the SCC." % measurement_id) | |
| 276 return None | |
| 277 | |
| 278 def get_measurement(self, measurement_id): | 276 def get_measurement(self, measurement_id): |
| 279 measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/%s/' % measurement_id) | 277 measurement_url = self.api_measurement_pattern.format(measurement_id) |
| 278 logger.debug("Measurement API URL: %s" % measurement_url) | |
| 280 | 279 |
| 281 response = self.session.get(measurement_url) | 280 response = self.session.get(measurement_url) |
| 282 | 281 |
| 283 if not response.ok: | 282 if not response.ok: |
| 284 logger.error('Could not access API. Status code %s.' % response.status_code) | 283 logger.error('Could not access API. Status code %s.' % response.status_code) |
| 299 | 298 |
| 300 The procedures is performed directly through the web interface and | 299 The procedures is performed directly through the web interface and |
| 301 NOT through the API. | 300 NOT through the API. |
| 302 """ | 301 """ |
| 303 # Get the measurement object | 302 # Get the measurement object |
| 304 measurement, status = self.get_measurement(measurement_id) | 303 measurement, _ = self.get_measurement(measurement_id) |
| 305 | 304 |
| 306 # Check that it exists | 305 # Check that it exists |
| 307 if measurement is None: | 306 if measurement is None: |
| 308 logger.warning("Nothing to delete.") | 307 logger.warning("Nothing to delete.") |
| 309 return None | 308 return None |
| 322 delete_page = self.session.post(delete_url, | 321 delete_page = self.session.post(delete_url, |
| 323 data={'post': 'yes'}, | 322 data={'post': 'yes'}, |
| 324 headers={'X-CSRFToken': confirm_page.cookies['csrftoken'], | 323 headers={'X-CSRFToken': confirm_page.cookies['csrftoken'], |
| 325 'referer': delete_url} | 324 'referer': delete_url} |
| 326 ) | 325 ) |
| 327 if delete_page.status_code != 200: | 326 if not delete_page.ok: |
| 328 logger.warning("Something went wrong. Delete page status: {0}".format( | 327 logger.warning("Something went wrong. Delete page status: {0}".format( |
| 329 delete_page.status_code)) | 328 delete_page.status_code)) |
| 330 return None | 329 return None |
| 331 | 330 |
| 332 logger.info("Deleted measurement {0}".format(measurement_id)) | 331 logger.info("Deleted measurement {0}".format(measurement_id)) |
| 333 return True | 332 return True |
| 334 | 333 |
| 335 def available_measurements(self): | 334 def available_measurements(self): |
| 336 """ Get a list of available measurement on the SCC. """ | 335 """ Get a list of available measurement on the SCC. """ |
| 337 measurement_url = urlparse.urljoin(self.api_base_url, 'measurements') | 336 response = self.session.get(self.api_measurements_url) |
| 338 response = self.session.get(measurement_url) | |
| 339 response_dict = response.json() | 337 response_dict = response.json() |
| 340 | 338 |
| 341 measurements = None | |
| 342 if response_dict: | 339 if response_dict: |
| 343 measurement_list = response_dict['objects'] | 340 measurement_list = response_dict['objects'] |
| 344 measurements = [Measurement(self.base_url, measurement_dict) for measurement_dict in measurement_list] | 341 measurements = [Measurement(self.base_url, measurement_dict) for measurement_dict in measurement_list] |
| 345 logger.info("Found %s measurements on the SCC." % len(measurements)) | 342 logger.info("Found %s measurements on the SCC." % len(measurements)) |
| 346 else: | 343 else: |
| 347 logger.warning("No response received from the SCC when asked for available measurements.") | 344 logger.warning("No response received from the SCC when asked for available measurements.") |
| 345 measurements = None | |
| 348 | 346 |
| 349 return measurements | 347 return measurements |
| 350 | 348 |
| 351 def list_measurements(self, station=None, system=None, start=None, stop=None, upload_status=None, | 349 def list_measurements(self, station=None, system=None, start=None, stop=None, upload_status=None, |
| 352 processing_status=None, optical_processing=None): | 350 processing_status=None, optical_processing=None): |
| 351 | |
| 352 # TODO: Change this to work through the API | |
| 353 | 353 |
| 354 # Need to set to empty string if not specified, we won't get any results | 354 # Need to set to empty string if not specified, we won't get any results |
| 355 params = { | 355 params = { |
| 356 "station": station if station is not None else "", | 356 "station": station if station is not None else "", |
| 357 "system": system if system is not None else "", | 357 "system": system if system is not None else "", |
| 359 "start": start if start is not None else "", | 359 "start": start if start is not None else "", |
| 360 "upload_status": upload_status if upload_status is not None else "", | 360 "upload_status": upload_status if upload_status is not None else "", |
| 361 "preprocessing_status": processing_status if processing_status is not None else "", | 361 "preprocessing_status": processing_status if processing_status is not None else "", |
| 362 "optical_processing_status": optical_processing if optical_processing is not None else "" | 362 "optical_processing_status": optical_processing if optical_processing is not None else "" |
| 363 } | 363 } |
| 364 resp = self.session.get(self.list_measurements_pattern, params=params).text | 364 |
| 365 response_txt = self.session.get(self.list_measurements_url, params=params).text | |
| 365 tbl_rgx = re.compile(r'<table id="measurements">(.*?)</table>', re.DOTALL) | 366 tbl_rgx = re.compile(r'<table id="measurements">(.*?)</table>', re.DOTALL) |
| 366 entry_rgx = re.compile(r'<tr>(.*?)</tr>', re.DOTALL) | 367 entry_rgx = re.compile(r'<tr>(.*?)</tr>', re.DOTALL) |
| 367 measurement_rgx = re.compile( | 368 measurement_rgx = re.compile( |
| 368 r'.*?<td><a[^>]*>(\w+)</a>.*?<td>.*?<td>([\w-]+ [\w:]+)</td>.*<td data-order="([-]?\d+),([-]?\d+),([-]?\d+)".*', | 369 r'.*?<td><a[^>]*>(\w+)</a>.*?<td>.*?<td>([\w-]+ [\w:]+)</td>.*<td data-order="([-]?\d+),([-]?\d+),([-]?\d+)".*', |
| 369 re.DOTALL) | 370 re.DOTALL) |
| 370 matches = tbl_rgx.findall(resp) | 371 matches = tbl_rgx.findall(response_txt) |
| 371 if len(matches) != 1: | 372 if len(matches) != 1: |
| 372 return [] | 373 return [] |
| 373 | 374 |
| 374 ret = [] | 375 ret = [] |
| 375 for entry in entry_rgx.finditer(matches[0]): | 376 for entry in entry_rgx.finditer(matches[0]): |
| 380 Measurement(self.base_url, {"id": name, "upload": int(upload), "pre_processing": int(preproc), | 381 Measurement(self.base_url, {"id": name, "upload": int(upload), "pre_processing": int(preproc), |
| 381 "processing": int(optical)})) | 382 "processing": int(optical)})) |
| 382 | 383 |
| 383 return ret | 384 return ret |
| 384 | 385 |
| 385 def measurement_id_for_date(self, t1, call_sign='bu', base_number=0): | 386 def measurement_id_for_date(self, t1, call_sign, base_number=0): |
| 386 """ Give the first available measurement id on the SCC for the specific | 387 """ Give the first available measurement id on the SCC for the specific |
| 387 date. | 388 date. |
| 388 """ | 389 """ |
| 389 date_str = t1.strftime('%Y%m%d') | 390 date_str = t1.strftime('%Y%m%d') |
| 390 search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % date_str) | 391 base_id = "%s%s" % (date_str, call_sign) |
| 392 search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % base_id) | |
| 391 | 393 |
| 392 response = self.session.get(search_url) | 394 response = self.session.get(search_url) |
| 393 | 395 |
| 394 response_dict = response.json() | 396 response_dict = response.json() |
| 395 | 397 |
| 396 measurement_id = None | 398 measurement_id = None |
| 397 | 399 |
| 398 if response_dict: | 400 if response_dict: |
| 399 measurement_list = response_dict['objects'] | 401 measurement_list = response_dict['objects'] |
| 402 | |
| 403 if len(measurement_list) == 100: | |
| 404 raise ValueError('No available measurement id found.') | |
| 405 | |
| 400 existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list] | 406 existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list] |
| 401 | 407 |
| 402 measurement_number = base_number | 408 measurement_number = base_number |
| 403 measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number) | 409 measurement_id = "%s%02i" % (base_id, measurement_number) |
| 404 | 410 |
| 405 while measurement_id in existing_ids: | 411 while measurement_id in existing_ids: |
| 406 measurement_number = measurement_number + 1 | 412 measurement_number = measurement_number + 1 |
| 407 measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number) | 413 measurement_id = "%s%02i" % (base_id, measurement_number) |
| 408 if measurement_number == 100: | |
| 409 raise ValueError('No available measurement id found.') | |
| 410 | 414 |
| 411 return measurement_id | 415 return measurement_id |
| 416 | |
| 417 class PageNotAccessibleError(RuntimeError): | |
| 418 pass | |
| 412 | 419 |
| 413 | 420 |
| 414 class ApiObject(object): | 421 class ApiObject(object): |
| 415 """ A generic class object. """ | 422 """ A generic class object. """ |
| 416 | 423 |
| 431 | 438 |
| 432 | 439 |
| 433 class Measurement(ApiObject): | 440 class Measurement(ApiObject): |
| 434 """ This class represents the measurement object as returned in the SCC API. | 441 """ This class represents the measurement object as returned in the SCC API. |
| 435 """ | 442 """ |
| 436 | |
| 437 @property | |
| 438 def is_running(self): | |
| 439 """ Returns True if the processing has not finished. | |
| 440 """ | |
| 441 if self.upload == 0: | |
| 442 return False | |
| 443 if self.pre_processing == -127: | |
| 444 return False | |
| 445 if self.pre_processing == 127: | |
| 446 if self.processing in [127, -127]: | |
| 447 return False | |
| 448 return True | |
| 449 | 443 |
| 450 @property | 444 @property |
| 451 def rerun_processing_url(self): | 445 def rerun_processing_url(self): |
| 452 url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-optical/') | 446 url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-optical/') |
| 453 return url_pattern.format(self.id) | 447 return url_pattern.format(self.id) |
