Mercurial > public > scc_access / changeset
changeset
Partial clean-up after merge.
Fri, 12 Oct 2018 16:38:28 +0300
- author
- Iannis <i.binietoglou@impworks.gr>
- date
- Fri, 12 Oct 2018 16:38:28 +0300
- changeset 31
- 020e80de1f64
- parent 30
- 4669876326d4
- child 32
- a7b7e0056fce
Partial clean-up after merge.
| scc_access/scc_access.py | file | annotate | diff | comparison | revisions |
--- a/scc_access/scc_access.py Thu Oct 11 15:25:55 2018 +0300 +++ b/scc_access/scc_access.py Fri Oct 12 16:38:28 2018 +0300 @@ -42,7 +42,11 @@ self.session = requests.Session() self.session.auth = auth self.session.verify = False + self.login_url = urlparse.urljoin(self.base_url, 'accounts/login/') + self.logout_url = urlparse.urljoin(self.base_url, 'accounts/logout/') + self.list_measurements_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/') + self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/') self.download_preprocessed_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-preprocessed/') @@ -51,11 +55,13 @@ self.download_graph_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-plots/') self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/') + self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/') - self.list_measurements_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/') + self.api_measurement_pattern = urlparse.urljoin(self.api_base_url, 'measurements/{0}/') + self.api_measurements_url = urlparse.urljoin(self.api_base_url, 'measurements') def login(self, credentials): - """ Login the the website. """ + """ Login to SCC. """ logger.debug("Attempting to login to SCC, username %s." % credentials[0]) login_credentials = {'username': credentials[0], 'password': credentials[1]} @@ -65,12 +71,10 @@ # Get upload form login_page = self.session.get(self.login_url) - # TODO: Do we need this? Mortiz removed it. - if login_page.status_code != 200: - logger.error('Could not access login pages. Status code %s' % login_page.status_code) - sys.exit(1) + if not login_page.ok: + raise self.PageNotAccessibleError('Could not access login pages. Status code %s' % login_page.status_code) - logger.debug("Submiting credentials.") + logger.debug("Submitting credentials.") # Submit the login data login_submit = self.session.post(self.login_url, data=login_credentials, @@ -79,9 +83,10 @@ return login_submit def logout(self): - pass + """ Logout from SCC """ + return self.session.get(self.logout_url, stream=True) - def upload_file(self, filename, system_id, rs_filename=None): + def upload_file(self, filename, system_id, rs_filename=None, ol_filename=None, lr_filename=None): """ Upload a filename for processing with a specific system. If the upload is successful, it returns the measurement id. """ # Get submit page @@ -92,8 +97,17 @@ files = {'data': open(filename, 'rb')} if rs_filename is not None: + logger.debug('Adding sounding file %s' % rs_filename) files['sounding_file'] = open(rs_filename, 'rb') + if ol_filename is not None: + logger.debug('Adding overlap file %s' % ol_filename) + files['overlap_file'] = open(ol_filename, 'rb') + + if lr_filename is not None: + logger.debug('Adding lidar ratio file %s' % lr_filename) + files['lidar_ratio_file'] = open(lr_filename, 'rb') + logger.info("Uploading of file %s started." % filename) upload_submit = self.session.post(self.upload_url, @@ -247,7 +261,7 @@ time.sleep(10) measurement, status = self.get_measurement(measurement_id) - logger.info("Measurement processing finished (status: %s, %s, %s).",measurement.upload, measurement.pre_processing, measurement.processing) + logger.info("Measurement processing finished (status: %s, %s, %s).", measurement.upload, measurement.pre_processing, measurement.processing) if measurement.pre_processing == 127: logger.info("Downloading preprocessed files.") self.download_preprocessed(measurement_id) @@ -259,24 +273,9 @@ logger.info("--- Processing finished. ---") return measurement - def get_status(self, measurement_id): - """ Get the processing status for a measurement id through the API. """ - measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__exact=%s' % measurement_id) - - response = self.session.get(measurement_url) - - response_dict = response.json() - - if response_dict['objects']: - measurement_list = response_dict['objects'] - measurement = Measurement(self.base_url, measurement_list[0]) - return measurement.upload, measurement.pre_processing, measurement.processing - else: - logger.error("No measurement with id %s found on the SCC." % measurement_id) - return None - def get_measurement(self, measurement_id): - measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/%s/' % measurement_id) + measurement_url = self.api_measurement_pattern.format(measurement_id) + logger.debug("Measurement API URL: %s" % measurement_url) response = self.session.get(measurement_url) @@ -301,7 +300,7 @@ NOT through the API. """ # Get the measurement object - measurement, status = self.get_measurement(measurement_id) + measurement, _ = self.get_measurement(measurement_id) # Check that it exists if measurement is None: @@ -324,7 +323,7 @@ headers={'X-CSRFToken': confirm_page.cookies['csrftoken'], 'referer': delete_url} ) - if delete_page.status_code != 200: + if not delete_page.ok: logger.warning("Something went wrong. Delete page status: {0}".format( delete_page.status_code)) return None @@ -334,23 +333,24 @@ def available_measurements(self): """ Get a list of available measurement on the SCC. """ - measurement_url = urlparse.urljoin(self.api_base_url, 'measurements') - response = self.session.get(measurement_url) + response = self.session.get(self.api_measurements_url) response_dict = response.json() - measurements = None if response_dict: measurement_list = response_dict['objects'] measurements = [Measurement(self.base_url, measurement_dict) for measurement_dict in measurement_list] logger.info("Found %s measurements on the SCC." % len(measurements)) else: logger.warning("No response received from the SCC when asked for available measurements.") + measurements = None return measurements def list_measurements(self, station=None, system=None, start=None, stop=None, upload_status=None, processing_status=None, optical_processing=None): + # TODO: Change this to work through the API + # Need to set to empty string if not specified, we won't get any results params = { "station": station if station is not None else "", @@ -361,13 +361,14 @@ "preprocessing_status": processing_status if processing_status is not None else "", "optical_processing_status": optical_processing if optical_processing is not None else "" } - resp = self.session.get(self.list_measurements_pattern, params=params).text + + response_txt = self.session.get(self.list_measurements_url, params=params).text tbl_rgx = re.compile(r'<table id="measurements">(.*?)</table>', re.DOTALL) entry_rgx = re.compile(r'<tr>(.*?)</tr>', re.DOTALL) measurement_rgx = re.compile( r'.*?<td><a[^>]*>(\w+)</a>.*?<td>.*?<td>([\w-]+ [\w:]+)</td>.*<td data-order="([-]?\d+),([-]?\d+),([-]?\d+)".*', re.DOTALL) - matches = tbl_rgx.findall(resp) + matches = tbl_rgx.findall(response_txt) if len(matches) != 1: return [] @@ -382,12 +383,13 @@ return ret - def measurement_id_for_date(self, t1, call_sign='bu', base_number=0): + def measurement_id_for_date(self, t1, call_sign, base_number=0): """ Give the first available measurement id on the SCC for the specific date. """ date_str = t1.strftime('%Y%m%d') - search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % date_str) + base_id = "%s%s" % (date_str, call_sign) + search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % base_id) response = self.session.get(search_url) @@ -397,19 +399,24 @@ if response_dict: measurement_list = response_dict['objects'] + + if len(measurement_list) == 100: + raise ValueError('No available measurement id found.') + existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list] measurement_number = base_number - measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number) + measurement_id = "%s%02i" % (base_id, measurement_number) while measurement_id in existing_ids: measurement_number = measurement_number + 1 - measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number) - if measurement_number == 100: - raise ValueError('No available measurement id found.') + measurement_id = "%s%02i" % (base_id, measurement_number) return measurement_id + class PageNotAccessibleError(RuntimeError): + pass + class ApiObject(object): """ A generic class object. """ @@ -435,19 +442,6 @@ """ @property - def is_running(self): - """ Returns True if the processing has not finished. - """ - if self.upload == 0: - return False - if self.pre_processing == -127: - return False - if self.pre_processing == 127: - if self.processing in [127, -127]: - return False - return True - - @property def rerun_processing_url(self): url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-optical/') return url_pattern.format(self.id)
