--- a/scc_access/scc_access.py Thu Oct 11 15:25:55 2018 +0300
+++ b/scc_access/scc_access.py Fri Oct 12 16:38:28 2018 +0300
@@ -42,7 +42,11 @@
self.session = requests.Session()
self.session.auth = auth
self.session.verify = False
+
self.login_url = urlparse.urljoin(self.base_url, 'accounts/login/')
+ self.logout_url = urlparse.urljoin(self.base_url, 'accounts/logout/')
+ self.list_measurements_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/')
+
self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/')
self.download_preprocessed_pattern = urlparse.urljoin(self.base_url,
'data_processing/measurements/{0}/download-preprocessed/')
@@ -51,11 +55,13 @@
self.download_graph_pattern = urlparse.urljoin(self.base_url,
'data_processing/measurements/{0}/download-plots/')
self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/')
+
self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/')
- self.list_measurements_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/')
+ self.api_measurement_pattern = urlparse.urljoin(self.api_base_url, 'measurements/{0}/')
+ self.api_measurements_url = urlparse.urljoin(self.api_base_url, 'measurements')
def login(self, credentials):
- """ Login the the website. """
+ """ Login to SCC. """
logger.debug("Attempting to login to SCC, username %s." % credentials[0])
login_credentials = {'username': credentials[0],
'password': credentials[1]}
@@ -65,12 +71,10 @@
# Get upload form
login_page = self.session.get(self.login_url)
- # TODO: Do we need this? Mortiz removed it.
- if login_page.status_code != 200:
- logger.error('Could not access login pages. Status code %s' % login_page.status_code)
- sys.exit(1)
+ if not login_page.ok:
+ raise self.PageNotAccessibleError('Could not access login pages. Status code %s' % login_page.status_code)
- logger.debug("Submiting credentials.")
+ logger.debug("Submitting credentials.")
# Submit the login data
login_submit = self.session.post(self.login_url,
data=login_credentials,
@@ -79,9 +83,10 @@
return login_submit
def logout(self):
- pass
+ """ Logout from SCC """
+ return self.session.get(self.logout_url, stream=True)
- def upload_file(self, filename, system_id, rs_filename=None):
+ def upload_file(self, filename, system_id, rs_filename=None, ol_filename=None, lr_filename=None):
""" Upload a filename for processing with a specific system. If the
upload is successful, it returns the measurement id. """
# Get submit page
@@ -92,8 +97,17 @@
files = {'data': open(filename, 'rb')}
if rs_filename is not None:
+ logger.debug('Adding sounding file %s' % rs_filename)
files['sounding_file'] = open(rs_filename, 'rb')
+ if ol_filename is not None:
+ logger.debug('Adding overlap file %s' % ol_filename)
+ files['overlap_file'] = open(ol_filename, 'rb')
+
+ if lr_filename is not None:
+ logger.debug('Adding lidar ratio file %s' % lr_filename)
+ files['lidar_ratio_file'] = open(lr_filename, 'rb')
+
logger.info("Uploading of file %s started." % filename)
upload_submit = self.session.post(self.upload_url,
@@ -247,7 +261,7 @@
time.sleep(10)
measurement, status = self.get_measurement(measurement_id)
- logger.info("Measurement processing finished (status: %s, %s, %s).",measurement.upload, measurement.pre_processing, measurement.processing)
+ logger.info("Measurement processing finished (status: %s, %s, %s).", measurement.upload, measurement.pre_processing, measurement.processing)
if measurement.pre_processing == 127:
logger.info("Downloading preprocessed files.")
self.download_preprocessed(measurement_id)
@@ -259,24 +273,9 @@
logger.info("--- Processing finished. ---")
return measurement
- def get_status(self, measurement_id):
- """ Get the processing status for a measurement id through the API. """
- measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__exact=%s' % measurement_id)
-
- response = self.session.get(measurement_url)
-
- response_dict = response.json()
-
- if response_dict['objects']:
- measurement_list = response_dict['objects']
- measurement = Measurement(self.base_url, measurement_list[0])
- return measurement.upload, measurement.pre_processing, measurement.processing
- else:
- logger.error("No measurement with id %s found on the SCC." % measurement_id)
- return None
-
def get_measurement(self, measurement_id):
- measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/%s/' % measurement_id)
+ measurement_url = self.api_measurement_pattern.format(measurement_id)
+ logger.debug("Measurement API URL: %s" % measurement_url)
response = self.session.get(measurement_url)
@@ -301,7 +300,7 @@
NOT through the API.
"""
# Get the measurement object
- measurement, status = self.get_measurement(measurement_id)
+ measurement, _ = self.get_measurement(measurement_id)
# Check that it exists
if measurement is None:
@@ -324,7 +323,7 @@
headers={'X-CSRFToken': confirm_page.cookies['csrftoken'],
'referer': delete_url}
)
- if delete_page.status_code != 200:
+ if not delete_page.ok:
logger.warning("Something went wrong. Delete page status: {0}".format(
delete_page.status_code))
return None
@@ -334,23 +333,24 @@
def available_measurements(self):
""" Get a list of available measurement on the SCC. """
- measurement_url = urlparse.urljoin(self.api_base_url, 'measurements')
- response = self.session.get(measurement_url)
+ response = self.session.get(self.api_measurements_url)
response_dict = response.json()
- measurements = None
if response_dict:
measurement_list = response_dict['objects']
measurements = [Measurement(self.base_url, measurement_dict) for measurement_dict in measurement_list]
logger.info("Found %s measurements on the SCC." % len(measurements))
else:
logger.warning("No response received from the SCC when asked for available measurements.")
+ measurements = None
return measurements
def list_measurements(self, station=None, system=None, start=None, stop=None, upload_status=None,
processing_status=None, optical_processing=None):
+ # TODO: Change this to work through the API
+
# Need to set to empty string if not specified, we won't get any results
params = {
"station": station if station is not None else "",
@@ -361,13 +361,14 @@
"preprocessing_status": processing_status if processing_status is not None else "",
"optical_processing_status": optical_processing if optical_processing is not None else ""
}
- resp = self.session.get(self.list_measurements_pattern, params=params).text
+
+ response_txt = self.session.get(self.list_measurements_url, params=params).text
tbl_rgx = re.compile(r'<table id="measurements">(.*?)</table>', re.DOTALL)
entry_rgx = re.compile(r'<tr>(.*?)</tr>', re.DOTALL)
measurement_rgx = re.compile(
r'.*?<td><a[^>]*>(\w+)</a>.*?<td>.*?<td>([\w-]+ [\w:]+)</td>.*<td data-order="([-]?\d+),([-]?\d+),([-]?\d+)".*',
re.DOTALL)
- matches = tbl_rgx.findall(resp)
+ matches = tbl_rgx.findall(response_txt)
if len(matches) != 1:
return []
@@ -382,12 +383,13 @@
return ret
- def measurement_id_for_date(self, t1, call_sign='bu', base_number=0):
+ def measurement_id_for_date(self, t1, call_sign, base_number=0):
""" Give the first available measurement id on the SCC for the specific
date.
"""
date_str = t1.strftime('%Y%m%d')
- search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % date_str)
+ base_id = "%s%s" % (date_str, call_sign)
+ search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % base_id)
response = self.session.get(search_url)
@@ -397,19 +399,24 @@
if response_dict:
measurement_list = response_dict['objects']
+
+ if len(measurement_list) == 100:
+ raise ValueError('No available measurement id found.')
+
existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list]
measurement_number = base_number
- measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number)
+ measurement_id = "%s%02i" % (base_id, measurement_number)
while measurement_id in existing_ids:
measurement_number = measurement_number + 1
- measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number)
- if measurement_number == 100:
- raise ValueError('No available measurement id found.')
+ measurement_id = "%s%02i" % (base_id, measurement_number)
return measurement_id
+ class PageNotAccessibleError(RuntimeError):
+ pass
+
class ApiObject(object):
""" A generic class object. """
@@ -435,19 +442,6 @@
"""
@property
- def is_running(self):
- """ Returns True if the processing has not finished.
- """
- if self.upload == 0:
- return False
- if self.pre_processing == -127:
- return False
- if self.pre_processing == 127:
- if self.processing in [127, -127]:
- return False
- return True
-
- @property
def rerun_processing_url(self):
url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-optical/')
return url_pattern.format(self.id)
