# HG changeset patch # User Iannis # Date 1539260755 -10800 # Node ID 4669876326d4f3fa7f4a4cf9b9a20389d33c48a9 # Parent 2d90204710af93d356605266c097349ec8ac448f# Parent 3e3e5bda6b7755fb89e9d6add5dc2ea2796663d1 Merge from Moritz branch. Clean-up is still needed. diff -r 2d90204710af -r 4669876326d4 README.rst --- a/README.rst Thu Oct 11 13:14:33 2018 +0300 +++ b/README.rst Thu Oct 11 15:25:55 2018 +0300 @@ -32,14 +32,16 @@ Settings -------- -You will need to provide some user-defined settings in a .yaml format. You -can rename the settings_sample.yaml file to settings.yaml and follow the instructions -there. +You will need to change some user-defined settings in a settings.yaml file. You +can copy the settings_sample.yaml file to settings.py and follow the instructions +there. You can copy the resulting file to your home directory as `.scc_access.yaml`. +This is the default location, `scc_access` will search there if no other location was +specified. Specifically, you will need to: -1. Change the BASIC_LOGIN and DJANGO_LOGIN to your credentials. -2. Change the OUTPUT_DIR to the location were the results will be stored. +1. Change the `basic_credentials` and `website_credentials` to your credentials. +2. Change the `output_dir` to the location were the results will be stored. Please not that it's not a good idea to store your stations management credentials in the settings file. The standard user has "Station Management" privileges and if the credentials @@ -53,17 +55,22 @@ You can upload a file specifying the username and the system id:: - scc_access 20110101po01.nc 125 + scc_access upload-file 20110101po01.nc 125 If you want to wait for the processing to finish and download the resulting files -you need to define the -p flag:: +you need to define the `process-file` command. Use the `-p` flag to wait for the +result:: - scc_access 20110101po01.nc 125 -p + scc_access process-file 20110101po01.nc 125 -p -If you want to delete an existing measurement id from the database use the -d flag and -the measurement id:: +If you want to delete an existing measurement id from the database use the `delete` +command and the measurement id:: - scc_access -d 20110101po01 + scc_access delete 20110101po01 + +You can list available measurements with the `list` command:: + + scc_access list For more information on the syntax type:: diff -r 2d90204710af -r 4669876326d4 scc_access/scc_access.py --- a/scc_access/scc_access.py Thu Oct 11 13:14:33 2018 +0300 +++ b/scc_access/scc_access.py Thu Oct 11 15:25:55 2018 +0300 @@ -1,7 +1,4 @@ import requests -requests.packages.urllib3.disable_warnings() - -import sys try: import urllib.parse as urlparse # Python 3 @@ -19,11 +16,9 @@ import urlparse from zipfile import ZipFile -import requests import yaml requests.packages.urllib3.disable_warnings() - logger = logging.getLogger(__name__) # The regex to find the measurement id from the measurement page @@ -40,47 +35,47 @@ """ def __init__(self, auth, output_dir, base_url): + self.auth = auth self.output_dir = output_dir self.base_url = base_url self.session = requests.Session() - self.construct_urls() - - def construct_urls(self): - """ Construct all URLs needed for processing. """ - # Construct the absolute URLs + self.session.auth = auth + self.session.verify = False self.login_url = urlparse.urljoin(self.base_url, 'accounts/login/') self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/') - self.download_preprocessed_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-preprocessed/') - self.download_optical_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-optical/') - self.download_graph_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-plots/') + self.download_preprocessed_pattern = urlparse.urljoin(self.base_url, + 'data_processing/measurements/{0}/download-preprocessed/') + self.download_optical_pattern = urlparse.urljoin(self.base_url, + 'data_processing/measurements/{0}/download-optical/') + self.download_graph_pattern = urlparse.urljoin(self.base_url, + 'data_processing/measurements/{0}/download-plots/') self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/') self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/') + self.list_measurements_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/') def login(self, credentials): """ Login the the website. """ logger.debug("Attempting to login to SCC, username %s." % credentials[0]) - self.login_credentials = {'username': credentials[0], - 'password': credentials[1]} + login_credentials = {'username': credentials[0], + 'password': credentials[1]} logger.debug("Accessing login page at %s." % self.login_url) # Get upload form - login_page = self.session.get(self.login_url, auth=self.auth, verify=False) + login_page = self.session.get(self.login_url) + # TODO: Do we need this? Mortiz removed it. if login_page.status_code != 200: logger.error('Could not access login pages. Status code %s' % login_page.status_code) sys.exit(1) logger.debug("Submiting credentials.") - # Submit the login data login_submit = self.session.post(self.login_url, - data=self.login_credentials, + data=login_credentials, headers={'X-CSRFToken': login_page.cookies['csrftoken'], - 'referer': self.login_url}, - verify=False, - auth=self.auth) + 'referer': self.login_url}) return login_submit def logout(self): @@ -90,9 +85,7 @@ """ Upload a filename for processing with a specific system. If the upload is successful, it returns the measurement id. """ # Get submit page - upload_page = self.session.get(self.upload_url, - auth=self.auth, - verify=False) + upload_page = self.session.get(self.upload_url) # Submit the data upload_data = {'system': system_id} @@ -107,9 +100,7 @@ data=upload_data, files=files, headers={'X-CSRFToken': upload_page.cookies['csrftoken'], - 'referer': self.upload_url,}, - verify=False, - auth=self.auth) + 'referer': self.upload_url}) if upload_submit.status_code != 200: logger.warning("Connection error. Status code: %s" % upload_submit.status_code) @@ -131,9 +122,10 @@ files etc. """ # Get the file - request = self.session.get(download_url, auth=self.auth, - verify=False, - stream=True) + request = self.session.get(download_url, stream=True) + + if not request.ok: + raise Exception("Could not download files for measurement '%s'" % measurement_id) # Create the dir if it does not exist local_dir = os.path.join(self.output_dir, measurement_id, subdir) @@ -180,9 +172,7 @@ measurement, status = self.get_measurement(measurement_id) if measurement: - request = self.session.get(measurement.rerun_processing_url, auth=self.auth, - verify=False, - stream=True) + request = self.session.get(measurement.rerun_processing_url, stream=True) if request.status_code != 200: logger.error( @@ -201,9 +191,7 @@ if measurement: logger.debug("Attempting to rerun all processing through %s." % measurement.rerun_all_url) - request = self.session.get(measurement.rerun_all_url, auth=self.auth, - verify=False, - stream=True) + request = self.session.get(measurement.rerun_all_url, stream=True) if request.status_code != 200: logger.error("Could not rerun pre processing for %s. Status code: %s" % @@ -213,7 +201,7 @@ if monitor: self.monitor_processing(measurement_id) - def process(self, filename, system_id, rs_filename=None): + def process(self, filename, system_id, monitor, rs_filename=None): """ Upload a file for processing and wait for the processing to finish. If the processing is successful, it will download all produced files. """ @@ -223,8 +211,9 @@ measurement_id = self.upload_file(filename, system_id, rs_filename=rs_filename) logger.info("--- Monitoring processing") - measurement = self.monitor_processing(measurement_id) - return measurement + if monitor: + return self.monitor_processing(measurement_id) + return None def monitor_processing(self, measurement_id): """ Monitor the processing progress of a measurement id""" @@ -253,7 +242,6 @@ logger.info('measurement %s found', measurement_id) if measurement is not None: - error_count = 0 while measurement.is_running: logger.info("Measurement is being processed (status: %s, %s, %s). Please wait.", measurement.upload, measurement.pre_processing, measurement.processing) time.sleep(10) @@ -275,9 +263,7 @@ """ Get the processing status for a measurement id through the API. """ measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__exact=%s' % measurement_id) - response = self.session.get(measurement_url, - auth=self.auth, - verify=False) + response = self.session.get(measurement_url) response_dict = response.json() @@ -292,15 +278,9 @@ def get_measurement(self, measurement_id): measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/%s/' % measurement_id) - response = self.session.get(measurement_url, - auth=self.auth, - verify=False) + response = self.session.get(measurement_url) - # maybe the measurements isn't already available on the database. - if response.status_code == 404: - return None, 404 - - if response.status_code != 200: + if not response.ok: logger.error('Could not access API. Status code %s.' % response.status_code) return None, response.status_code @@ -329,11 +309,9 @@ return None # Go the the page confirming the deletion - delete_url = self.delete_measurement_pattern.format(measurement.id) + delete_url = self.delete_measurement_pattern.format(measurement_id) - confirm_page = self.session.get(delete_url, - auth=self.auth, - verify=False) + confirm_page = self.session.get(delete_url) # Check that the page opened properly if confirm_page.status_code != 200: @@ -342,8 +320,6 @@ # Delete the measurement delete_page = self.session.post(delete_url, - auth=self.auth, - verify=False, data={'post': 'yes'}, headers={'X-CSRFToken': confirm_page.cookies['csrftoken'], 'referer': delete_url} @@ -359,9 +335,7 @@ def available_measurements(self): """ Get a list of available measurement on the SCC. """ measurement_url = urlparse.urljoin(self.api_base_url, 'measurements') - response = self.session.get(measurement_url, - auth=self.auth, - verify=False) + response = self.session.get(measurement_url) response_dict = response.json() measurements = None @@ -374,6 +348,40 @@ return measurements + def list_measurements(self, station=None, system=None, start=None, stop=None, upload_status=None, + processing_status=None, optical_processing=None): + + # Need to set to empty string if not specified, we won't get any results + params = { + "station": station if station is not None else "", + "system": system if system is not None else "", + "stop": stop if stop is not None else "", + "start": start if start is not None else "", + "upload_status": upload_status if upload_status is not None else "", + "preprocessing_status": processing_status if processing_status is not None else "", + "optical_processing_status": optical_processing if optical_processing is not None else "" + } + resp = self.session.get(self.list_measurements_pattern, params=params).text + tbl_rgx = re.compile(r'(.*?)
', re.DOTALL) + entry_rgx = re.compile(r'(.*?)', re.DOTALL) + measurement_rgx = re.compile( + r'.*?]*>(\w+).*?.*?([\w-]+ [\w:]+).*