Mercurial > public > scc_access / changeset
changeset
Merge from Moritz branch. Clean-up is still needed.
Thu, 11 Oct 2018 15:25:55 +0300
- author
- Iannis <i.binietoglou@impworks.gr>
- date
- Thu, 11 Oct 2018 15:25:55 +0300
- changeset 30
- 4669876326d4
- parent 28
- 2d90204710af (diff)
- parent 29
- 3e3e5bda6b77 (current diff)
- child 31
- 020e80de1f64
Merge from Moritz branch. Clean-up is still needed.
| README.rst | file | annotate | diff | comparison | revisions | |
| scc_access/scc_access.py | file | annotate | diff | comparison | revisions |
--- a/.hgignore Mon Jan 08 14:59:21 2018 +0100 +++ b/.hgignore Thu Oct 11 15:25:55 2018 +0300 @@ -1,6 +1,8 @@ -syntax: glob -*.rst~ -*.py~ -*.pyc -re:^settings\.py$ -re:^\.idea/ +syntax: glob +*.rst~ +*.py~ +*.pyc +re:^settings\.py$ +re:^\.idea/ +scc_access.egg-info/ +re:^\.pytest_cache/ \ No newline at end of file
--- a/CHANGELOG.rst Mon Jan 08 14:59:21 2018 +0100 +++ b/CHANGELOG.rst Thu Oct 11 15:25:55 2018 +0300 @@ -1,5 +1,18 @@ -0.5.0 (2015-06-23) -------------------- +Changelog +========= + +0.6.2 - 2018-01-10 +------------------ +* Fixed bug when download optical files. +* Changes config file path to positional argument. + +0.6.1 - 2017-12-15 +------------------ +* Converted script to python module +* Settings are now read from .yaml file. + +0.5.0 - 2015-06-23 +------------------ * Moved configuration settings to a separate file * Added lincence information (MIT) * Added version number
--- a/README.rst Mon Jan 08 14:59:21 2018 +0100 +++ b/README.rst Thu Oct 11 15:25:55 2018 +0300 @@ -1,7 +1,7 @@ Overview -================= +======== -This package provides a script which permits interacting with the +This package provides a tool for interacting with the Single Calculus Chain through the command line. Specifically, with the script you can: * Upload a file to the SCC for processing @@ -25,13 +25,9 @@ The easiest way to install this module is from the python package index using pip:: - pip install scc_access - -See http://docs.python-requests.org/en/latest/user/install/ for more details. + pip install hg+https://bitbucket.org/iannis_b/scc-access#egg=scc-access -You can also use the script by cloning this mercurial repository. Alternatively, you -can just copy the scc_access.py and and settings.sample.py files to a local -directory. +You can also use the script by cloning this mercurial repository. Settings @@ -47,7 +43,7 @@ 1. Change the `basic_credentials` and `website_credentials` to your credentials. 2. Change the `output_dir` to the location were the results will be stored. -Please not that it's not a good idea to store your own credentials in the settings +Please not that it's not a good idea to store your stations management credentials in the settings file. The standard user has "Station Management" privileges and if the credentials are stolen, someone could change/delete the stations settings from the SCC database. For this, it is better to use a used account with minimum access settings, that
--- a/requirements.txt Mon Jan 08 14:59:21 2018 +0100 +++ b/requirements.txt Thu Oct 11 15:25:55 2018 +0300 @@ -1,1 +1,1 @@ -requests +. \ No newline at end of file
--- a/scc_access/__init__.py Mon Jan 08 14:59:21 2018 +0100 +++ b/scc_access/__init__.py Thu Oct 11 15:25:55 2018 +0300 @@ -1,1 +1,1 @@ -__version__ = "0.6.1" \ No newline at end of file +__version__ = "0.6.2" \ No newline at end of file
--- a/scc_access/scc_access.py Mon Jan 08 14:59:21 2018 +0100 +++ b/scc_access/scc_access.py Thu Oct 11 15:25:55 2018 +0300 @@ -1,18 +1,24 @@ import requests -requests.packages.urllib3.disable_warnings() +try: + import urllib.parse as urlparse # Python 3 +except ImportError: + import urlparse # Python 2 -import urlparse import argparse +import datetime +import logging import os import re +from io import StringIO +import sys import time -import StringIO +import urlparse from zipfile import ZipFile -import datetime -import logging + import yaml +requests.packages.urllib3.disable_warnings() logger = logging.getLogger(__name__) # The regex to find the measurement id from the measurement page @@ -21,11 +27,11 @@ class SCC: - """ A simple class that will attempt to upload a file on the SCC server. + """A simple class that will attempt to upload a file on the SCC server. The uploading is done by simulating a normal browser session. In the current - version no check is performed, and no feedback is given if the upload - was successful. If everything is setup correctly, it will work. + version no check is performed, and no feedback is given if the upload + was successful. If everything is setup correctly, it will work. """ def __init__(self, auth, output_dir, base_url): @@ -59,6 +65,11 @@ # Get upload form login_page = self.session.get(self.login_url) + # TODO: Do we need this? Mortiz removed it. + if login_page.status_code != 200: + logger.error('Could not access login pages. Status code %s' % login_page.status_code) + sys.exit(1) + logger.debug("Submiting credentials.") # Submit the login data login_submit = self.session.post(self.login_url, @@ -70,8 +81,8 @@ def logout(self): pass - def upload_file(self, filename, system_id): - """ Upload a filename for processing with a specific system. If the + def upload_file(self, filename, system_id, rs_filename=None): + """ Upload a filename for processing with a specific system. If the upload is successful, it returns the measurement id. """ # Get submit page upload_page = self.session.get(self.upload_url) @@ -80,6 +91,9 @@ upload_data = {'system': system_id} files = {'data': open(filename, 'rb')} + if rs_filename is not None: + files['sounding_file'] = open(rs_filename, 'rb') + logger.info("Uploading of file %s started." % filename) upload_submit = self.session.post(self.upload_url, @@ -98,13 +112,13 @@ logger.error("Uploaded file rejected! Try to upload manually to see the error.") else: measurement_id = re.findall(regex, upload_submit.text)[0] - logger.error("Successfully uploaded measurement with id %s." % measurement_id) + logger.info("Successfully uploaded measurement with id %s." % measurement_id) return measurement_id def download_files(self, measurement_id, subdir, download_url): """ Downloads some files from the download_url to the specified - subdir. This method is used to download preprocessed file, optical + subdir. This method is used to download preprocessed file, optical files etc. """ # Get the file @@ -119,7 +133,7 @@ os.makedirs(local_dir) # Save the file by chunk, needed if the file is big. - memory_file = StringIO.StringIO() + memory_file = StringIO() for chunk in request.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks @@ -155,7 +169,7 @@ self.download_files(measurement_id, 'scc_plots', download_url) def rerun_processing(self, measurement_id, monitor=True): - measurement = self.get_measurement(measurement_id) + measurement, status = self.get_measurement(measurement_id) if measurement: request = self.session.get(measurement.rerun_processing_url, stream=True) @@ -172,7 +186,7 @@ logger.debug("Started rerun_all procedure.") logger.debug("Getting measurement %s" % measurement_id) - measurement = self.get_measurement(measurement_id) + measurement, status = self.get_measurement(measurement_id) if measurement: logger.debug("Attempting to rerun all processing through %s." % measurement.rerun_all_url) @@ -187,14 +201,16 @@ if monitor: self.monitor_processing(measurement_id) - def process(self, filename, system_id, monitor): + def process(self, filename, system_id, monitor, rs_filename=None): """ Upload a file for processing and wait for the processing to finish. If the processing is successful, it will download all produced files. """ logger.info("--- Processing started on %s. ---" % datetime.datetime.now()) # Upload file - measurement_id = self.upload_file(filename, system_id) + logger.info("--- Uploading file") + measurement_id = self.upload_file(filename, system_id, rs_filename=rs_filename) + logger.info("--- Monitoring processing") if monitor: return self.monitor_processing(measurement_id) return None @@ -202,17 +218,36 @@ def monitor_processing(self, measurement_id): """ Monitor the processing progress of a measurement id""" - measurement = self.get_measurement(measurement_id) + # try to deal with error 404 + error_count = 0 + error_max = 6 + time_sleep = 10 + + # try to wait for measurement to appear in API + measurement = None + logger.info("looking for measurement %s in SCC", measurement_id) + while error_count < error_max: + time.sleep(time_sleep) + measurement, status = self.get_measurement(measurement_id) + if status != 200 and error_count < error_max: + logger.error("measurement not found. waiting %ds", time_sleep) + error_count += 1 + else: + break + + if error_count == error_max: + logger.critical("measurement %s doesn't seem to exist", measurement_id) + sys.exit(1) + + logger.info('measurement %s found', measurement_id) + if measurement is not None: while measurement.is_running: - logger.info("Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload, - measurement.pre_processing, - measurement.processing)) + logger.info("Measurement is being processed (status: %s, %s, %s). Please wait.", measurement.upload, measurement.pre_processing, measurement.processing) time.sleep(10) - measurement = self.get_measurement(measurement_id) - logger.info("Measurement processing finished (status: %s, %s, %s)." % (measurement.upload, - measurement.pre_processing, - measurement.processing)) + measurement, status = self.get_measurement(measurement_id) + + logger.info("Measurement processing finished (status: %s, %s, %s).",measurement.upload, measurement.pre_processing, measurement.processing) if measurement.pre_processing == 127: logger.info("Downloading preprocessed files.") self.download_preprocessed(measurement_id) @@ -247,26 +282,26 @@ if not response.ok: logger.error('Could not access API. Status code %s.' % response.status_code) - return None + return None, response.status_code response_dict = response.json() if response_dict: measurement = Measurement(self.base_url, response_dict) - return measurement + return measurement, response.status_code else: logger.error("No measurement with id %s found on the SCC." % measurement_id) - return None + return None, response.status_code def delete_measurement(self, measurement_id): """ Deletes a measurement with the provided measurement id. The user - should have the appropriate permissions. - + should have the appropriate permissions. + The procedures is performed directly through the web interface and NOT through the API. """ # Get the measurement object - measurement = self.get_measurement(measurement_id) + measurement, status = self.get_measurement(measurement_id) # Check that it exists if measurement is None: @@ -349,7 +384,7 @@ def measurement_id_for_date(self, t1, call_sign='bu', base_number=0): """ Give the first available measurement id on the SCC for the specific - date. + date. """ date_str = t1.strftime('%Y%m%d') search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % date_str) @@ -376,7 +411,7 @@ return measurement_id -class ApiObject: +class ApiObject(object): """ A generic class object. """ def __init__(self, base_url, dict_response): @@ -385,7 +420,11 @@ if dict_response: # Add the dictionary key value pairs as object properties for key, value in dict_response.items(): - setattr(self, key, value) + # logger.debug('Setting key {0} to value {1}'.format(key, value)) + try: + setattr(self, key, value) + except: + logger.warning('Could not set attribute {0} to value {1}'.format(key, value)) self.exists = True else: self.exists = False @@ -425,23 +464,24 @@ self.processing) -def upload_file(filename, system_id, settings): +def upload_file(filename, system_id, settings, rs_filename=None): """ Shortcut function to upload a file to the SCC. """ logger.info("Uploading file %s, using sytem %s" % (filename, system_id)) scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) scc.login(settings['website_credentials']) - measurement_id = scc.upload_file(filename, system_id) + measurement_id = scc.upload_file(filename, system_id, rs_filename=rs_filename) scc.logout() return measurement_id -def process_file(filename, system_id, monitor, settings): +def process_file(filename, system_id, settings, rs_filename=None): """ Shortcut function to process a file to the SCC. """ logger.info("Processing file %s, using sytem %s" % (filename, system_id)) + scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) scc.login(settings['website_credentials']) - measurement = scc.process(filename, system_id, monitor) + measurement = scc.process(filename, system_id, rs_filename=rs_filename) scc.logout() return measurement @@ -562,10 +602,11 @@ def setup_process_file(parser): def process_file_from_args(parsed): - process_file(parsed.file, parsed.system, parsed.process, parsed.config) + process_file(parsed.file, parsed.system, parsed.process, parsed.config, parsed.radiosounding) parser.add_argument("filename", help="Measurement file name or path.") parser.add_argument("system", help="Processing system id.") + parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path") parser.add_argument("-p", "--process", help="Wait for the results of the processing.", action="store_true") parser.set_defaults(execute=process_file_from_args) @@ -573,10 +614,12 @@ def setup_upload_file(parser): def upload_file_from_args(parsed): - upload_file(parsed.file, parsed.system, parsed.config) + upload_file(parsed.file, parsed.system, parsed.config, parsed.radiosounding) parser.add_argument("filename", help="Measurement file name or path.") parser.add_argument("system", help="Processing system id.") + parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path") + parser.set_defaults(execute=upload_file_from_args)
--- a/settings_sample.yaml Mon Jan 08 14:59:21 2018 +0100 +++ b/settings_sample.yaml Thu Oct 11 15:25:55 2018 +0300 @@ -1,6 +1,6 @@ # This file contains the user-specific settings for the scc_access script. # -# You should rename the file settings_sample.yaml to and move it outside the module repository. Take care to set the +# You should rename the file settings_sample.yaml (e.g. to settings.yaml) and move it outside the module repository. Take care to set the # minimum required permissions, as this file contains SCC access codes. For website login, it is recommended to # use credential for a user without station-management privileges. basic_credentials: ['username', 'password'] # The HTTP user name and password that is needed to access the SCC site.
