SCC access: comparison scc_access/scc

-:8ebbba085eb0
+:2d90204710af
 import urllib.parse as urlparse  # Python 3
 except ImportError:
 import urlparse  # Python 2
 import argparse
+import datetime
+import logging
 import os
 import re
+from io import StringIO
+import sys
 import time
-from io import StringIO
+import urlparse
 from zipfile import ZipFile
-import datetime
-import logging
+import requests
 import yaml
+requests.packages.urllib3.disable_warnings()
 logger = logging.getLogger(__name__)
 # The regex to find the measurement id from the measurement page
 # This should be read from the uploaded file, but would require an extra NetCDF module.
 regex = "<h3>Measurement (?P<measurement_id>.{12}) <small>"
 class SCC:
-""" A simple class that will attempt to upload a file on the SCC server.
+"""A simple class that will attempt to upload a file on the SCC server.
 The uploading is done by simulating a normal browser session. In the current
 version no check is performed, and no feedback is given if the upload
 was successful. If everything is setup correctly, it will work.
 """
 def __init__(self, auth, output_dir, base_url):
 self.auth = auth
 self.output_dir = output_dir
 if login_page.status_code != 200:
 logger.error('Could not access login pages. Status code %s' % login_page.status_code)
 sys.exit(1)
 logger.debug("Submiting credentials.")
 # Submit the login data
 login_submit = self.session.post(self.login_url,
 data=self.login_credentials,
 headers={'X-CSRFToken': login_page.cookies['csrftoken'],
 'referer': self.login_url},
 return login_submit
 def logout(self):
 pass
-def upload_file(self, filename, system_id):
+def upload_file(self, filename, system_id, rs_filename=None):
 """ Upload a filename for processing with a specific system. If the
 upload is successful, it returns the measurement id. """
 # Get submit page
 upload_page = self.session.get(self.upload_url,
 auth=self.auth,
 verify=False)
 # Submit the data
 upload_data = {'system': system_id}
 files = {'data': open(filename, 'rb')}
+if rs_filename is not None:
+files['sounding_file'] = open(rs_filename, 'rb')
 logger.info("Uploading of file %s started." % filename)
 upload_submit = self.session.post(self.upload_url,
 data=upload_data,
 return measurement_id
 def download_files(self, measurement_id, subdir, download_url):
 """ Downloads some files from the download_url to the specified
 subdir. This method is used to download preprocessed file, optical
 files etc.
 """
 # Get the file
 request = self.session.get(download_url, auth=self.auth,
 verify=False,
 # Construct the download url
 download_url = self.download_graph_pattern.format(measurement_id)
 self.download_files(measurement_id, 'scc_plots', download_url)
 def rerun_processing(self, measurement_id, monitor=True):
-measurement = self.get_measurement(measurement_id)
+measurement, status = self.get_measurement(measurement_id)
 if measurement:
 request = self.session.get(measurement.rerun_processing_url, auth=self.auth,
 verify=False,
 stream=True)
 def rerun_all(self, measurement_id, monitor=True):
 logger.debug("Started rerun_all procedure.")
 logger.debug("Getting measurement %s" % measurement_id)
-measurement = self.get_measurement(measurement_id)
+measurement, status = self.get_measurement(measurement_id)
 if measurement:
 logger.debug("Attempting to rerun all processing through %s." % measurement.rerun_all_url)
 request = self.session.get(measurement.rerun_all_url, auth=self.auth,
 return
 if monitor:
 self.monitor_processing(measurement_id)
-def process(self, filename, system_id):
+def process(self, filename, system_id, rs_filename=None):
 """ Upload a file for processing and wait for the processing to finish.
 If the processing is successful, it will download all produced files.
 """
 logger.info("--- Processing started on %s. ---" % datetime.datetime.now())
 # Upload file
-measurement_id = self.upload_file(filename, system_id)
+logger.info("--- Uploading file")
+measurement_id = self.upload_file(filename, system_id, rs_filename=rs_filename)
+logger.info("--- Monitoring processing")
 measurement = self.monitor_processing(measurement_id)
 return measurement
 def monitor_processing(self, measurement_id):
 """ Monitor the processing progress of a measurement id"""
-measurement = self.get_measurement(measurement_id)
+# try to deal with error 404
+error_count = 0
+error_max = 6
+time_sleep = 10
+# try to wait for measurement to appear in API
+measurement = None
+logger.info("looking for measurement %s in SCC", measurement_id)
+while error_count < error_max:
+time.sleep(time_sleep)
+measurement, status = self.get_measurement(measurement_id)
+if status != 200 and error_count < error_max:
+logger.error("measurement not found. waiting %ds", time_sleep)
+error_count += 1
+else:
+break
+if error_count == error_max:
+logger.critical("measurement %s doesn't seem to exist", measurement_id)
+sys.exit(1)
+logger.info('measurement %s found', measurement_id)
 if measurement is not None:
+error_count = 0
 while measurement.is_running:
-logger.info("Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload,
+logger.info("Measurement is being processed (status: %s, %s, %s). Please wait.", measurement.upload, measurement.pre_processing, measurement.processing)
-measurement.pre_processing,
-measurement.processing))
 time.sleep(10)
-measurement = self.get_measurement(measurement_id)
+measurement, status = self.get_measurement(measurement_id)
-logger.info("Measurement processing finished (status: %s, %s, %s)." % (measurement.upload,
-measurement.pre_processing,
+logger.info("Measurement processing finished (status: %s, %s, %s).",measurement.upload, measurement.pre_processing, measurement.processing)
-measurement.processing))
 if measurement.pre_processing == 127:
 logger.info("Downloading preprocessed files.")
 self.download_preprocessed(measurement_id)
 if measurement.processing == 127:
 logger.info("Downloading optical files.")
 response = self.session.get(measurement_url,
 auth=self.auth,
 verify=False)
+# maybe the measurements isn't already available on the database.
+if response.status_code == 404:
+return None, 404
 if response.status_code != 200:
 logger.error('Could not access API. Status code %s.' % response.status_code)
-sys.exit(1)
+return None, response.status_code
 response_dict = response.json()
 if response_dict:
-measurement = Measurement(self.base_url,response_dict)
+measurement = Measurement(self.base_url, response_dict)
-return measurement
+return measurement, response.status_code
 else:
 logger.error("No measurement with id %s found on the SCC." % measurement_id)
-return None
+return None, response.status_code
 def delete_measurement(self, measurement_id):
 """ Deletes a measurement with the provided measurement id. The user
 should have the appropriate permissions.
 The procedures is performed directly through the web interface and
 NOT through the API.
 """
 # Get the measurement object
-measurement = self.get_measurement(measurement_id)
+measurement, status = self.get_measurement(measurement_id)
 # Check that it exists
 if measurement is None:
 logger.warning("Nothing to delete.")
 return None
 return measurements
 def measurement_id_for_date(self, t1, call_sign='bu', base_number=0):
 """ Give the first available measurement id on the SCC for the specific
 date.
 """
 date_str = t1.strftime('%Y%m%d')
 search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % date_str)
 response = self.session.get(search_url,
 self.upload,
 self.pre_processing,
 self.processing)
-def upload_file(filename, system_id, settings):
+def upload_file(filename, system_id, settings, rs_filename=None):
 """ Shortcut function to upload a file to the SCC. """
 logger.info("Uploading file %s, using sytem %s" % (filename, system_id))
 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
 scc.login(settings['website_credentials'])
-measurement_id = scc.upload_file(filename, system_id)
+measurement_id = scc.upload_file(filename, system_id, rs_filename=rs_filename)
 scc.logout()
 return measurement_id
-def process_file(filename, system_id, settings):
+def process_file(filename, system_id, settings, rs_filename=None):
 """ Shortcut function to process a file to the SCC. """
 logger.info("Processing file %s, using sytem %s" % (filename, system_id))
 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
 scc.login(settings['website_credentials'])
-measurement = scc.process(filename, system_id)
+measurement = scc.process(filename, system_id, rs_filename=rs_filename)
 scc.logout()
 return measurement
 def delete_measurement(measurement_id, settings):
 action="store_true")
 parser.add_argument("--delete", help="Measurement ID to delete.")
 parser.add_argument("--rerun-all", help="Measurement ID to rerun.")
 parser.add_argument("--rerun-processing", help="Measurement ID to rerun processing routines.")
+# others files
+parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path")
 # Verbosity settings from http://stackoverflow.com/a/20663028
 parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const",
 dest="loglevel", const=logging.DEBUG, default=logging.INFO,
 )
 parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const",
 else:
 if (args.filename == '') or (args.system == 0):
 parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n')
 if args.process:
-process_file(args.filename, args.system, settings)
+process_file(args.filename, args.system, settings, rs_filename=args.radiosounding)
 else:
-upload_file(args.filename, args.system, settings)
+upload_file(args.filename, args.system, settings, rs_filename=args.radiosounding)

comparison: scc_access/scc_access.py

scc_access/scc_access.py

Mercurial > public > scc_access / file comparison

comparison: scc_access/scc_access.py

scc_access/scc_access.py