--- a/scc_access/scc_access.py Thu Oct 11 12:45:05 2018 +0300
+++ b/scc_access/scc_access.py Thu Oct 11 13:14:33 2018 +0300
@@ -9,15 +9,20 @@
import urlparse # Python 2
import argparse
+import datetime
+import logging
import os
import re
+from io import StringIO
+import sys
import time
-from io import StringIO
+import urlparse
from zipfile import ZipFile
-import datetime
-import logging
+
+import requests
import yaml
+requests.packages.urllib3.disable_warnings()
logger = logging.getLogger(__name__)
@@ -27,11 +32,11 @@
class SCC:
- """ A simple class that will attempt to upload a file on the SCC server.
+ """A simple class that will attempt to upload a file on the SCC server.
The uploading is done by simulating a normal browser session. In the current
- version no check is performed, and no feedback is given if the upload
- was successful. If everything is setup correctly, it will work.
+ version no check is performed, and no feedback is given if the upload
+ was successful. If everything is setup correctly, it will work.
"""
def __init__(self, auth, output_dir, base_url):
@@ -68,7 +73,7 @@
sys.exit(1)
logger.debug("Submiting credentials.")
-
+
# Submit the login data
login_submit = self.session.post(self.login_url,
data=self.login_credentials,
@@ -81,8 +86,8 @@
def logout(self):
pass
- def upload_file(self, filename, system_id):
- """ Upload a filename for processing with a specific system. If the
+ def upload_file(self, filename, system_id, rs_filename=None):
+ """ Upload a filename for processing with a specific system. If the
upload is successful, it returns the measurement id. """
# Get submit page
upload_page = self.session.get(self.upload_url,
@@ -93,6 +98,9 @@
upload_data = {'system': system_id}
files = {'data': open(filename, 'rb')}
+ if rs_filename is not None:
+ files['sounding_file'] = open(rs_filename, 'rb')
+
logger.info("Uploading of file %s started." % filename)
upload_submit = self.session.post(self.upload_url,
@@ -119,7 +127,7 @@
def download_files(self, measurement_id, subdir, download_url):
""" Downloads some files from the download_url to the specified
- subdir. This method is used to download preprocessed file, optical
+ subdir. This method is used to download preprocessed file, optical
files etc.
"""
# Get the file
@@ -169,7 +177,7 @@
self.download_files(measurement_id, 'scc_plots', download_url)
def rerun_processing(self, measurement_id, monitor=True):
- measurement = self.get_measurement(measurement_id)
+ measurement, status = self.get_measurement(measurement_id)
if measurement:
request = self.session.get(measurement.rerun_processing_url, auth=self.auth,
@@ -188,7 +196,7 @@
logger.debug("Started rerun_all procedure.")
logger.debug("Getting measurement %s" % measurement_id)
- measurement = self.get_measurement(measurement_id)
+ measurement, status = self.get_measurement(measurement_id)
if measurement:
logger.debug("Attempting to rerun all processing through %s." % measurement.rerun_all_url)
@@ -205,31 +213,53 @@
if monitor:
self.monitor_processing(measurement_id)
- def process(self, filename, system_id):
+ def process(self, filename, system_id, rs_filename=None):
""" Upload a file for processing and wait for the processing to finish.
If the processing is successful, it will download all produced files.
"""
logger.info("--- Processing started on %s. ---" % datetime.datetime.now())
# Upload file
- measurement_id = self.upload_file(filename, system_id)
+ logger.info("--- Uploading file")
+ measurement_id = self.upload_file(filename, system_id, rs_filename=rs_filename)
+ logger.info("--- Monitoring processing")
measurement = self.monitor_processing(measurement_id)
return measurement
def monitor_processing(self, measurement_id):
""" Monitor the processing progress of a measurement id"""
- measurement = self.get_measurement(measurement_id)
+ # try to deal with error 404
+ error_count = 0
+ error_max = 6
+ time_sleep = 10
+
+ # try to wait for measurement to appear in API
+ measurement = None
+ logger.info("looking for measurement %s in SCC", measurement_id)
+ while error_count < error_max:
+ time.sleep(time_sleep)
+ measurement, status = self.get_measurement(measurement_id)
+ if status != 200 and error_count < error_max:
+ logger.error("measurement not found. waiting %ds", time_sleep)
+ error_count += 1
+ else:
+ break
+
+ if error_count == error_max:
+ logger.critical("measurement %s doesn't seem to exist", measurement_id)
+ sys.exit(1)
+
+ logger.info('measurement %s found', measurement_id)
+
if measurement is not None:
+ error_count = 0
while measurement.is_running:
- logger.info("Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload,
- measurement.pre_processing,
- measurement.processing))
+ logger.info("Measurement is being processed (status: %s, %s, %s). Please wait.", measurement.upload, measurement.pre_processing, measurement.processing)
time.sleep(10)
- measurement = self.get_measurement(measurement_id)
- logger.info("Measurement processing finished (status: %s, %s, %s)." % (measurement.upload,
- measurement.pre_processing,
- measurement.processing))
+ measurement, status = self.get_measurement(measurement_id)
+
+ logger.info("Measurement processing finished (status: %s, %s, %s).",measurement.upload, measurement.pre_processing, measurement.processing)
if measurement.pre_processing == 127:
logger.info("Downloading preprocessed files.")
self.download_preprocessed(measurement_id)
@@ -266,28 +296,32 @@
auth=self.auth,
verify=False)
+ # maybe the measurements isn't already available on the database.
+ if response.status_code == 404:
+ return None, 404
+
if response.status_code != 200:
logger.error('Could not access API. Status code %s.' % response.status_code)
- sys.exit(1)
+ return None, response.status_code
response_dict = response.json()
if response_dict:
- measurement = Measurement(self.base_url,response_dict)
- return measurement
+ measurement = Measurement(self.base_url, response_dict)
+ return measurement, response.status_code
else:
logger.error("No measurement with id %s found on the SCC." % measurement_id)
- return None
+ return None, response.status_code
def delete_measurement(self, measurement_id):
""" Deletes a measurement with the provided measurement id. The user
- should have the appropriate permissions.
-
+ should have the appropriate permissions.
+
The procedures is performed directly through the web interface and
NOT through the API.
"""
# Get the measurement object
- measurement = self.get_measurement(measurement_id)
+ measurement, status = self.get_measurement(measurement_id)
# Check that it exists
if measurement is None:
@@ -342,7 +376,7 @@
def measurement_id_for_date(self, t1, call_sign='bu', base_number=0):
""" Give the first available measurement id on the SCC for the specific
- date.
+ date.
"""
date_str = t1.strftime('%Y%m%d')
search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % date_str)
@@ -424,24 +458,24 @@
self.processing)
-def upload_file(filename, system_id, settings):
+def upload_file(filename, system_id, settings, rs_filename=None):
""" Shortcut function to upload a file to the SCC. """
logger.info("Uploading file %s, using sytem %s" % (filename, system_id))
scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
scc.login(settings['website_credentials'])
- measurement_id = scc.upload_file(filename, system_id)
+ measurement_id = scc.upload_file(filename, system_id, rs_filename=rs_filename)
scc.logout()
return measurement_id
-def process_file(filename, system_id, settings):
+def process_file(filename, system_id, settings, rs_filename=None):
""" Shortcut function to process a file to the SCC. """
logger.info("Processing file %s, using sytem %s" % (filename, system_id))
scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
scc.login(settings['website_credentials'])
- measurement = scc.process(filename, system_id)
+ measurement = scc.process(filename, system_id, rs_filename=rs_filename)
scc.logout()
return measurement
@@ -511,6 +545,9 @@
parser.add_argument("--rerun-all", help="Measurement ID to rerun.")
parser.add_argument("--rerun-processing", help="Measurement ID to rerun processing routines.")
+ # others files
+ parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path")
+
# Verbosity settings from http://stackoverflow.com/a/20663028
parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const",
dest="loglevel", const=logging.DEBUG, default=logging.INFO,
@@ -539,6 +576,6 @@
parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n')
if args.process:
- process_file(args.filename, args.system, settings)
+ process_file(args.filename, args.system, settings, rs_filename=args.radiosounding)
else:
- upload_file(args.filename, args.system, settings)
+ upload_file(args.filename, args.system, settings, rs_filename=args.radiosounding)
