diff -r 1c170a1ae099 -r c02712d2ab9e scc_access.py
--- a/scc_access.py Fri Nov 24 17:32:29 2017 +0200
+++ b/scc_access.py Fri Nov 24 18:50:29 2017 +0200
@@ -25,17 +25,17 @@
__version__ = "0.6.0"
-
# Try to read the settings from the settings.py file
try:
from settings import *
except:
raise ImportError(
- """A settings file (setting.py) is required to run the script.
- You can use settings.sample.py as a template.""")
-
+ """A settings file (setting.py) is required to run the script.
+ You can use settings.sample.py as a template.""")
import requests
+requests.packages.urllib3.disable_warnings()
+
import urlparse
import argparse
import os
@@ -44,6 +44,9 @@
import StringIO
from zipfile import ZipFile
import datetime
+import logging
+
+logger = logging.getLogger(__name__)
# Construct the absolute URLs
@@ -55,115 +58,117 @@
RERUN_ALL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/rerun-all/')
RERUN_PROCESSING = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/rerun-optical/')
-DELETE_MEASUREMENT = urlparse.urljoin(BASE_URL, 'admin/database/measurements/{0}/delete/')
+DELETE_MEASUREMENT = urlparse.urljoin(BASE_URL, 'admin/database/measurements/{0}/delete/')
API_BASE_URL = urlparse.urljoin(BASE_URL, 'api/v1/')
# The regex to find the measurement id from the measurement page
-# This should be read from the uploaded file, but would require an extra module
+# This should be read from the uploaded file, but would require an extra NetCDF module.
regex = "
Measurement (?P.{12}) "
class SCC:
""" A simple class that will attempt to upload a file on the SCC server.
- The uploading is done by simulation a normal browser session. In the current
+ The uploading is done by simulating a normal browser session. In the current
version no check is performed, and no feedback is given if the upload
was successful. If everything is setup correctly, it will work.
"""
- def __init__(self, auth = BASIC_LOGIN, output_dir = OUTPUT_DIR):
+
+ def __init__(self, auth=BASIC_LOGIN, output_dir=OUTPUT_DIR):
self.auth = auth
- self.output_dir = OUTPUT_DIR
+ self.output_dir = output_dir
self.session = requests.Session()
-
- def login(self, credential = DJANGO_LOGIN):
+
+ def login(self, credentials=DJANGO_LOGIN):
""" Login the the website. """
- self.login_credentials = {'username': credential[0],
- 'password': credential[1]}
-
+ logger.debug("Attempting to login to SCC, username %s." % credentials[0])
+ self.login_credentials = {'username': credentials[0],
+ 'password': credentials[1]}
+
+ logger.debug("Accessing login page at %s." % LOGIN_URL)
+
# Get upload form
- login_page = self.session.get(LOGIN_URL,
- auth = self.auth, verify = False)
-
+ login_page = self.session.get(LOGIN_URL,
+ auth=self.auth, verify=False)
+
+ logger.debug("Submiting credentials.")
# Submit the login data
- login_submit = self.session.post(LOGIN_URL,
- data = self.login_credentials,
- headers = {'X-CSRFToken': login_page.cookies['csrftoken'],
- 'referer': LOGIN_URL},
- verify = False,
- auth = self.auth)
+ login_submit = self.session.post(LOGIN_URL,
+ data=self.login_credentials,
+ headers={'X-CSRFToken': login_page.cookies['csrftoken'],
+ 'referer': LOGIN_URL},
+ verify=False,
+ auth=self.auth)
return login_submit
def logout(self):
pass
-
+
def upload_file(self, filename, system_id):
""" Upload a filename for processing with a specific system. If the
upload is successful, it returns the measurement id. """
# Get submit page
- upload_page = self.session.get(UPLOAD_URL,
- auth = self.auth,
- verify = False)
-
- # Submit the data
+ upload_page = self.session.get(UPLOAD_URL,
+ auth=self.auth,
+ verify=False)
+
+ # Submit the data
upload_data = {'system': system_id}
- files = {'data': open(filename, 'rb')}
-
- print "Uploading of file %s started." % filename
-
- upload_submit = self.session.post(UPLOAD_URL,
- data = upload_data,
- files = files,
- headers = {'X-CSRFToken': upload_page.cookies['csrftoken'],
- 'referer': UPLOAD_URL},
- verify = False,
- auth = self.auth)
-
+ files = {'data': open(filename, 'rb')}
+
+ logging.info("Uploading of file %s started." % filename)
+
+ upload_submit = self.session.post(UPLOAD_URL,
+ data=upload_data,
+ files=files,
+ headers={'X-CSRFToken': upload_page.cookies['csrftoken'],
+ 'referer': UPLOAD_URL},
+ verify=False,
+ auth=self.auth)
+
if upload_submit.status_code != 200:
- print "Connection error. Status code: %s" % upload_submit.status_code
+ logging.warning("Connection error. Status code: %s" % upload_submit.status_code)
return False
-
- measurement_id = True
-
+
# Check if there was a redirect to a new page.
if upload_submit.url == UPLOAD_URL:
measurement_id = False
- print "Uploaded file rejected! Try to upload manually to see the error."
+ logging.error("Uploaded file rejected! Try to upload manually to see the error.")
else:
measurement_id = re.findall(regex, upload_submit.text)[0]
- print "Successfully uploaded measurement with id %s." % measurement_id
+ logging.error("Successfully uploaded measurement with id %s." % measurement_id)
return measurement_id
-
+
def download_files(self, measurement_id, subdir, download_url):
""" Downloads some files from the download_url to the specified
subdir. This method is used to download preprocessed file, optical
files etc.
- """
+ """
# Get the file
- request = self.session.get(download_url, auth = self.auth,
- verify = False,
- stream=True)
-
+ request = self.session.get(download_url, auth=self.auth,
+ verify=False,
+ stream=True)
+
# Create the dir if it does not exist
local_dir = os.path.join(self.output_dir, measurement_id, subdir)
if not os.path.exists(local_dir):
os.makedirs(local_dir)
-
-
+
# Save the file by chunk, needed if the file is big.
memory_file = StringIO.StringIO()
-
- for chunk in request.iter_content(chunk_size=1024):
- if chunk: # filter out keep-alive new chunks
+
+ for chunk in request.iter_content(chunk_size=1024):
+ if chunk: # filter out keep-alive new chunks
memory_file.write(chunk)
memory_file.flush()
-
+
zip_file = ZipFile(memory_file)
-
+
for ziped_name in zip_file.namelist():
basename = os.path.basename(ziped_name)
-
+
local_file = os.path.join(local_dir, basename)
-
+
with open(local_file, 'wb') as f:
f.write(zip_file.read(ziped_name))
@@ -172,13 +177,13 @@
# Construct the download url
download_url = DOWNLOAD_PREPROCESSED.format(measurement_id)
self.download_files(measurement_id, 'scc_preprocessed', download_url)
-
+
def download_optical(self, measurement_id):
""" Download optical files for the measurement id. """
# Construct the download url
download_url = DOWNLOAD_OPTICAL.format(measurement_id)
self.download_files(measurement_id, 'scc_optical', download_url)
-
+
def download_graphs(self, measurement_id):
""" Download profile graphs for the measurement id. """
# Construct the download url
@@ -194,22 +199,28 @@
stream=True)
if request.status_code != 200:
- print "Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code)
+ logging.error("Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code))
return
if monitor:
self.monitor_processing(measurement_id)
def rerun_all(self, measurement_id, monitor=True):
+ logger.debug("Started rerun_all procedure.")
+
+ logger.debug("Getting measurement %s" % measurement_id)
measurement = self.get_measurement(measurement_id)
if measurement:
+ logger.debug("Attempting to rerun all processing through %s." % measurement.rerun_all_url)
+
request = self.session.get(measurement.rerun_all_url, auth=self.auth,
verify=False,
stream=True)
if request.status_code != 200:
- print "Could not rerun pre processing for %s. Status code: %s" % (measurement_id, request.status_code)
+ logger.error("Could not rerun pre processing for %s. Status code: %s" %
+ (measurement_id, request.status_code))
return
if monitor:
@@ -219,7 +230,7 @@
""" Upload a file for processing and wait for the processing to finish.
If the processing is successful, it will download all produced files.
"""
- print "--- Processing started on %s. ---" % datetime.datetime.now()
+ logger.info("--- Processing started on %s. ---" % datetime.datetime.now())
# Upload file
measurement_id = self.upload_file(filename, system_id)
@@ -232,59 +243,59 @@
measurement = self.get_measurement(measurement_id)
if measurement is not None:
while measurement.is_running:
- print "Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload,
+ logger.info("Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload,
measurement.pre_processing,
- measurement.opt_retrievals)
+ measurement.processing))
time.sleep(10)
measurement = self.get_measurement(measurement_id)
- print "Measurement processing finished (status: %s, %s, %s)." % (measurement.upload,
+ logger.info("Measurement processing finished (status: %s, %s, %s)." % (measurement.upload,
measurement.pre_processing,
- measurement.opt_retrievals)
+ measurement.processing))
if measurement.pre_processing == 127:
- print "Downloading preprocessed files."
+ logger.info("Downloading preprocessed files.")
self.download_preprocessed(measurement_id)
- if measurement.opt_retrievals == 127:
- print "Downloading optical files."
+ if measurement.processing == 127:
+ logger.info("Downloading optical files.")
self.download_optical(measurement_id)
- print "Downloading graphs."
+ logger.info("Downloading graphs.")
self.download_graphs(measurement_id)
- print "--- Processing finished. ---"
+ logger.info("--- Processing finished. ---")
return measurement
def get_status(self, measurement_id):
""" Get the processing status for a measurement id through the API. """
measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__exact=%s' % measurement_id)
-
- response = self.session.get(measurement_url,
- auth = self.auth,
- verify = False)
-
+
+ response = self.session.get(measurement_url,
+ auth=self.auth,
+ verify=False)
+
response_dict = response.json()
-
+
if response_dict['objects']:
measurement_list = response_dict['objects']
measurement = Measurement(measurement_list[0])
- return (measurement.upload, measurement.pre_processing, measurement.opt_retrievals)
+ return (measurement.upload, measurement.pre_processing, measurement.processing)
else:
- print "No measurement with id %s found on the SCC." % measurement_id
+ logger.error("No measurement with id %s found on the SCC." % measurement_id)
return None
-
+
def get_measurement(self, measurement_id):
measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/%s/' % measurement_id)
-
- response = self.session.get(measurement_url,
- auth = self.auth,
- verify = False)
-
+
+ response = self.session.get(measurement_url,
+ auth=self.auth,
+ verify=False)
+
response_dict = response.json()
-
+
if response_dict:
measurement = Measurement(response_dict)
return measurement
else:
- print "No measurement with id %s found on the SCC." % measurement_id
+ logger.error("No measurement with id %s found on the SCC." % measurement_id)
return None
-
+
def delete_measurement(self, measurement_id):
""" Deletes a measurement with the provided measurement id. The user
should have the appropriate permissions.
@@ -294,94 +305,94 @@
"""
# Get the measurement object
measurement = self.get_measurement(measurement_id)
-
+
# Check that it exists
if measurement is None:
- print "Nothing to delete."
+ logger.warning("Nothing to delete.")
return None
-
+
# Go the the page confirming the deletion
delete_url = DELETE_MEASUREMENT.format(measurement.id)
-
- confirm_page = self.session.get(delete_url,
- auth = self.auth,
- verify = False)
-
+
+ confirm_page = self.session.get(delete_url,
+ auth=self.auth,
+ verify=False)
+
# Check that the page opened properly
if confirm_page.status_code != 200:
- print "Could not open delete page. Status: {0}".format(confirm_page.status_code)
+ logger.warning("Could not open delete page. Status: {0}".format(confirm_page.status_code))
return None
-
+
# Delete the measurement
- delete_page = self.session.post(delete_url,
- auth=self.auth,
- verify=False,
- data={'post':'yes'},
- headers={'X-CSRFToken': confirm_page.cookies['csrftoken'],
+ delete_page = self.session.post(delete_url,
+ auth=self.auth,
+ verify=False,
+ data={'post': 'yes'},
+ headers={'X-CSRFToken': confirm_page.cookies['csrftoken'],
'referer': delete_url}
- )
+ )
if delete_page.status_code != 200:
- print "Something went wrong. Delete page status: {0}".format(
- delete_page.status_code)
+ logger.warning("Something went wrong. Delete page status: {0}".format(
+ delete_page.status_code))
return None
-
- print "Deleted measurement {0}".format(measurement_id)
- return True
+
+ logger.info("Deleted measurement {0}".format(measurement_id))
+ return True
def available_measurements(self):
""" Get a list of available measurement on the SCC. """
measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements')
- response = self.session.get(measurement_url,
- auth = self.auth,
- verify = False)
+ response = self.session.get(measurement_url,
+ auth=self.auth,
+ verify=False)
response_dict = response.json()
-
+
measurements = None
if response_dict:
measurement_list = response_dict['objects']
measurements = [Measurement(measurement_dict) for measurement_dict in measurement_list]
- print "Found %s measurements on the SCC." % len(measurements)
+ logger.info("Found %s measurements on the SCC." % len(measurements))
else:
- print "No response received from the SCC when asked for available measurements."
+ logger.warning("No response received from the SCC when asked for available measurements.")
return measurements
-
- def measurement_id_for_date(self, t1, call_sign = 'bu', base_number = 0):
+
+ def measurement_id_for_date(self, t1, call_sign='bu', base_number=0):
""" Give the first available measurement id on the SCC for the specific
date.
"""
date_str = t1.strftime('%Y%m%d')
search_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__startswith=%s' % date_str)
-
- response = self.session.get(search_url,
- auth = self.auth,
- verify = False)
-
+
+ response = self.session.get(search_url,
+ auth=self.auth,
+ verify=False)
+
response_dict = response.json()
-
+
measurement_id = None
-
+
if response_dict:
measurement_list = response_dict['objects']
existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list]
-
+
measurement_number = base_number
measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number)
-
+
while measurement_id in existing_ids:
measurement_number = measurement_number + 1
measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number)
if measurement_number == 100:
raise ValueError('No available measurement id found.')
- return measurement_id
-
-
+ return measurement_id
+
+
class ApiObject:
""" A generic class object. """
-
+
def __init__(self, dict_response):
-
+
if dict_response:
# Add the dictionary key value pairs as object properties
for key, value in dict_response.items():
@@ -394,6 +405,7 @@
class Measurement(ApiObject):
""" This class represents the measurement object as returned in the SCC API.
"""
+
@property
def is_running(self):
""" Returns True if the processing has not finished.
@@ -403,74 +415,100 @@
if self.pre_processing == -127:
return False
if self.pre_processing == 127:
- if self.opt_retrievals in [127, -127]:
+ if self.processing in [127, -127]:
return False
return True
-
+
+ @property
def rerun_processing_url(self):
return RERUN_PROCESSING.format(self.id)
+ @property
def rerun_all_url(self):
return RERUN_ALL.format(self.id)
def __str__(self):
- return "%s: %s, %s, %s" % (self.id,
- self.upload,
- self.pre_processing,
- self.opt_retrievals)
-
-
-def upload_file(filename, system_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN):
+ return "%s: %s, %s, %s" % (self.id,
+ self.upload,
+ self.pre_processing,
+ self.processing)
+
+
+def upload_file(filename, system_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN):
""" Shortcut function to upload a file to the SCC. """
+ logger.info("Uploading file %s, using sytem %s" % (filename, system_id))
+
scc = SCC(auth)
scc.login(credential)
measurement_id = scc.upload_file(filename, system_id)
scc.logout()
return measurement_id
-def process_file(filename, system_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN):
+
+def process_file(filename, system_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN):
""" Shortcut function to process a file to the SCC. """
+ logger.info("Processing file %s, using sytem %s" % (filename, system_id))
+
scc = SCC(auth)
scc.login(credential)
measurement = scc.process(filename, system_id)
scc.logout()
- return measurement
+ return measurement
+
-def delete_measurement(measurement_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN):
+def delete_measurement(measurement_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN):
""" Shortcut function to delete a measurement from the SCC. """
+ logger.info("Deleting %s" % measurement_id)
scc = SCC(auth)
scc.login(credential)
scc.delete_measurement(measurement_id)
scc.logout()
-def rerun_all(measurement_id, monitor, auth = BASIC_LOGIN, credential = DJANGO_LOGIN):
+
+def rerun_all(measurement_id, monitor, auth=BASIC_LOGIN, credential=DJANGO_LOGIN):
""" Shortcut function to delete a measurement from the SCC. """
+ logger.info("Rerunning all products for %s" % measurement_id)
scc = SCC(auth)
scc.login(credential)
scc.rerun_all(measurement_id, monitor)
scc.logout()
-def rerun_processing(measurement_id, monitor, auth = BASIC_LOGIN, credential = DJANGO_LOGIN):
+
+def rerun_processing(measurement_id, monitor, auth=BASIC_LOGIN, credential=DJANGO_LOGIN):
""" Shortcut function to delete a measurement from the SCC. """
+ logger.info("Rerunning (optical) processing for %s" % measurement_id)
scc = SCC(auth)
scc.login(credential)
scc.rerun_processing(measurement_id, monitor)
scc.logout()
+
# When running through terminal
if __name__ == '__main__':
-
+
# Define the command line arguments.
parser = argparse.ArgumentParser()
- parser.add_argument("filename", nargs='?', help = "Measurement file name or path.", default='')
- parser.add_argument("system", nargs='?', help = "Processing system id.", default=0)
+ parser.add_argument("filename", nargs='?', help="Measurement file name or path.", default='')
+ parser.add_argument("system", nargs='?', help="Processing system id.", default=0)
parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
- action="store_true")
- parser.add_argument("-d", "--delete", help="Measurement ID to delete.")
+ action="store_true")
+ parser.add_argument("--delete", help="Measurement ID to delete.")
parser.add_argument("--rerun-all", help="Measurement ID to rerun.")
parser.add_argument("--rerun-processing", help="Measurement ID to rerun processing routings.")
+
+ # Verbosity settings from http://stackoverflow.com/a/20663028
+ parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const",
+ dest="loglevel", const=logging.DEBUG, default=logging.INFO,
+ )
+ parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const",
+ dest="loglevel", const=logging.WARNING
+ )
+
args = parser.parse_args()
-
+
+ # Get the logger with the appropriate level
+ logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel)
+
# If the arguments are OK, try to login on the site and upload.
if args.delete:
# If the delete is provided, do nothing else
@@ -480,9 +518,9 @@
elif args.rerun_processing:
rerun_processing(args.rerun_processing, args.process)
else:
- if (args.filename == '') or (args.system == 0):
+ if (args.filename == '') or (args.system == 0):
parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n')
-
+
if args.process:
process_file(args.filename, args.system)
else: