--- a/scc_access/scc_access.py Fri Dec 15 22:51:47 2017 +0200
+++ b/scc_access/scc_access.py Fri Dec 15 22:53:17 2017 +0200
@@ -1,41 +1,7 @@
-#!/usr/bin/env python
-"""
-The MIT License (MIT)
-
-Copyright (c) 2015, Ioannis Binietoglou
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-"""
-
-__version__ = "0.6.0"
-
-# Try to read the settings from the settings.py file
-try:
- from settings import *
-except:
- raise ImportError(
- """A settings file (setting.py) is required to run the script.
- You can use settings.sample.py as a template.""")
-
import requests
requests.packages.urllib3.disable_warnings()
+import sys
import urlparse
import argparse
import os
@@ -45,21 +11,10 @@
from zipfile import ZipFile
import datetime
import logging
-
-logger = logging.getLogger(__name__)
+import yaml
-# Construct the absolute URLs
-LOGIN_URL = urlparse.urljoin(BASE_URL, 'accounts/login/')
-UPLOAD_URL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/quick/')
-DOWNLOAD_PREPROCESSED = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-preprocessed/')
-DOWNLOAD_OPTICAL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-optical/')
-DOWNLOAD_GRAPH = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-plots/')
-RERUN_ALL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/rerun-all/')
-RERUN_PROCESSING = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/rerun-optical/')
-
-DELETE_MEASUREMENT = urlparse.urljoin(BASE_URL, 'admin/database/measurements/{0}/delete/')
-API_BASE_URL = urlparse.urljoin(BASE_URL, 'api/v1/')
+logger = logging.getLogger(__name__)
# The regex to find the measurement id from the measurement page
# This should be read from the uploaded file, but would require an extra NetCDF module.
@@ -68,34 +23,52 @@
class SCC:
""" A simple class that will attempt to upload a file on the SCC server.
+
The uploading is done by simulating a normal browser session. In the current
version no check is performed, and no feedback is given if the upload
was successful. If everything is setup correctly, it will work.
"""
- def __init__(self, auth=BASIC_LOGIN, output_dir=OUTPUT_DIR):
+ def __init__(self, auth, output_dir, base_url):
self.auth = auth
self.output_dir = output_dir
+ self.base_url = base_url
self.session = requests.Session()
+ self.construct_urls()
- def login(self, credentials=DJANGO_LOGIN):
+ def construct_urls(self):
+ """ Construct all URLs needed for processing. """
+ # Construct the absolute URLs
+ self.login_url = urlparse.urljoin(self.base_url, 'accounts/login/')
+ self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/')
+ self.download_preprocessed_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-preprocessed/')
+ self.download_optical_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-optical/')
+ self.download_graph_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-plots/')
+ self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/')
+ self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/')
+
+ def login(self, credentials):
""" Login the the website. """
logger.debug("Attempting to login to SCC, username %s." % credentials[0])
self.login_credentials = {'username': credentials[0],
'password': credentials[1]}
- logger.debug("Accessing login page at %s." % LOGIN_URL)
+ logger.debug("Accessing login page at %s." % self.login_url)
# Get upload form
- login_page = self.session.get(LOGIN_URL,
- auth=self.auth, verify=False)
+ login_page = self.session.get(self.login_url, auth=self.auth, verify=False)
+
+ if login_page.status_code != 200:
+ logger.error('Could not access login pages. Status code %s' % login_page.status_code)
+ sys.exit(1)
logger.debug("Submiting credentials.")
+
# Submit the login data
- login_submit = self.session.post(LOGIN_URL,
+ login_submit = self.session.post(self.login_url,
data=self.login_credentials,
headers={'X-CSRFToken': login_page.cookies['csrftoken'],
- 'referer': LOGIN_URL},
+ 'referer': self.login_url},
verify=False,
auth=self.auth)
return login_submit
@@ -107,7 +80,7 @@
""" Upload a filename for processing with a specific system. If the
upload is successful, it returns the measurement id. """
# Get submit page
- upload_page = self.session.get(UPLOAD_URL,
+ upload_page = self.session.get(self.upload_url,
auth=self.auth,
verify=False)
@@ -115,27 +88,27 @@
upload_data = {'system': system_id}
files = {'data': open(filename, 'rb')}
- logging.info("Uploading of file %s started." % filename)
+ logger.info("Uploading of file %s started." % filename)
- upload_submit = self.session.post(UPLOAD_URL,
+ upload_submit = self.session.post(self.upload_url,
data=upload_data,
files=files,
headers={'X-CSRFToken': upload_page.cookies['csrftoken'],
- 'referer': UPLOAD_URL},
+ 'referer': self.upload_url,},
verify=False,
auth=self.auth)
if upload_submit.status_code != 200:
- logging.warning("Connection error. Status code: %s" % upload_submit.status_code)
+ logger.warning("Connection error. Status code: %s" % upload_submit.status_code)
return False
# Check if there was a redirect to a new page.
- if upload_submit.url == UPLOAD_URL:
+ if upload_submit.url == self.upload_url:
measurement_id = False
- logging.error("Uploaded file rejected! Try to upload manually to see the error.")
+ logger.error("Uploaded file rejected! Try to upload manually to see the error.")
else:
measurement_id = re.findall(regex, upload_submit.text)[0]
- logging.error("Successfully uploaded measurement with id %s." % measurement_id)
+ logger.error("Successfully uploaded measurement with id %s." % measurement_id)
return measurement_id
@@ -175,19 +148,19 @@
def download_preprocessed(self, measurement_id):
""" Download preprocessed files for the measurement id. """
# Construct the download url
- download_url = DOWNLOAD_PREPROCESSED.format(measurement_id)
+ download_url = self.download_preprocessed_pattern.format(measurement_id)
self.download_files(measurement_id, 'scc_preprocessed', download_url)
def download_optical(self, measurement_id):
""" Download optical files for the measurement id. """
# Construct the download url
- download_url = DOWNLOAD_OPTICAL.format(measurement_id)
+ download_url = self.download_optical.format(measurement_id)
self.download_files(measurement_id, 'scc_optical', download_url)
def download_graphs(self, measurement_id):
""" Download profile graphs for the measurement id. """
# Construct the download url
- download_url = DOWNLOAD_GRAPH.format(measurement_id)
+ download_url = self.download_graph_pattern.format(measurement_id)
self.download_files(measurement_id, 'scc_plots', download_url)
def rerun_processing(self, measurement_id, monitor=True):
@@ -199,7 +172,8 @@
stream=True)
if request.status_code != 200:
- logging.error("Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code))
+ logger.error(
+ "Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code))
return
if monitor:
@@ -244,13 +218,13 @@
if measurement is not None:
while measurement.is_running:
logger.info("Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload,
- measurement.pre_processing,
- measurement.processing))
+ measurement.pre_processing,
+ measurement.processing))
time.sleep(10)
measurement = self.get_measurement(measurement_id)
logger.info("Measurement processing finished (status: %s, %s, %s)." % (measurement.upload,
- measurement.pre_processing,
- measurement.processing))
+ measurement.pre_processing,
+ measurement.processing))
if measurement.pre_processing == 127:
logger.info("Downloading preprocessed files.")
self.download_preprocessed(measurement_id)
@@ -264,7 +238,7 @@
def get_status(self, measurement_id):
""" Get the processing status for a measurement id through the API. """
- measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__exact=%s' % measurement_id)
+ measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__exact=%s' % measurement_id)
response = self.session.get(measurement_url,
auth=self.auth,
@@ -274,23 +248,27 @@
if response_dict['objects']:
measurement_list = response_dict['objects']
- measurement = Measurement(measurement_list[0])
- return (measurement.upload, measurement.pre_processing, measurement.processing)
+ measurement = Measurement(self.base_url, measurement_list[0])
+ return measurement.upload, measurement.pre_processing, measurement.processing
else:
logger.error("No measurement with id %s found on the SCC." % measurement_id)
return None
def get_measurement(self, measurement_id):
- measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/%s/' % measurement_id)
+ measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/%s/' % measurement_id)
response = self.session.get(measurement_url,
auth=self.auth,
verify=False)
+ if response.status_code != 200:
+ logger.error('Could not access API. Status code %s.' % response.status_code)
+ sys.exit(1)
+
response_dict = response.json()
if response_dict:
- measurement = Measurement(response_dict)
+ measurement = Measurement(self.base_url,response_dict)
return measurement
else:
logger.error("No measurement with id %s found on the SCC." % measurement_id)
@@ -312,7 +290,7 @@
return None
# Go the the page confirming the deletion
- delete_url = DELETE_MEASUREMENT.format(measurement.id)
+ delete_url = self.delete_measurement_pattern.format(measurement.id)
confirm_page = self.session.get(delete_url,
auth=self.auth,
@@ -341,7 +319,7 @@
def available_measurements(self):
""" Get a list of available measurement on the SCC. """
- measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements')
+ measurement_url = urlparse.urljoin(self.api_base_url, 'measurements')
response = self.session.get(measurement_url,
auth=self.auth,
verify=False)
@@ -350,7 +328,7 @@
measurements = None
if response_dict:
measurement_list = response_dict['objects']
- measurements = [Measurement(measurement_dict) for measurement_dict in measurement_list]
+ measurements = [Measurement(self.base_url, measurement_dict) for measurement_dict in measurement_list]
logger.info("Found %s measurements on the SCC." % len(measurements))
else:
logger.warning("No response received from the SCC when asked for available measurements.")
@@ -362,7 +340,7 @@
date.
"""
date_str = t1.strftime('%Y%m%d')
- search_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__startswith=%s' % date_str)
+ search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % date_str)
response = self.session.get(search_url,
auth=self.auth,
@@ -391,7 +369,8 @@
class ApiObject:
""" A generic class object. """
- def __init__(self, dict_response):
+ def __init__(self, base_url, dict_response):
+ self.base_url = base_url
if dict_response:
# Add the dictionary key value pairs as object properties
@@ -421,11 +400,13 @@
@property
def rerun_processing_url(self):
- return RERUN_PROCESSING.format(self.id)
+ url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-optical/')
+ return url_pattern.format(self.id)
@property
def rerun_all_url(self):
- return RERUN_ALL.format(self.id)
+ ulr_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-all/')
+ return ulr_pattern.format(self.id)
def __str__(self):
return "%s: %s, %s, %s" % (self.id,
@@ -434,64 +415,92 @@
self.processing)
-def upload_file(filename, system_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN):
+def upload_file(filename, system_id, settings):
""" Shortcut function to upload a file to the SCC. """
logger.info("Uploading file %s, using sytem %s" % (filename, system_id))
- scc = SCC(auth)
- scc.login(credential)
+ scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
+ scc.login(settings['website_credentials'])
measurement_id = scc.upload_file(filename, system_id)
scc.logout()
return measurement_id
-def process_file(filename, system_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN):
+def process_file(filename, system_id, settings):
""" Shortcut function to process a file to the SCC. """
logger.info("Processing file %s, using sytem %s" % (filename, system_id))
- scc = SCC(auth)
- scc.login(credential)
+ scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
+ scc.login(settings['website_credentials'])
measurement = scc.process(filename, system_id)
scc.logout()
return measurement
-def delete_measurement(measurement_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN):
+def delete_measurement(measurement_id, settings):
""" Shortcut function to delete a measurement from the SCC. """
logger.info("Deleting %s" % measurement_id)
- scc = SCC(auth)
- scc.login(credential)
+
+ scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
+ scc.login(settings['website_credentials'])
scc.delete_measurement(measurement_id)
scc.logout()
-def rerun_all(measurement_id, monitor, auth=BASIC_LOGIN, credential=DJANGO_LOGIN):
+def rerun_all(measurement_id, monitor, settings):
""" Shortcut function to delete a measurement from the SCC. """
logger.info("Rerunning all products for %s" % measurement_id)
- scc = SCC(auth)
- scc.login(credential)
+
+ scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
+ scc.login(settings['website_credentials'])
scc.rerun_all(measurement_id, monitor)
scc.logout()
-def rerun_processing(measurement_id, monitor, auth=BASIC_LOGIN, credential=DJANGO_LOGIN):
+def rerun_processing(measurement_id, monitor, settings):
""" Shortcut function to delete a measurement from the SCC. """
logger.info("Rerunning (optical) processing for %s" % measurement_id)
- scc = SCC(auth)
- scc.login(credential)
+
+ scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
+ scc.login(settings['website_credentials'])
scc.rerun_processing(measurement_id, monitor)
scc.logout()
-
+
+
+def import_settings(config_file_path):
+ """ Read the configuration file.
+
+ The file should be in YAML syntax."""
+
+ if not os.path.isfile(config_file_path):
+ logger.error("Wrong path for configuration file (%s)" % config_file_path)
+ sys.exit(1)
+
+ with open(config_file_path) as yaml_file:
+ try:
+ settings = yaml.safe_load(yaml_file)
+ logger.debug("Read settings file(%s)" % config_file_path)
+ except:
+ logger.error("Could not parse YAML file (%s)" % config_file_path)
+ sys.exit(1)
+
+ # YAML limitation: does not read tuples
+ settings['basic_credentials'] = tuple(settings['basic_credentials'])
+ settings['website_credentials'] = tuple(settings['website_credentials'])
+ return settings
+
+
def main():
# Define the command line arguments.
parser = argparse.ArgumentParser()
parser.add_argument("filename", nargs='?', help="Measurement file name or path.", default='')
parser.add_argument("system", nargs='?', help="Processing system id.", default=0)
+ parser.add_argument("-c", "--config", nargs='?', help="Path to configuration file")
parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
action="store_true")
parser.add_argument("--delete", help="Measurement ID to delete.")
parser.add_argument("--rerun-all", help="Measurement ID to rerun.")
- parser.add_argument("--rerun-processing", help="Measurement ID to rerun processing routings.")
+ parser.add_argument("--rerun-processing", help="Measurement ID to rerun processing routines.")
# Verbosity settings from http://stackoverflow.com/a/20663028
parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const",
@@ -506,24 +515,21 @@
# Get the logger with the appropriate level
logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel)
- # If the arguments are OK, try to login on the site and upload.
+ settings = import_settings(args.config)
+
+ # If the arguments are OK, try to log-in to SCC and upload.
if args.delete:
# If the delete is provided, do nothing else
- delete_measurement(args.delete)
+ delete_measurement(args.delete, settings)
elif args.rerun_all:
- rerun_all(args.rerun_all, args.process)
+ rerun_all(args.rerun_all, args.process, settings)
elif args.rerun_processing:
- rerun_processing(args.rerun_processing, args.process)
+ rerun_processing(args.rerun_processing, args.process, settings)
else:
if (args.filename == '') or (args.system == 0):
parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n')
if args.process:
- process_file(args.filename, args.system)
+ process_file(args.filename, args.system, settings)
else:
- upload_file(args.filename, args.system)
-
-
-# When running through terminal
-if __name__ == '__main__':
- main()
+ upload_file(args.filename, args.system, settings)
