--- a/scc_access/scc_access.py Wed Feb 03 16:18:14 2021 +0200
+++ b/scc_access/scc_access.py Thu Feb 04 10:19:13 2021 +0200
@@ -52,6 +52,7 @@
self.list_measurements_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/')
self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/')
+ self.measurement_page_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/')
self.download_hirelpp_pattern = urlparse.urljoin(self.base_url,
'data_processing/measurements/{0}/download-hirelpp/')
self.download_cloudmask_pattern = urlparse.urljoin(self.base_url,
@@ -100,7 +101,7 @@
""" Logout from SCC """
return self.session.get(self.logout_url, stream=True)
- def upload_file(self, filename, system_id, force_upload, delete_related, rs_filename=None, ov_filename=None, lr_filename=None):
+ def upload_file(self, filename, system_id, force_upload, delete_related, delay=0, rs_filename=None, ov_filename=None, lr_filename=None):
""" Upload a filename for processing with a specific system. If the
upload is successful, it returns the measurement id. """
measurement_id = self.measurement_id_from_file(filename)
@@ -123,7 +124,11 @@
upload_page = self.session.get(self.upload_url)
# Submit the data
- upload_data = {'system': system_id}
+ upload_data = {'system': system_id,
+ 'delay': delay}
+
+ logger.debug("Submitted processing parameters - System: {}, Delay: {}".format(system_id, delay))
+
files = {'data': open(filename, 'rb')}
if rs_filename is not None:
@@ -173,7 +178,7 @@
else:
measurement_id = re.findall(regex, upload_submit.text)[0]
logger.info("Successfully uploaded measurement with id %s." % measurement_id)
-
+ logger.info("You can monitor the processing progress online: {}".format(self.measurement_page_pattern.format(measurement_id)))
return measurement_id
@staticmethod
@@ -339,7 +344,7 @@
if monitor:
self.monitor_processing(measurement_id)
- def process(self, filename, system_id, monitor, force_upload, delete_related, rs_filename=None, lr_filename=None, ov_filename=None):
+ def process(self, filename, system_id, monitor, force_upload, delete_related, delay=0, rs_filename=None, lr_filename=None, ov_filename=None):
""" Upload a file for processing and wait for the processing to finish.
If the processing is successful, it will download all produced files.
"""
@@ -347,12 +352,17 @@
# Upload file
logger.info("Uploading file.")
measurement_id = self.upload_file(filename, system_id, force_upload, delete_related,
+ delay=delay,
rs_filename=rs_filename,
lr_filename=lr_filename,
ov_filename=ov_filename)
+ if monitor and (delay > 0):
+ logger.warning("Will not start monitoring, since a delay was specified: {} hours.".format(delay))
+ return None
+
if measurement_id and monitor:
- logger.info("Monitoring processing")
+ logger.info("Monitoring processing.")
return self.monitor_processing(measurement_id)
return None
@@ -708,7 +718,10 @@
def _log_module_status(self, name, status, exit_code):
if exit_code:
- logger.warning("{0} exit code: {2[exit_code]} - {2[description]}".format(name, status, exit_code))
+ if exit_code['exit_code'] > 0:
+ logger.warning("{0} exit code: {2[exit_code]} - {2[description]}".format(name, status, exit_code))
+ else:
+ logger.info("{0} exit code: {2[exit_code]} - {2[description]}".format(name, status, exit_code))
else:
logger.info("{0} exit code: {2}".format(name, status, exit_code))
@@ -748,7 +761,7 @@
def process_file(filename, system_id, settings, force_upload, delete_related,
- monitor=True, rs_filename=None, lr_filename=None, ov_filename=None):
+ delay=0, monitor=True, rs_filename=None, lr_filename=None, ov_filename=None):
""" Shortcut function to process a file to the SCC. """
logger.info("Processing file %s, using system %s" % (filename, system_id))
@@ -757,6 +770,7 @@
measurement = scc.process(filename, system_id,
force_upload=force_upload,
delete_related=delete_related,
+ delay=delay,
monitor=monitor,
rs_filename=rs_filename,
lr_filename=lr_filename,
@@ -770,7 +784,7 @@
with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
scc.login(settings['website_credentials'])
for m_id in measurement_ids:
- logger.info("Deleting %s" % m_id)
+ logger.info("Deleting %s." % m_id)
scc.delete_measurement(m_id, delete_related)
scc.logout()
@@ -781,7 +795,7 @@
with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
scc.login(settings['website_credentials'])
for m_id in measurement_ids:
- logger.info("Rerunning all products for %s" % m_id)
+ logger.info("Rerunning all products for %s." % m_id)
scc.rerun_all(m_id, monitor)
scc.logout()
@@ -886,6 +900,7 @@
""" Upload but do not monitor processing progress. """
def upload_file_from_args(parsed):
process_file(parsed.filename, parsed.system, parsed.config,
+ delay=parsed.delay,
monitor=parsed.process,
force_upload=parsed.force_upload,
delete_related=False, # For now, use this as default
@@ -893,8 +908,21 @@
ov_filename=parsed.overlap,
lr_filename=parsed.lidarratio)
+ def delay(arg):
+ try:
+ int_arg = int(arg)
+ except ValueError:
+ raise argparse.ArgumentTypeError("Could not convert delay argument {} to integer.".format(arg))
+
+ if 0 <= int_arg <= 96:
+ return int_arg
+ else:
+ raise argparse.ArgumentTypeError("Delay should be an integer between 0 and 96.")
+
parser.add_argument("filename", help="Measurement file name or path.")
parser.add_argument("system", help="Processing system id.")
+ parser.add_argument("--delay", help="Delay processing by the specified number of hours (0 to 96).",
+ default=0, type=delay)
parser.add_argument("-p", "--process", help="Wait for the processing results.",
action="store_true")
parser.add_argument("--force_upload", help="If measurement ID exists on SCC, delete before uploading.",
