diff -r 934a5e0a05c9 -r 5a2e8efdaae9 scc_access/scc_access.py --- a/scc_access/scc_access.py Wed Feb 03 16:18:14 2021 +0200 +++ b/scc_access/scc_access.py Thu Feb 04 10:19:13 2021 +0200 @@ -52,6 +52,7 @@ self.list_measurements_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/') self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/') + self.measurement_page_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/') self.download_hirelpp_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-hirelpp/') self.download_cloudmask_pattern = urlparse.urljoin(self.base_url, @@ -100,7 +101,7 @@ """ Logout from SCC """ return self.session.get(self.logout_url, stream=True) - def upload_file(self, filename, system_id, force_upload, delete_related, rs_filename=None, ov_filename=None, lr_filename=None): + def upload_file(self, filename, system_id, force_upload, delete_related, delay=0, rs_filename=None, ov_filename=None, lr_filename=None): """ Upload a filename for processing with a specific system. If the upload is successful, it returns the measurement id. """ measurement_id = self.measurement_id_from_file(filename) @@ -123,7 +124,11 @@ upload_page = self.session.get(self.upload_url) # Submit the data - upload_data = {'system': system_id} + upload_data = {'system': system_id, + 'delay': delay} + + logger.debug("Submitted processing parameters - System: {}, Delay: {}".format(system_id, delay)) + files = {'data': open(filename, 'rb')} if rs_filename is not None: @@ -173,7 +178,7 @@ else: measurement_id = re.findall(regex, upload_submit.text)[0] logger.info("Successfully uploaded measurement with id %s." % measurement_id) - + logger.info("You can monitor the processing progress online: {}".format(self.measurement_page_pattern.format(measurement_id))) return measurement_id @staticmethod @@ -339,7 +344,7 @@ if monitor: self.monitor_processing(measurement_id) - def process(self, filename, system_id, monitor, force_upload, delete_related, rs_filename=None, lr_filename=None, ov_filename=None): + def process(self, filename, system_id, monitor, force_upload, delete_related, delay=0, rs_filename=None, lr_filename=None, ov_filename=None): """ Upload a file for processing and wait for the processing to finish. If the processing is successful, it will download all produced files. """ @@ -347,12 +352,17 @@ # Upload file logger.info("Uploading file.") measurement_id = self.upload_file(filename, system_id, force_upload, delete_related, + delay=delay, rs_filename=rs_filename, lr_filename=lr_filename, ov_filename=ov_filename) + if monitor and (delay > 0): + logger.warning("Will not start monitoring, since a delay was specified: {} hours.".format(delay)) + return None + if measurement_id and monitor: - logger.info("Monitoring processing") + logger.info("Monitoring processing.") return self.monitor_processing(measurement_id) return None @@ -708,7 +718,10 @@ def _log_module_status(self, name, status, exit_code): if exit_code: - logger.warning("{0} exit code: {2[exit_code]} - {2[description]}".format(name, status, exit_code)) + if exit_code['exit_code'] > 0: + logger.warning("{0} exit code: {2[exit_code]} - {2[description]}".format(name, status, exit_code)) + else: + logger.info("{0} exit code: {2[exit_code]} - {2[description]}".format(name, status, exit_code)) else: logger.info("{0} exit code: {2}".format(name, status, exit_code)) @@ -748,7 +761,7 @@ def process_file(filename, system_id, settings, force_upload, delete_related, - monitor=True, rs_filename=None, lr_filename=None, ov_filename=None): + delay=0, monitor=True, rs_filename=None, lr_filename=None, ov_filename=None): """ Shortcut function to process a file to the SCC. """ logger.info("Processing file %s, using system %s" % (filename, system_id)) @@ -757,6 +770,7 @@ measurement = scc.process(filename, system_id, force_upload=force_upload, delete_related=delete_related, + delay=delay, monitor=monitor, rs_filename=rs_filename, lr_filename=lr_filename, @@ -770,7 +784,7 @@ with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: scc.login(settings['website_credentials']) for m_id in measurement_ids: - logger.info("Deleting %s" % m_id) + logger.info("Deleting %s." % m_id) scc.delete_measurement(m_id, delete_related) scc.logout() @@ -781,7 +795,7 @@ with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: scc.login(settings['website_credentials']) for m_id in measurement_ids: - logger.info("Rerunning all products for %s" % m_id) + logger.info("Rerunning all products for %s." % m_id) scc.rerun_all(m_id, monitor) scc.logout() @@ -886,6 +900,7 @@ """ Upload but do not monitor processing progress. """ def upload_file_from_args(parsed): process_file(parsed.filename, parsed.system, parsed.config, + delay=parsed.delay, monitor=parsed.process, force_upload=parsed.force_upload, delete_related=False, # For now, use this as default @@ -893,8 +908,21 @@ ov_filename=parsed.overlap, lr_filename=parsed.lidarratio) + def delay(arg): + try: + int_arg = int(arg) + except ValueError: + raise argparse.ArgumentTypeError("Could not convert delay argument {} to integer.".format(arg)) + + if 0 <= int_arg <= 96: + return int_arg + else: + raise argparse.ArgumentTypeError("Delay should be an integer between 0 and 96.") + parser.add_argument("filename", help="Measurement file name or path.") parser.add_argument("system", help="Processing system id.") + parser.add_argument("--delay", help="Delay processing by the specified number of hours (0 to 96).", + default=0, type=delay) parser.add_argument("-p", "--process", help="Wait for the processing results.", action="store_true") parser.add_argument("--force_upload", help="If measurement ID exists on SCC, delete before uploading.",