# HG changeset patch # User Iannis # Date 1511537434 -7200 # Node ID 809c63be8a4008c5ee789db7ea7ebfc9879ea556 # Parent 2f7cde6f836ef262ccccd3517917a382e805c17c Commiting old left-over changes. Added re-run options. diff -r 2f7cde6f836e -r 809c63be8a40 .hgignore --- a/.hgignore Tue Jun 23 11:13:04 2015 +0300 +++ b/.hgignore Fri Nov 24 17:30:34 2017 +0200 @@ -3,3 +3,4 @@ *.py~ *.pyc re:^settings\.py$ +re:^\.idea/ diff -r 2f7cde6f836e -r 809c63be8a40 scc_access.py --- a/scc_access.py Tue Jun 23 11:13:04 2015 +0300 +++ b/scc_access.py Fri Nov 24 17:30:34 2017 +0200 @@ -23,7 +23,7 @@ THE SOFTWARE. """ -__version__ = "0.5.0" +__version__ = "0.6.0" # Try to read the settings from the settings.py file @@ -32,7 +32,7 @@ except: raise ImportError( """A settings file (setting.py) is required to run the script. - You can use settings.sample.py for a template.""") + You can use settings.sample.py as a template.""") import requests @@ -52,6 +52,9 @@ DOWNLOAD_PREPROCESSED = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-preprocessed/') DOWNLOAD_OPTICAL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-optical/') DOWNLOAD_GRAPH = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-plots/') +RERUN_ALL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/rerun-all/') +RERUN_PROCESSING = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/rerun-optical/') + DELETE_MEASUREMENT = urlparse.urljoin(BASE_URL, 'admin/database/measurements/{0}/delete/') API_BASE_URL = urlparse.urljoin(BASE_URL, 'api/v1/') @@ -181,26 +184,61 @@ # Construct the download url download_url = DOWNLOAD_GRAPH.format(measurement_id) self.download_files(measurement_id, 'scc_plots', download_url) - + + def rerun_processing(self, measurement_id, monitor=True): + measurement = self.get_measurement(measurement_id) + + if measurement: + request = self.session.get(measurement.rerun_processing_url, auth=self.auth, + verify=False, + stream=True) + + if request.status_code != 200: + print "Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code) + return + + if monitor: + self.monitor_processing(measurement_id) + + def rerun_all(self, measurement_id, monitor=True): + measurement = self.get_measurement(measurement_id) + + if measurement: + request = self.session.get(measurement.rerun_all_url, auth=self.auth, + verify=False, + stream=True) + + if request.status_code != 200: + print "Could not rerun pre processing for %s. Status code: %s" % (measurement_id, request.status_code) + return + + if monitor: + self.monitor_processing(measurement_id) + def process(self, filename, system_id): """ Upload a file for processing and wait for the processing to finish. If the processing is successful, it will download all produced files. """ print "--- Processing started on %s. ---" % datetime.datetime.now() # Upload file - measurement_id = self.upload_file(filename, system_id) - - measurement = None - if measurement_id: - measurement = self.get_measurement(measurement_id) + measurement_id = self.upload_file(filename, system_id) + + measurement = self.monitor_processing(measurement_id) + return measurement + + def monitor_processing(self, measurement_id): + """ Monitor the processing progress of a measurement id""" + + measurement = self.get_measurement(measurement_id) + if measurement is not None: while measurement.is_running: - print "Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload, - measurement.pre_processing, + print "Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload, + measurement.pre_processing, measurement.opt_retrievals) time.sleep(10) measurement = self.get_measurement(measurement_id) - print "Measurement processing finished (status: %s, %s, %s)." % (measurement.upload, - measurement.pre_processing, + print "Measurement processing finished (status: %s, %s, %s)." % (measurement.upload, + measurement.pre_processing, measurement.opt_retrievals) if measurement.pre_processing == 127: print "Downloading preprocessed files." @@ -210,9 +248,9 @@ self.download_optical(measurement_id) print "Downloading graphs." self.download_graphs(measurement_id) - print "--- Processing finished. ---" + print "--- Processing finished. ---" return measurement - + def get_status(self, measurement_id): """ Get the processing status for a measurement id through the API. """ measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__exact=%s' % measurement_id) @@ -369,10 +407,12 @@ return False return True - def delete(self): - """ Delete the entry from the SCC database. """ - - + def rerun_processing_url(self): + return RERUN_PROCESSING.format(self.id) + + def rerun_all_url(self): + return RERUN_ALL.format(self.id) + def __str__(self): return "%s: %s, %s, %s" % (self.id, self.upload, @@ -403,6 +443,20 @@ scc.delete_measurement(measurement_id) scc.logout() +def rerun_all(measurement_id, monitor, auth = BASIC_LOGIN, credential = DJANGO_LOGIN): + """ Shortcut function to delete a measurement from the SCC. """ + scc = SCC(auth) + scc.login(credential) + scc.rerun_all(measurement_id, monitor) + scc.logout() + +def rerun_processing(measurement_id, monitor, auth = BASIC_LOGIN, credential = DJANGO_LOGIN): + """ Shortcut function to delete a measurement from the SCC. """ + scc = SCC(auth) + scc.login(credential) + scc.rerun_processing(measurement_id, monitor) + scc.logout() + # When running through terminal if __name__ == '__main__': @@ -413,12 +467,18 @@ parser.add_argument("-p", "--process", help="Wait for the results of the processing.", action="store_true") parser.add_argument("-d", "--delete", help="Measurement ID to delete.") + parser.add_argument("--rerun-all", help="Measurement ID to rerun.") + parser.add_argument("--rerun-processing", help="Measurement ID to rerun processing routings.") args = parser.parse_args() # If the arguments are OK, try to login on the site and upload. if args.delete: # If the delete is provided, do nothing else delete_measurement(args.delete) + elif args.rerun_all: + rerun_all(args.rerun_all, args.process) + elif args.rerun_processing: + rerun_processing(args.rerun_processing, args.process) else: if (args.filename == '') or (args.system == 0): parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n')