# HG changeset patch # User Iannis B # Date 1547227720 -7200 # Node ID 609a3f4b3c27f2179a4dacf81ee8f04c9e1a33d4 # Parent ce6eefd1f4b19e35342f658ca9fd6f9c4ec06a4d Donwloading of new products (Hirelpp, cloudmask, etc.) diff -r ce6eefd1f4b1 -r 609a3f4b3c27 CHANGELOG.rst --- a/CHANGELOG.rst Fri Jan 11 18:05:26 2019 +0200 +++ b/CHANGELOG.rst Fri Jan 11 19:28:40 2019 +0200 @@ -1,6 +1,11 @@ Changelog ========= +0.7.0 - 2019-01-11 +------------------ +* Download method for HiRElPP, cloudmask, and other new datasets. +* Since 0.6.2: Restructuring of input arguments, ancillary file upload, code improvements. + 0.6.2 - 2018-01-10 ------------------ * Fixed bug when download optical files. diff -r ce6eefd1f4b1 -r 609a3f4b3c27 scc_access/__init__.py --- a/scc_access/__init__.py Fri Jan 11 18:05:26 2019 +0200 +++ b/scc_access/__init__.py Fri Jan 11 19:28:40 2019 +0200 @@ -1,1 +1,1 @@ -__version__ = "0.6.3" \ No newline at end of file +__version__ = "0.7.0" \ No newline at end of file diff -r ce6eefd1f4b1 -r 609a3f4b3c27 scc_access/scc_access.py --- a/scc_access/scc_access.py Fri Jan 11 18:05:26 2019 +0200 +++ b/scc_access/scc_access.py Fri Jan 11 19:28:40 2019 +0200 @@ -48,12 +48,19 @@ self.list_measurements_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/') self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/') + self.download_hirelpp_pattern = urlparse.urljoin(self.base_url, + 'data_processing/measurements/{0}/download-hirelpp/') + self.download_cloudmask_pattern = urlparse.urljoin(self.base_url, + 'data_processing/measurements/{0}/download-cloudmask/') + self.download_preprocessed_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-preprocessed/') self.download_optical_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-optical/') self.download_graph_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-plots/') + self.download_elic_pattern = urlparse.urljoin(self.base_url, + 'data_processing/measurements/{0}/download-elic/') self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/') self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/') @@ -165,6 +172,18 @@ with open(local_file, 'wb') as f: f.write(zip_file.read(ziped_name)) + def download_hirelpp(self, measurement_id): + """ Download HiRElPP files for the measurement id. """ + # Construct the download url + download_url = self.download_hirelpp_pattern.format(measurement_id) + self.download_files(measurement_id, 'hirelpp', download_url) + + def download_cloudmask(self, measurement_id): + """ Download preprocessed files for the measurement id. """ + # Construct the download url + download_url = self.download_cloudmask_pattern.format(measurement_id) + self.download_files(measurement_id, 'cloudmask', download_url) + def download_preprocessed(self, measurement_id): """ Download preprocessed files for the measurement id. """ # Construct the download url @@ -183,7 +202,11 @@ download_url = self.download_graph_pattern.format(measurement_id) self.download_files(measurement_id, 'scc_plots', download_url) - # TODO: Add download method for other types of files. + def download_elic(self, measurement_id): + """ Download profile graphs for the measurement id. """ + # Construct the download url + download_url = self.download_elic_pattern.format(measurement_id) + self.download_files(measurement_id, 'elic', download_url) def rerun_processing(self, measurement_id, monitor=True): measurement, status = self.get_measurement(measurement_id) @@ -264,19 +287,28 @@ if measurement is not None: while measurement.is_running: - logger.info("Measurement is being processed (status: %s, %s, %s). Please wait.", measurement.upload, measurement.pre_processing, measurement.processing) + logger.info("Measurement is being processed. Please wait.") time.sleep(10) measurement, status = self.get_measurement(measurement_id) - logger.info("Measurement processing finished (status: %s, %s, %s).", measurement.upload, measurement.pre_processing, measurement.processing) - if measurement.pre_processing == 127: + logger.info("Measurement processing finished.") + if measurement.hirelpp == 127: + logger.info("Downloading hirelpp files.") + self.download_hirelpp(measurement_id) + if measurement.cloudmask == 127: + logger.info("Downloading cloudmask files.") + self.download_cloudmask(measurement_id) + if measurement.elpp == 127: logger.info("Downloading preprocessed files.") self.download_preprocessed(measurement_id) - if measurement.processing == 127: + if measurement.elda == 127: logger.info("Downloading optical files.") self.download_optical(measurement_id) logger.info("Downloading graphs.") self.download_graphs(measurement_id) + if measurement.elic == 127: + logger.info("Downloading preprocessed files.") + self.download_elic(measurement_id) logger.info("--- Processing finished. ---") return measurement @@ -450,7 +482,7 @@ @property def rerun_processing_url(self): - url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-optical/') + url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-elda/') return url_pattern.format(self.id) @property