Mercurial > public > scc-netcdf-checker / changeset
changeset
Small changes and formating.
Thu, 13 Oct 2016 13:20:12 +0300
- author
- Ioannis Binietoglou <binietoglou@noa.gr>
- date
- Thu, 13 Oct 2016 13:20:12 +0300
- changeset 4
- 5a18d437cd96
- parent 3
- 5fa09a129470
Small changes and formating.
| .hgignore | file | annotate | diff | comparison | revisions | |
| LICENCE.txt | file | annotate | diff | comparison | revisions | |
| README.rst | file | annotate | diff | comparison | revisions | |
| netcdf_checker.py | file | annotate | diff | comparison | revisions |
--- a/netcdf_checker.py Fri Oct 10 15:04:40 2014 +0200 +++ b/netcdf_checker.py Thu Oct 13 13:20:12 2016 +0300 @@ -1,8 +1,8 @@ +import fnmatch +import netCDF4 as netcdf import os -import fnmatch + import numpy as np -import netCDF4 as netcdf - ERROR_ORDER = {'notification': 1, 'warning': 2, @@ -15,7 +15,6 @@ Output summary: Errors: {1[error]}, Warnings: {1[warning]}, Notifications: {1[notification]}""" - # We firstly define the possible specification elements @@ -26,7 +25,7 @@ def __repr__(self): return "{0}: {1}".format(self.level.title(), self.message) - + class SpecError(SpecGenericError): def __init__(self, message): self.message = message @@ -49,53 +48,56 @@ @property def continue_check(self): return True - + + class DimensionMandatory(GenericSpecification): - def __init__(self, is_mandatory = True): - self.block_next = True # if true the next checks for this dimension will not be performed. + def __init__(self, is_mandatory=True): + self.block_next = True # if true the next checks for this dimension will not be performed. self.is_mandatory = is_mandatory - + def check(self, netcdf_file, dimension_name): the_dimension = netcdf_file.dimensions.get(dimension_name, None) - + error = [] if the_dimension: # If the dimension is found in the file self.dimension_exists = True check_passed = True - + else: self.dimension_exists = False if self.is_mandatory: # If a mandatory dimension is not found in the file check_passed = False - error.append(SpecError('The dimension {0} is obligatory but was not found in the file.'.format(dimension_name))) + error.append( + SpecError('The dimension {0} is obligatory but was not found in the file.'.format(dimension_name))) else: check_passed = True - error.append(SpecNotification('The optional dimension {0} was not found in the file.'.format(dimension_name))) - + error.append( + SpecNotification('The optional dimension {0} was not found in the file.'.format(dimension_name))) + self.check_passed = check_passed self.error = error - + return check_passed, error - + @property def continue_check(self): if (not self.dimension_exists) and (self.block_next): return False else: return True - + class DimensionUnlimited(GenericSpecification): def __init__(self, is_unlimited): self.block_next = False self.is_unlimited = is_unlimited - + def check(self, netcdf_file, dimension_name): the_dimension = netcdf_file.dimensions.get(dimension_name, None) error = [] - + if the_dimension: if the_dimension.isunlimited() == True and self.is_unlimited == False: check_passed = False @@ -107,43 +109,46 @@ check_passed = True else: check_passed = True - error.append(SpecError('Dimension {0} should be unlimited, but was not found in the file.'.format(dimension_name))) - + error.append( + SpecError('Dimension {0} should be unlimited, but was not found in the file.'.format(dimension_name))) + self.check_passed = check_passed self.error = error - - return check_passed, error + + return check_passed, error class VariableMandatory(GenericSpecification): - def __init__(self, is_mandatory = True): - self.block_next = True # if true the next checks for this variable will not be performed. + def __init__(self, is_mandatory=True): + self.block_next = True # if true the next checks for this variable will not be performed. self.is_mandatory = is_mandatory - + def check(self, netcdf_file, variable_name): the_variable = netcdf_file.variables.get(variable_name, None) error = [] - + if the_variable != None: # If the variable is found in the file self.variable_exists = True check_passed = True - + else: self.variable_exists = False if self.is_mandatory: # If a mandatory variable is not found in the file check_passed = False - error.append(SpecError('The variable {0} is obligatory but was not found in the file.'.format(variable_name))) + error.append( + SpecError('The variable {0} is obligatory but was not found in the file.'.format(variable_name))) else: check_passed = True - error.append(SpecNotification('The optional variable {0} was not found in the file.'.format(variable_name))) - + error.append( + SpecNotification('The optional variable {0} was not found in the file.'.format(variable_name))) + self.check_passed = check_passed self.error = error - + return check_passed, error - + @property def continue_check(self): if (not self.variable_exists) and (self.block_next): @@ -151,10 +156,11 @@ else: return True + class VariableDimensions(GenericSpecification): def __init__(self, dimensions): self.dimensions = dimensions - + def check(self, netcdf_file, variable_name): the_variable = netcdf_file.variables.get(variable_name, None) @@ -165,67 +171,76 @@ for dimension in self.dimensions: if not (dimension in variable_dimensions): check_passed = False - error.append(SpecError("Variable {0} does not have dimension {1}.".format(variable_name, dimension))) - + error.append( + SpecError("Variable {0} does not have dimension {1}.".format(variable_name, dimension))) + # If all dimensions are present, check if the variables are in the # correct order. if check_passed: if list(self.dimensions) != variable_dimensions: check_passed = False - error.append(SpecError("Variable {0} has wrong dimension order: {1} instead of {2}.".format(variable_name, - variable_dimensions, - list(self.dimensions)))) + error.append( + SpecError("Variable {0} has wrong dimension order: {1} instead of {2}.".format(variable_name, + variable_dimensions, + list( + self.dimensions)))) for dimension in variable_dimensions: if dimension not in self.dimensions: - error.append(SpecWarning('Dimension {0} found in variable {1} but is not defined in the specifications'.format(dimension, variable_name))) - + error.append(SpecWarning( + 'Dimension {0} found in variable {1} but is not defined in the specifications'.format(dimension, + variable_name))) + else: check_passed = False - error = [SpecError('Variable {0} should be checked for dimensions, but was not found in the file.'.format(variable_name)),] - + error = [SpecError('Variable {0} should be checked for dimensions, but was not found in the file.'.format( + variable_name)), ] + self.check_passed = check_passed self.error = error - + return check_passed, error class VariableType(GenericSpecification): def __init__(self, dtype): self.dtype = dtype - + def check(self, netcdf_file, variable_name): the_variable = netcdf_file.variables.get(variable_name, None) error = [] - + if the_variable != None: # Get the internal python type and not the numpy.dtype. # The conversions guarantee (?) that a single element is always returned variable_type_python = type(np.asscalar(np.asarray(np.asarray(the_variable[:]).item(0)))) - + if not (variable_type_python == self.dtype): check_passed = False - error.append(SpecError('Variable {0} is of type {1} while it should be {2}'.format(variable_name, the_variable.dtype, self.dtype))) + error.append(SpecError( + 'Variable {0} is of type {1} while it should be {2}'.format(variable_name, the_variable.dtype, + self.dtype))) else: check_passed = True else: check_passed = False - error.append(SpecError('Variable {0} should be checked for type, but was not found in the file.'.format(variable_name))) - + error.append(SpecError( + 'Variable {0} should be checked for type, but was not found in the file.'.format(variable_name))) + self.check_passed = check_passed self.error = error - + return check_passed, error class AttributeMandatory(GenericSpecification): - def __init__(self, is_mandatory = True): - self.block_next = True # if true the next checks for this variable will not be performed. + def __init__(self, is_mandatory=True): + self.block_next = True # if true the next checks for this variable will not be performed. self.is_mandatory = is_mandatory - + def check(self, netcdf_file, attribute_name): the_attribute = getattr(netcdf_file, attribute_name, None) error = [] - + if the_attribute: # If the variable is found in the file self.attribute_exists = True @@ -236,16 +251,18 @@ if self.is_mandatory: # If a mandatory variable is not found in the file check_passed = False - error.append(SpecError('The attribute {0} is obligatory but was not found in the file.'.format(attribute_name))) + error.append( + SpecError('The attribute {0} is obligatory but was not found in the file.'.format(attribute_name))) else: check_passed = True - error.append(SpecNotification('The optional attribute {0} was not found in the file.'.format(attribute_name))) - + error.append( + SpecNotification('The optional attribute {0} was not found in the file.'.format(attribute_name))) + self.check_passed = check_passed self.error = error - + return check_passed, error - + @property def continue_check(self): if (not self.attribute_exists) and (self.block_next): @@ -253,16 +270,16 @@ else: return True - + class AttributeType(GenericSpecification): - def __init__(self, dtype, block_next = True): + def __init__(self, dtype, block_next=True): self.block_next = block_next self.dtype = dtype - + def check(self, netcdf_file, attribute_name): the_attribute = getattr(netcdf_file, attribute_name, None) error = [] - + if the_attribute: # Get the internal python type and not the numpy.dtype. # The conversions guarantee (?) that a single element is always returned @@ -270,20 +287,24 @@ attribute_type_python = type(np.asscalar(np.asarray(np.asarray(the_attribute[:]).item(0)))) except: attribute_type_python = type(np.asscalar(the_attribute)) - + if not (attribute_type_python == self.dtype): check_passed = False - error.append(SpecError('Attribute {0} is of type {1} while it should be {2}'.format(attribute_name, type(the_attribute).__name__, self.dtype.__name__))) + error.append(SpecError('Attribute {0} is of type {1} while it should be {2}'.format(attribute_name, + type( + the_attribute).__name__, + self.dtype.__name__))) else: error = None check_passed = True else: check_passed = False - error.append(SpecError('Attribute {0} should be checked for type, but was not found in the file.'.format(attribute_name))) - + error.append(SpecError( + 'Attribute {0} should be checked for type, but was not found in the file.'.format(attribute_name))) + self.check_passed = check_passed self.error = error - + return check_passed, error @property @@ -295,48 +316,49 @@ class AttributeStrLength(GenericSpecification): - def __init__(self, length): self.length = length - - + def check(self, netcdf_file, attribute_name): the_attribute = getattr(netcdf_file, attribute_name, None) error = [] - + if the_attribute: if len(the_attribute) != self.length: check_passed = False - error.append(SpecError('Attribute {0} should be of length {1} while it has length {2}'.format(attribute_name, self.length, len(the_attribute)))) + error.append(SpecError( + 'Attribute {0} should be of length {1} while it has length {2}'.format(attribute_name, self.length, + len(the_attribute)))) else: check_passed = True else: check_passed = False - error.append(SpecError('Attribute {0} should be checked for length, but was not found in the file.'.format(attribute_name))) - + error.append(SpecError( + 'Attribute {0} should be checked for length, but was not found in the file.'.format(attribute_name))) + self.check_passed = check_passed self.error = error - - return check_passed, error + + return check_passed, error class FilenameShellPattern(GenericSpecification): def __init__(self, shell_pattern): self.pattern = shell_pattern - + def check(self, netcdf_file, filename): error = [] - + if fnmatch.fnmatch(filename, self.pattern): check_passed = True else: check_passed = False error.append(SpecError('Filename {0} does not match patter {1}'.format(filename, self.pattern))) - + self.check_passed = check_passed self.error = error - - return check_passed, error + + return check_passed, error # This is the main class of the script. @@ -349,8 +371,7 @@ file_checker.print_report('error') """ - - + def __init__(self, filepath, specs): self.file = None self.checks_run = False @@ -359,26 +380,26 @@ self.specs = specs self.check_results = {} self.check_results['general'] = [] - + def __enter__(self): self.open_file() return self - + def __exit__(self, type, value, traceback): if self.file: self.file.close() - + def open_file(self): try: self.file = netcdf.Dataset(self.filepath) except: self.check_results['general'].append(SpecError('Could not open file {0}.'.format(self.filename))) - + def close_file(self): self.file.close() - + def run_checks(self): - if self.file: + if self.file: self.check_file() self.check_attributes() self.check_dimensions() @@ -387,109 +408,111 @@ def check_file(self): self.check_results['file'] = [] - + try: specs_file = self.specs['file'] except: specs_file = [] - + for file_spec in specs_file: - check_passed, error = file_spec.check(self.file, self.filename) - + check_passed, error = file_spec.check(self.file, self.filename) + if error: self.check_results['file'].extend(list(error)) - + if not file_spec.continue_check: - break - + break + def check_attributes(self): """ Check if attributes are according to specs """ - + self.check_results['attributes'] = [] - + try: spec_attributes = self.specs['attributes'].keys() except: spec_attributes = [] - + for attribute_name in spec_attributes: attribute_specs = self.specs['attributes'][attribute_name] for attribute_spec in attribute_specs: check_passed, error = attribute_spec.check(self.file, attribute_name) if error: - self.check_results['attributes'].extend(list(error)) + self.check_results['attributes'].extend(list(error)) if not attribute_spec.continue_check: - break # Don't continue checking specifications if a blocking check failed. - + break # Don't continue checking specifications if a blocking check failed. + for attribute_name in self.file.ncattrs(): if attribute_name not in spec_attributes: - self.check_results['attributes'].append(SpecWarning('Attribute {0} found in the file but is not defined in the specifications'.format(attribute_name))) - + self.check_results['attributes'].append(SpecWarning( + 'Attribute {0} found in the file but is not defined in the specifications'.format(attribute_name))) + def check_dimensions(self): """ Check if dimension are according to specs """ self.check_results['dimensions'] = [] - + try: spec_dimensions = self.specs['dimensions'].keys() except: spec_dimensions = [] - - + for dimension_name in spec_dimensions: dimension_specs = self.specs['dimensions'][dimension_name] for dimension_spec in dimension_specs: check_passed, error = dimension_spec.check(self.file, dimension_name) - + if error: - self.check_results['dimensions'].extend(list(error)) - + self.check_results['dimensions'].extend(list(error)) + if not dimension_spec.continue_check: - break # Don't continue checking specifications if a blocking check failed. - + break # Don't continue checking specifications if a blocking check failed. + for dimension in self.file.dimensions: if dimension not in spec_dimensions: - self.check_results['dimensions'].append(SpecWarning('Dimension {0} found in the file but is not defined in the specifications'.format(dimension))) - + self.check_results['dimensions'].append(SpecWarning( + 'Dimension {0} found in the file but is not defined in the specifications'.format(dimension))) + def check_variables(self): """ Check if variables are according to specs """ - + self.check_results['variables'] = [] - + try: spec_variables = self.specs['variables'].keys() except: spec_variables = [] - + for variable_name in spec_variables: variable_specs = self.specs['variables'][variable_name] for variable_spec in variable_specs: check_passed, error = variable_spec.check(self.file, variable_name) if error: - self.check_results['variables'].extend(list(error)) + self.check_results['variables'].extend(list(error)) if not variable_spec.continue_check: - break # Don't continue checking specifications if a blocking check failed. - + break # Don't continue checking specifications if a blocking check failed. + for variable_name in self.file.variables: if variable_name not in spec_variables: - self.check_results['variables'].append(SpecWarning('Variable {0} found in the file but is not defined in the specifications'.format(variable_name))) + self.check_results['variables'].append(SpecWarning( + 'Variable {0} found in the file but is not defined in the specifications'.format(variable_name))) - def file_ok(self, level = 'error'): + def file_ok(self, level='error'): """ Check if the file checked is ok. What ok means is defined by the level variable """ - - status = None + + status = None if self.checks_run: status = True for category, result_list in self.check_results.items(): for result in result_list: if ERROR_ORDER[result.level] >= ERROR_ORDER[level]: status = False - + return status - + def results_for_level(self, level): """ Returns all the results of a specific level """ results = None @@ -499,293 +522,295 @@ for result in result_list: if ERROR_ORDER[result.level] == ERROR_ORDER[level]: results.append(result) - + return results - + def results_by_level(self): """ Returns a dictionary with the results by level. """ - + results = {} for level, order in ERROR_ORDER.items(): results[level] = self.results_for_level(level) - + return results - + def result_count(self): """ Returns a dictionary with the number of results per category. """ - + result_number = {} results = self.results_by_level() - + for category, error_list in results.items(): if error_list is None: result_number[category] = 0 else: result_number[category] = len(error_list) return result_number - + def print_report(self, level): """ Print a report for the given level. """ - + print header_template.format(self, self.result_count()) - + results = self.results_by_level() - + for result_level in ['error', 'warning', 'notification']: if ERROR_ORDER[result_level] >= ERROR_ORDER[level]: print "\n{0} details".format(result_level.capitalize()) print "----------------" for result in results[result_level]: - print result - + print result + + + # Sounding file specifications + -# Sounding file specifications -sounding_specs = {'file': [FilenameShellPattern('rs_*.nc'),], - 'dimensions': {'points': [DimensionMandatory(True), - DimensionUnlimited(False),], - }, - 'variables': {'Altitude': [VariableMandatory(True), - VariableDimensions(['points',]), - VariableType(float)], - 'Temperature': [VariableMandatory(True), - VariableDimensions(['points',]), - VariableType(float)], - 'Pressure': [VariableMandatory(True), - VariableDimensions(['points',]), - VariableType(float)], - 'RelativeHumidity': [VariableMandatory(False), - VariableDimensions(['points',]), - VariableType(float)], - }, - 'attributes': {'Latitude_degrees_north': [AttributeMandatory(True), - AttributeType(float),], - 'Longitude_degrees_east': [AttributeMandatory(True), - AttributeType(float),], - 'Altitude_meter_asl': [AttributeMandatory(True), - AttributeType(float),], - 'Location': [AttributeMandatory(False), - AttributeType(unicode),], - 'Sounding_Station_Name': [AttributeMandatory(False), - AttributeType(unicode),], - 'WMO_Station_Number': [AttributeMandatory(False), - AttributeType(unicode),], - 'WBAN_Station_Number':[AttributeMandatory(False), - AttributeType(unicode),], - 'Sounding_Start_Date':[AttributeMandatory(True), - AttributeType(unicode, block_next = True), - AttributeStrLength(8)], - 'Sounding_Start_Time_UT':[AttributeMandatory(True), - AttributeType(unicode, block_next = True), - AttributeStrLength(6)], - 'Sounding_Stop_Time_UT':[AttributeMandatory(False), - AttributeType(unicode, block_next = True), - AttributeStrLength(6)], - }, - 'name': "SCC Sounding file" - } +sounding_specs = {'file': [FilenameShellPattern('rs_*.nc'), ], + 'dimensions': {'points': [DimensionMandatory(True), + DimensionUnlimited(False), ], + }, + 'variables': {'Altitude': [VariableMandatory(True), + VariableDimensions(['points', ]), + VariableType(float)], + 'Temperature': [VariableMandatory(True), + VariableDimensions(['points', ]), + VariableType(float)], + 'Pressure': [VariableMandatory(True), + VariableDimensions(['points', ]), + VariableType(float)], + 'RelativeHumidity': [VariableMandatory(False), + VariableDimensions(['points', ]), + VariableType(float)], + }, + 'attributes': {'Latitude_degrees_north': [AttributeMandatory(True), + AttributeType(float), ], + 'Longitude_degrees_east': [AttributeMandatory(True), + AttributeType(float), ], + 'Altitude_meter_asl': [AttributeMandatory(True), + AttributeType(float), ], + 'Location': [AttributeMandatory(False), + AttributeType(unicode), ], + 'Sounding_Station_Name': [AttributeMandatory(False), + AttributeType(unicode), ], + 'WMO_Station_Number': [AttributeMandatory(False), + AttributeType(unicode), ], + 'WBAN_Station_Number': [AttributeMandatory(False), + AttributeType(unicode), ], + 'Sounding_Start_Date': [AttributeMandatory(True), + AttributeType(unicode, block_next=True), + AttributeStrLength(8)], + 'Sounding_Start_Time_UT': [AttributeMandatory(True), + AttributeType(unicode, block_next=True), + AttributeStrLength(6)], + 'Sounding_Stop_Time_UT': [AttributeMandatory(False), + AttributeType(unicode, block_next=True), + AttributeStrLength(6)], + }, + 'name': "SCC Sounding file" + } # Lidar ratio file specifications -lidar_ratio_specs = {'file': [FilenameShellPattern('*.nc'),], - 'dimensions': {'points': [DimensionMandatory(True), - DimensionUnlimited(False),], - 'products': [DimensionMandatory(True), - DimensionUnlimited(False),], - }, - 'variables': {'Altitude': [VariableMandatory(True), - VariableDimensions(['points',]), +lidar_ratio_specs = {'file': [FilenameShellPattern('*.nc'), ], + 'dimensions': {'points': [DimensionMandatory(True), + DimensionUnlimited(False), ], + 'products': [DimensionMandatory(True), + DimensionUnlimited(False), ], + }, + 'variables': {'Altitude': [VariableMandatory(True), + VariableDimensions(['points', ]), + VariableType(float)], + 'Lidar_Ratio': [VariableMandatory(True), + VariableDimensions(['products', 'points']), VariableType(float)], - 'Lidar_Ratio': [VariableMandatory(True), - VariableDimensions(['products', 'points']), - VariableType(float)], - 'product_ID': [VariableMandatory(True), - VariableDimensions(['products',]), - VariableType(int)], - }, - 'attributes': {'Lidar_Station_Name': [AttributeMandatory(True), - AttributeType(unicode),], - }, - 'name': "SCC Lidar ratio file" - } + 'product_ID': [VariableMandatory(True), + VariableDimensions(['products', ]), + VariableType(int)], + }, + 'attributes': {'Lidar_Station_Name': [AttributeMandatory(True), + AttributeType(unicode), ], + }, + 'name': "SCC Lidar ratio file" + } # Overlap file specifications -overlap_specs = {'file': [FilenameShellPattern('ov_*.nc'),], - 'dimensions': {'points': [DimensionMandatory(True), - DimensionUnlimited(False),], - 'channels': [DimensionMandatory(True), - DimensionUnlimited(False),], - }, - 'variables': {'Altitude': [VariableMandatory(True), - VariableDimensions(['points',]), +overlap_specs = {'file': [FilenameShellPattern('ov_*.nc'), ], + 'dimensions': {'points': [DimensionMandatory(True), + DimensionUnlimited(False), ], + 'channels': [DimensionMandatory(True), + DimensionUnlimited(False), ], + }, + 'variables': {'Altitude': [VariableMandatory(True), + VariableDimensions(['points', ]), VariableType(float)], - 'Overlap_Function': [VariableMandatory(True), - VariableDimensions(['channels', 'points']), + 'Overlap_Function': [VariableMandatory(True), + VariableDimensions(['channels', 'points']), VariableType(float)], - 'channel_ID': [VariableMandatory(True), - VariableDimensions(['channels',]), + 'channel_ID': [VariableMandatory(True), + VariableDimensions(['channels', ]), VariableType(int)], - }, + }, 'attributes': {'Lidar_Station_Name': [AttributeMandatory(True), - AttributeType(unicode, block_next = True), + AttributeType(unicode, block_next=True), AttributeStrLength(2)], 'Overlap_Measurement_Date': [AttributeMandatory(True), - AttributeType(unicode, block_next = True), + AttributeType(unicode, block_next=True), AttributeStrLength(8)], - }, + }, 'name': "SCC Overlap file" - } + } # Raw data file specifications -data_specs = {'file': [FilenameShellPattern('*.nc'),], - 'dimensions': {'points': [DimensionMandatory(True), - DimensionUnlimited(False),], - 'channels': [DimensionMandatory(True), - DimensionUnlimited(False),], - 'nb_of_time_scales': [DimensionMandatory(True), - DimensionUnlimited(False),], - 'time': [DimensionMandatory(True), - DimensionUnlimited(True),], - 'time_bck': [DimensionMandatory(False), - DimensionUnlimited(False),], - 'scan_angles': [DimensionMandatory(True), - DimensionUnlimited(False),], - }, - 'variables': {'channel_ID': [VariableMandatory(True), - VariableDimensions(['channels',]), - VariableType(int)], - 'Laser_Repetition_Rate': [VariableMandatory(False), - VariableDimensions(['channels',]), - VariableType(int)], - 'Laser_Pointing_Angle': [VariableMandatory(True), - VariableDimensions(['scan_angles',]), - VariableType(float)], - 'ID_Range': [VariableMandatory(False), - VariableDimensions(['channels',]), - VariableType(int)], - 'Scattering_Mechanism': [VariableMandatory(False), - VariableDimensions(['channels',]), - VariableType(int)], - 'Emitted_Wavelength': [VariableMandatory(False), - VariableDimensions(['channels',]), - VariableType(float)], - 'Detected_Wavelength': [VariableMandatory(False), - VariableDimensions(['channels',]), - VariableType(float)], - 'Raw_Data_Range_Resolution': [VariableMandatory(False), - VariableDimensions(['channels',]), - VariableType(float)], - 'Background_Mode': [VariableMandatory(False), - VariableDimensions(['channels',]), - VariableType(int)], - 'Background_Low': [VariableMandatory(True), - VariableDimensions(['channels',]), - VariableType(float)], - 'Background_High': [VariableMandatory(True), - VariableDimensions(['channels',]), - VariableType(float)], - 'Molecular_Calc': [VariableMandatory(True), - VariableDimensions([]), - VariableType(int)], - 'id_timescale': [VariableMandatory(True), - VariableDimensions(['channels',]), - VariableType(int)], - 'Dead_Time_Corr_Type': [VariableMandatory(False), - VariableDimensions(['channels',]), - VariableType(int)], - 'Dead_Time': [VariableMandatory(False), - VariableDimensions(['channels',]), - VariableType(float)], - 'Acquisition_Mode': [VariableMandatory(False), - VariableDimensions(['channels',]), - VariableType(int)], - 'Trigger_Delay': [VariableMandatory(False), - VariableDimensions(['channels',]), - VariableType(float)], - 'Laser_Pointing_Angle_of_Profiles': [VariableMandatory(True), - VariableDimensions(['time','nb_of_time_scales',]), - VariableType(int)], - 'Raw_Data_Start_Time': [VariableMandatory(True), - VariableDimensions(['time','nb_of_time_scales',]), - VariableType(int)], - 'Raw_Data_Stop_Time': [VariableMandatory(True), - VariableDimensions(['time','nb_of_time_scales',]), +data_specs = {'file': [FilenameShellPattern('*.nc'), ], + 'dimensions': {'points': [DimensionMandatory(True), + DimensionUnlimited(False), ], + 'channels': [DimensionMandatory(True), + DimensionUnlimited(False), ], + 'nb_of_time_scales': [DimensionMandatory(True), + DimensionUnlimited(False), ], + 'time': [DimensionMandatory(True), + DimensionUnlimited(True), ], + 'time_bck': [DimensionMandatory(False), + DimensionUnlimited(False), ], + 'scan_angles': [DimensionMandatory(True), + DimensionUnlimited(False), ], + }, + 'variables': {'channel_ID': [VariableMandatory(True), + VariableDimensions(['channels', ]), + VariableType(int)], + 'Laser_Repetition_Rate': [VariableMandatory(False), + VariableDimensions(['channels', ]), VariableType(int)], - 'Laser_Shots': [VariableMandatory(True), - VariableDimensions(['time','channels',]), + 'Laser_Pointing_Angle': [VariableMandatory(True), + VariableDimensions(['scan_angles', ]), + VariableType(float)], + 'ID_Range': [VariableMandatory(False), + VariableDimensions(['channels', ]), + VariableType(int)], + 'Scattering_Mechanism': [VariableMandatory(False), + VariableDimensions(['channels', ]), + VariableType(int)], + 'Emitted_Wavelength': [VariableMandatory(False), + VariableDimensions(['channels', ]), + VariableType(float)], + 'Detected_Wavelength': [VariableMandatory(False), + VariableDimensions(['channels', ]), + VariableType(float)], + 'Raw_Data_Range_Resolution': [VariableMandatory(False), + VariableDimensions(['channels', ]), + VariableType(float)], + 'Background_Mode': [VariableMandatory(False), + VariableDimensions(['channels', ]), + VariableType(int)], + 'Background_Low': [VariableMandatory(True), + VariableDimensions(['channels', ]), + VariableType(float)], + 'Background_High': [VariableMandatory(True), + VariableDimensions(['channels', ]), + VariableType(float)], + 'Molecular_Calc': [VariableMandatory(True), + VariableDimensions([]), VariableType(int)], - 'Raw_Lidar_Data': [VariableMandatory(False), - VariableDimensions(['time', 'channels', 'points']), - VariableType(float)], - 'Depolarization_Factor': [VariableMandatory(False), - VariableDimensions(['channels',]), - VariableType(float)], - 'LR_Input': [VariableMandatory(False), - VariableDimensions(['channels',]), + 'id_timescale': [VariableMandatory(True), + VariableDimensions(['channels', ]), + VariableType(int)], + 'Dead_Time_Corr_Type': [VariableMandatory(False), + VariableDimensions(['channels', ]), + VariableType(int)], + 'Dead_Time': [VariableMandatory(False), + VariableDimensions(['channels', ]), + VariableType(float)], + 'Acquisition_Mode': [VariableMandatory(False), + VariableDimensions(['channels', ]), + VariableType(int)], + 'Trigger_Delay': [VariableMandatory(False), + VariableDimensions(['channels', ]), + VariableType(float)], + 'Laser_Pointing_Angle_of_Profiles': [VariableMandatory(True), + VariableDimensions(['time', 'nb_of_time_scales', ]), + VariableType(int)], + 'Raw_Data_Start_Time': [VariableMandatory(True), + VariableDimensions(['time', 'nb_of_time_scales', ]), + VariableType(int)], + 'Raw_Data_Stop_Time': [VariableMandatory(True), + VariableDimensions(['time', 'nb_of_time_scales', ]), + VariableType(int)], + 'Laser_Shots': [VariableMandatory(True), + VariableDimensions(['time', 'channels', ]), VariableType(int)], - 'DAQ_Range': [VariableMandatory(False), - VariableDimensions(['channels',]), - VariableType(float)], - 'Pressure_at_Lidar_Station': [VariableMandatory(False), - VariableDimensions([]), + 'Raw_Lidar_Data': [VariableMandatory(False), + VariableDimensions(['time', 'channels', 'points']), + VariableType(float)], + 'Depolarization_Factor': [VariableMandatory(False), + VariableDimensions(['channels', ]), + VariableType(float)], + 'LR_Input': [VariableMandatory(False), + VariableDimensions(['channels', ]), + VariableType(int)], + 'DAQ_Range': [VariableMandatory(False), + VariableDimensions(['channels', ]), + VariableType(float)], + 'Pressure_at_Lidar_Station': [VariableMandatory(False), + VariableDimensions([]), + VariableType(float)], + 'Temperature_at_Lidar_Station': [VariableMandatory(False), + VariableDimensions([]), VariableType(float)], - 'Temperature_at_Lidar_Station': [VariableMandatory(False), - VariableDimensions([]), - VariableType(float)], - 'Background_Profile': [VariableMandatory(False), - VariableDimensions(['time_bck', 'channels', 'points']), - VariableType(float)], - 'Raw_Bck_Start_Time': [VariableMandatory(False), - VariableDimensions(['time_bck','nb_of_time_scales',]), + 'Background_Profile': [VariableMandatory(False), + VariableDimensions(['time_bck', 'channels', 'points']), + VariableType(float)], + 'Raw_Bck_Start_Time': [VariableMandatory(False), + VariableDimensions(['time_bck', 'nb_of_time_scales', ]), + VariableType(int)], + 'Raw_Bck_Stop_Time': [VariableMandatory(False), + VariableDimensions(['time_bck', 'nb_of_time_scales', ]), + VariableType(int)], + 'Error_On_Raw_Lidar_Data': [VariableMandatory(False), + VariableDimensions(['time', 'channels', 'points']), + VariableType(float)], + 'First_Signal_Rangebin': [VariableMandatory(False), + VariableDimensions(['channels', ]), VariableType(int)], - 'Raw_Bck_Stop_Time': [VariableMandatory(False), - VariableDimensions(['time_bck','nb_of_time_scales',]), - VariableType(int)], - 'Error_On_Raw_Lidar_Data': [VariableMandatory(False), - VariableDimensions(['time','channels', 'points']), - VariableType(float)], - 'First_Signal_Rangebin': [VariableMandatory(False), - VariableDimensions(['channels',]), - VariableType(int)], + }, + 'attributes': {'Measurement_ID': [AttributeMandatory(True), + AttributeType(unicode, block_next=True), + AttributeStrLength(12)], + 'RawData_Start_Date': [AttributeMandatory(True), + AttributeType(unicode, block_next=True), + AttributeStrLength(8)], + 'RawData_Start_Time_UT': [AttributeMandatory(True), + AttributeType(unicode, block_next=True), + AttributeStrLength(6)], + 'RawData_Stop_Time_UT': [AttributeMandatory(True), + AttributeType(unicode, block_next=True), + AttributeStrLength(6)], + 'RawBck_Start_Date': [AttributeMandatory(False), + AttributeType(unicode, block_next=True), + AttributeStrLength(8)], + 'RawBck_Start_Time_UT': [AttributeMandatory(False), + AttributeType(unicode, block_next=True), + AttributeStrLength(6)], + 'RawBck_Stop_Time_UT': [AttributeMandatory(False), + AttributeType(unicode, block_next=True), + AttributeStrLength(6)], + 'Sounding_File_Name': [AttributeMandatory(False), + AttributeType(unicode), ], + 'LR_File_Name': [AttributeMandatory(False), + AttributeType(unicode), ], + 'Overlap_File_Name': [AttributeMandatory(False), + AttributeType(unicode), ], + 'Location': [AttributeMandatory(False), + AttributeType(unicode), ], + 'System': [AttributeMandatory(False), + AttributeType(unicode), ], + 'Latitude_degrees_north': [AttributeMandatory(False), + AttributeType(float), ], + 'Longitude_degrees_east': [AttributeMandatory(False), + AttributeType(float), ], + 'Altitude_meter_asl': [AttributeMandatory(False), + AttributeType(float), ], }, - 'attributes': {'Measurement_ID': [AttributeMandatory(True), - AttributeType(unicode, block_next = True), - AttributeStrLength(12)], - 'RawData_Start_Date': [AttributeMandatory(True), - AttributeType(unicode, block_next = True), - AttributeStrLength(8)], - 'RawData_Start_Time_UT': [AttributeMandatory(True), - AttributeType(unicode, block_next = True), - AttributeStrLength(6)], - 'RawData_Stop_Time_UT': [AttributeMandatory(True), - AttributeType(unicode, block_next = True), - AttributeStrLength(6)], - 'RawBck_Start_Date': [AttributeMandatory(False), - AttributeType(unicode, block_next = True), - AttributeStrLength(8)], - 'RawBck_Start_Time_UT': [AttributeMandatory(False), - AttributeType(unicode, block_next = True), - AttributeStrLength(6)], - 'RawBck_Stop_Time_UT': [AttributeMandatory(False), - AttributeType(unicode, block_next = True), - AttributeStrLength(6)], - 'Sounding_File_Name': [AttributeMandatory(False), - AttributeType(unicode),], - 'LR_File_Name': [AttributeMandatory(False), - AttributeType(unicode),], - 'Overlap_File_Name': [AttributeMandatory(False), - AttributeType(unicode),], - 'Location': [AttributeMandatory(False), - AttributeType(unicode),], - 'System': [AttributeMandatory(False), - AttributeType(unicode),], - 'Latitude_degrees_north': [AttributeMandatory(False), - AttributeType(float),], - 'Longitude_degrees_east': [AttributeMandatory(False), - AttributeType(float),], - 'Altitude_meter_asl': [AttributeMandatory(False), - AttributeType(float),], - }, - 'name': "SCC Raw input file" - } + 'name': "SCC Raw input file" + } # Used for the command line arguments spec_shorthands = {'sounding': sounding_specs, @@ -793,27 +818,24 @@ 'overlap': overlap_specs, 'data': data_specs,} - if __name__ == "__main__": - # For use from a terminal import argparse - + parser = argparse.ArgumentParser() - parser.add_argument("file", help = "The path of the file to be checked") - parser.add_argument("-s", "--specs", default = 'data', - help = "The specificiations to use", - choices = ['data', 'overlap', 'lidar_ratio', 'sounding']) - parser.add_argument("-l", "--level", default = 'warning', - help = "The output level", - choices = ['error', 'warning', 'notification']) - + parser.add_argument("file", help="The path of the file to be checked") + parser.add_argument("-s", "--specs", default='data', + help="The specificiations to use", + choices=['data', 'overlap', 'lidar_ratio', 'sounding']) + parser.add_argument("-l", "--level", default='warning', + help="The output level", + choices=['error', 'warning', 'notification']) + # Check the arguments args = parser.parse_args() - + specs = spec_shorthands[args.specs] - - with FileChecker(args.file, specs) as file_checker: + + with FileChecker(args.file, specs) as file_checker: file_checker.run_checks() file_checker.print_report(args.level) -
