binietoglou@4: import fnmatch
binietoglou@4: import netCDF4 as netcdf
ulalume3@0: import os
binietoglou@4: 
ulalume3@0: import numpy as np
ulalume3@0: 
ulalume3@0: ERROR_ORDER = {'notification': 1,
ulalume3@0:                'warning': 2,
ulalume3@0:                'error': 3,
ulalume3@0:                }
ulalume3@0: 
ulalume3@0: # Used when printing the report
ulalume3@0: header_template = """\n----- Report for file {0.filename}  ----- 
ulalume3@0: Checked against specification: {0.specs[name]}.
ulalume3@0: 
ulalume3@0: Output summary: 
ulalume3@0:     Errors: {1[error]}, Warnings: {1[warning]}, Notifications: {1[notification]}"""
ulalume3@0: 
ulalume3@0: 
ulalume3@0: # We firstly define the possible specification elements
ulalume3@0: # In the end of this file we use them to define complete specifications
ulalume3@0: 
ulalume3@0: 
ulalume3@0: class SpecGenericError:
ulalume3@0:     def __repr__(self):
ulalume3@0:         return "{0}: {1}".format(self.level.title(), self.message)
ulalume3@0: 
binietoglou@4: 
ulalume3@0: class SpecError(SpecGenericError):
ulalume3@0:     def __init__(self, message):
ulalume3@0:         self.message = message
ulalume3@0:         self.level = 'error'
ulalume3@0: 
ulalume3@0: 
ulalume3@0: class SpecWarning(SpecGenericError):
ulalume3@0:     def __init__(self, message):
ulalume3@0:         self.message = message
ulalume3@0:         self.level = 'warning'
ulalume3@0: 
ulalume3@0: 
ulalume3@0: class SpecNotification(SpecGenericError):
ulalume3@0:     def __init__(self, message):
ulalume3@0:         self.message = message
ulalume3@0:         self.level = 'notification'
ulalume3@0: 
ulalume3@0: 
ulalume3@0: class GenericSpecification:
ulalume3@0:     @property
ulalume3@0:     def continue_check(self):
ulalume3@0:         return True
binietoglou@4: 
binietoglou@4: 
ulalume3@0: class DimensionMandatory(GenericSpecification):
binietoglou@4:     def __init__(self, is_mandatory=True):
binietoglou@4:         self.block_next = True  # if true the next checks for this dimension will not be performed.
ulalume3@0:         self.is_mandatory = is_mandatory
binietoglou@4: 
ulalume3@0:     def check(self, netcdf_file, dimension_name):
ulalume3@0:         the_dimension = netcdf_file.dimensions.get(dimension_name, None)
binietoglou@4: 
ulalume3@0:         error = []
ulalume3@0:         if the_dimension:
ulalume3@0:             # If the dimension is found in the file 
ulalume3@0:             self.dimension_exists = True
ulalume3@0:             check_passed = True
binietoglou@4: 
ulalume3@0:         else:
ulalume3@0:             self.dimension_exists = False
ulalume3@0:             if self.is_mandatory:
ulalume3@0:                 # If a mandatory dimension is not found in the file
ulalume3@0:                 check_passed = False
binietoglou@4:                 error.append(
binietoglou@4:                     SpecError('The dimension {0} is obligatory but was not found in the file.'.format(dimension_name)))
ulalume3@0:             else:
ulalume3@0:                 check_passed = True
binietoglou@4:                 error.append(
binietoglou@4:                     SpecNotification('The optional dimension {0} was not found in the file.'.format(dimension_name)))
binietoglou@4: 
ulalume3@0:         self.check_passed = check_passed
ulalume3@0:         self.error = error
binietoglou@4: 
ulalume3@0:         return check_passed, error
binietoglou@4: 
ulalume3@0:     @property
ulalume3@0:     def continue_check(self):
ulalume3@0:         if (not self.dimension_exists) and (self.block_next):
ulalume3@0:             return False
ulalume3@0:         else:
ulalume3@0:             return True
binietoglou@4: 
ulalume3@0: 
ulalume3@0: class DimensionUnlimited(GenericSpecification):
ulalume3@0:     def __init__(self, is_unlimited):
ulalume3@0:         self.block_next = False
ulalume3@0:         self.is_unlimited = is_unlimited
binietoglou@4: 
ulalume3@0:     def check(self, netcdf_file, dimension_name):
ulalume3@0:         the_dimension = netcdf_file.dimensions.get(dimension_name, None)
ulalume3@0:         error = []
binietoglou@4: 
ulalume3@0:         if the_dimension:
ulalume3@0:             if the_dimension.isunlimited() == True and self.is_unlimited == False:
ulalume3@0:                 check_passed = False
ulalume3@0:                 error.append(SpecWarning('Dimension {0} should not be unlimited but is.'.format(dimension_name)))
ulalume3@0:             elif the_dimension.isunlimited() == False and self.is_unlimited == True:
ulalume3@0:                 check_passed = False
ulalume3@0:                 error.append(SpecWarning('Dimension {0} should be unlimited but it is not.'.format(dimension_name)))
ulalume3@0:             else:
ulalume3@0:                 check_passed = True
ulalume3@0:         else:
ulalume3@0:             check_passed = True
binietoglou@4:             error.append(
binietoglou@4:                 SpecError('Dimension {0} should be unlimited, but was not found in the file.'.format(dimension_name)))
binietoglou@4: 
ulalume3@0:         self.check_passed = check_passed
ulalume3@0:         self.error = error
binietoglou@4: 
binietoglou@4:         return check_passed, error
ulalume3@0: 
ulalume3@0: 
ulalume3@0: class VariableMandatory(GenericSpecification):
binietoglou@4:     def __init__(self, is_mandatory=True):
binietoglou@4:         self.block_next = True  # if true the next checks for this variable will not be performed.
ulalume3@0:         self.is_mandatory = is_mandatory
binietoglou@4: 
ulalume3@0:     def check(self, netcdf_file, variable_name):
ulalume3@0:         the_variable = netcdf_file.variables.get(variable_name, None)
ulalume3@0:         error = []
binietoglou@4: 
ulalume3@0:         if the_variable != None:
ulalume3@0:             # If the variable is found in the file 
ulalume3@0:             self.variable_exists = True
ulalume3@0:             check_passed = True
binietoglou@4: 
ulalume3@0:         else:
ulalume3@0:             self.variable_exists = False
ulalume3@0:             if self.is_mandatory:
ulalume3@0:                 # If a mandatory variable is not found in the file
ulalume3@0:                 check_passed = False
binietoglou@4:                 error.append(
binietoglou@4:                     SpecError('The variable {0} is obligatory but was not found in the file.'.format(variable_name)))
ulalume3@0:             else:
ulalume3@0:                 check_passed = True
binietoglou@4:                 error.append(
binietoglou@4:                     SpecNotification('The optional variable {0} was not found in the file.'.format(variable_name)))
binietoglou@4: 
ulalume3@0:         self.check_passed = check_passed
ulalume3@0:         self.error = error
binietoglou@4: 
ulalume3@0:         return check_passed, error
binietoglou@4: 
ulalume3@0:     @property
ulalume3@0:     def continue_check(self):
ulalume3@0:         if (not self.variable_exists) and (self.block_next):
ulalume3@0:             return False
ulalume3@0:         else:
ulalume3@0:             return True
ulalume3@0: 
binietoglou@4: 
ulalume3@0: class VariableDimensions(GenericSpecification):
ulalume3@0:     def __init__(self, dimensions):
ulalume3@0:         self.dimensions = dimensions
binietoglou@4: 
ulalume3@0:     def check(self, netcdf_file, variable_name):
ulalume3@0:         the_variable = netcdf_file.variables.get(variable_name, None)
ulalume3@0: 
ulalume3@0:         if the_variable != None:
ulalume3@0:             variable_dimensions = list(the_variable.dimensions)
ulalume3@0:             error = []
ulalume3@0:             check_passed = True
ulalume3@0:             for dimension in self.dimensions:
ulalume3@0:                 if not (dimension in variable_dimensions):
ulalume3@0:                     check_passed = False
binietoglou@4:                     error.append(
binietoglou@4:                         SpecError("Variable {0} does not have dimension {1}.".format(variable_name, dimension)))
binietoglou@4: 
ulalume3@0:             # If all dimensions are present, check if the variables are in the 
ulalume3@0:             # correct order.
ulalume3@0:             if check_passed:
ulalume3@0:                 if list(self.dimensions) != variable_dimensions:
ulalume3@0:                     check_passed = False
binietoglou@4:                     error.append(
binietoglou@4:                         SpecError("Variable {0} has wrong dimension order: {1} instead of {2}.".format(variable_name,
binietoglou@4:                                                                                                        variable_dimensions,
binietoglou@4:                                                                                                        list(
binietoglou@4:                                                                                                            self.dimensions))))
ulalume3@0:             for dimension in variable_dimensions:
ulalume3@0:                 if dimension not in self.dimensions:
binietoglou@4:                     error.append(SpecWarning(
binietoglou@4:                         'Dimension {0} found in variable {1} but is not defined in the specifications'.format(dimension,
binietoglou@4:                                                                                                               variable_name)))
binietoglou@4: 
ulalume3@0:         else:
ulalume3@0:             check_passed = False
binietoglou@4:             error = [SpecError('Variable {0} should be checked for dimensions, but was not found in the file.'.format(
binietoglou@4:                 variable_name)), ]
binietoglou@4: 
ulalume3@0:         self.check_passed = check_passed
ulalume3@0:         self.error = error
binietoglou@4: 
ulalume3@0:         return check_passed, error
ulalume3@0: 
ulalume3@0: 
ulalume3@0: class VariableType(GenericSpecification):
ulalume3@0:     def __init__(self, dtype):
ulalume3@0:         self.dtype = dtype
binietoglou@4: 
ulalume3@0:     def check(self, netcdf_file, variable_name):
ulalume3@0:         the_variable = netcdf_file.variables.get(variable_name, None)
ulalume3@0:         error = []
binietoglou@4: 
ulalume3@0:         if the_variable != None:
ulalume3@0:             # Get the internal python type and not the numpy.dtype.
ulalume3@0:             # The conversions guarantee (?) that a single element is always returned
ulalume3@0:             variable_type_python = type(np.asscalar(np.asarray(np.asarray(the_variable[:]).item(0))))
binietoglou@4: 
ulalume3@0:             if not (variable_type_python == self.dtype):
ulalume3@0:                 check_passed = False
binietoglou@4:                 error.append(SpecError(
binietoglou@4:                     'Variable {0} is of type {1} while it should be {2}'.format(variable_name, the_variable.dtype,
binietoglou@4:                                                                                 self.dtype)))
ulalume3@0:             else:
ulalume3@0:                 check_passed = True
ulalume3@0:         else:
ulalume3@0:             check_passed = False
binietoglou@4:             error.append(SpecError(
binietoglou@4:                 'Variable {0} should be checked for type, but was not found in the file.'.format(variable_name)))
binietoglou@4: 
ulalume3@0:         self.check_passed = check_passed
ulalume3@0:         self.error = error
binietoglou@4: 
ulalume3@0:         return check_passed, error
ulalume3@0: 
ulalume3@0: 
ulalume3@0: class AttributeMandatory(GenericSpecification):
binietoglou@4:     def __init__(self, is_mandatory=True):
binietoglou@4:         self.block_next = True  # if true the next checks for this variable will not be performed.
ulalume3@0:         self.is_mandatory = is_mandatory
binietoglou@4: 
ulalume3@0:     def check(self, netcdf_file, attribute_name):
ulalume3@0:         the_attribute = getattr(netcdf_file, attribute_name, None)
ulalume3@0:         error = []
binietoglou@4: 
ulalume3@0:         if the_attribute:
ulalume3@0:             # If the variable is found in the file 
ulalume3@0:             self.attribute_exists = True
ulalume3@0:             check_passed = True
ulalume3@0: 
ulalume3@0:         else:
ulalume3@0:             self.attribute_exists = False
ulalume3@0:             if self.is_mandatory:
ulalume3@0:                 # If a mandatory variable is not found in the file
ulalume3@0:                 check_passed = False
binietoglou@4:                 error.append(
binietoglou@4:                     SpecError('The attribute {0} is obligatory but was not found in the file.'.format(attribute_name)))
ulalume3@0:             else:
ulalume3@0:                 check_passed = True
binietoglou@4:                 error.append(
binietoglou@4:                     SpecNotification('The optional attribute {0} was not found in the file.'.format(attribute_name)))
binietoglou@4: 
ulalume3@0:         self.check_passed = check_passed
ulalume3@0:         self.error = error
binietoglou@4: 
ulalume3@0:         return check_passed, error
binietoglou@4: 
ulalume3@0:     @property
ulalume3@0:     def continue_check(self):
ulalume3@0:         if (not self.attribute_exists) and (self.block_next):
ulalume3@0:             return False
ulalume3@0:         else:
ulalume3@0:             return True
ulalume3@0: 
binietoglou@4: 
ulalume3@0: class AttributeType(GenericSpecification):
binietoglou@4:     def __init__(self, dtype, block_next=True):
ulalume3@0:         self.block_next = block_next
ulalume3@0:         self.dtype = dtype
binietoglou@4: 
ulalume3@0:     def check(self, netcdf_file, attribute_name):
ulalume3@0:         the_attribute = getattr(netcdf_file, attribute_name, None)
ulalume3@0:         error = []
binietoglou@4: 
ulalume3@0:         if the_attribute:
ulalume3@0:             # Get the internal python type and not the numpy.dtype.
ulalume3@0:             # The conversions guarantee (?) that a single element is always returned
ulalume3@0:             try:
ulalume3@0:                 attribute_type_python = type(np.asscalar(np.asarray(np.asarray(the_attribute[:]).item(0))))
ulalume3@0:             except:
ulalume3@0:                 attribute_type_python = type(np.asscalar(the_attribute))
binietoglou@4: 
ulalume3@0:             if not (attribute_type_python == self.dtype):
ulalume3@0:                 check_passed = False
binietoglou@4:                 error.append(SpecError('Attribute {0} is of type {1} while it should be {2}'.format(attribute_name,
binietoglou@4:                                                                                                     type(
binietoglou@4:                                                                                                         the_attribute).__name__,
binietoglou@4:                                                                                                     self.dtype.__name__)))
ulalume3@0:             else:
ulalume3@0:                 error = None
ulalume3@0:                 check_passed = True
ulalume3@0:         else:
ulalume3@0:             check_passed = False
binietoglou@4:             error.append(SpecError(
binietoglou@4:                 'Attribute {0} should be checked for type, but was not found in the file.'.format(attribute_name)))
binietoglou@4: 
ulalume3@0:         self.check_passed = check_passed
ulalume3@0:         self.error = error
binietoglou@4: 
ulalume3@0:         return check_passed, error
ulalume3@0: 
ulalume3@0:     @property
ulalume3@0:     def continue_check(self):
ulalume3@0:         if (not self.check_passed) and (self.block_next):
ulalume3@0:             return False
ulalume3@0:         else:
ulalume3@0:             return True
ulalume3@0: 
ulalume3@0: 
ulalume3@0: class AttributeStrLength(GenericSpecification):
ulalume3@0:     def __init__(self, length):
ulalume3@0:         self.length = length
binietoglou@4: 
ulalume3@0:     def check(self, netcdf_file, attribute_name):
ulalume3@0:         the_attribute = getattr(netcdf_file, attribute_name, None)
ulalume3@0:         error = []
binietoglou@4: 
ulalume3@0:         if the_attribute:
ulalume3@0:             if len(the_attribute) != self.length:
ulalume3@0:                 check_passed = False
binietoglou@4:                 error.append(SpecError(
binietoglou@4:                     'Attribute {0} should be of length {1} while it has length {2}'.format(attribute_name, self.length,
binietoglou@4:                                                                                            len(the_attribute))))
ulalume3@0:             else:
ulalume3@0:                 check_passed = True
ulalume3@0:         else:
ulalume3@0:             check_passed = False
binietoglou@4:             error.append(SpecError(
binietoglou@4:                 'Attribute {0} should be checked for length, but was not found in the file.'.format(attribute_name)))
binietoglou@4: 
ulalume3@0:         self.check_passed = check_passed
ulalume3@0:         self.error = error
binietoglou@4: 
binietoglou@4:         return check_passed, error
ulalume3@0: 
ulalume3@0: 
ulalume3@0: class FilenameShellPattern(GenericSpecification):
ulalume3@0:     def __init__(self, shell_pattern):
ulalume3@0:         self.pattern = shell_pattern
binietoglou@4: 
ulalume3@0:     def check(self, netcdf_file, filename):
ulalume3@0:         error = []
binietoglou@4: 
ulalume3@0:         if fnmatch.fnmatch(filename, self.pattern):
ulalume3@0:             check_passed = True
ulalume3@0:         else:
ulalume3@0:             check_passed = False
ulalume3@0:             error.append(SpecError('Filename {0} does not match patter {1}'.format(filename, self.pattern)))
binietoglou@4: 
ulalume3@0:         self.check_passed = check_passed
ulalume3@0:         self.error = error
binietoglou@4: 
binietoglou@4:         return check_passed, error
ulalume3@0: 
ulalume3@0: 
ulalume3@0: # This is the main class of the script.
ulalume3@0: class FileChecker:
ulalume3@0:     """ It uses the provided specifications to check the a file. 
ulalume3@0:     It can be used with the 'with' statement. For example:
ulalume3@0:     
ulalume3@0:     with FileChecker(filename, specs) as file_checker: 
ulalume3@0:         file_checker.run_checks()
ulalume3@0:         file_checker.print_report('error')
ulalume3@0:     
ulalume3@0:     """
binietoglou@4: 
ulalume3@0:     def __init__(self, filepath, specs):
ulalume3@0:         self.file = None
ulalume3@0:         self.checks_run = False
ulalume3@0:         self.filepath = filepath
ulalume3@0:         self.filename = os.path.basename(filepath)
ulalume3@0:         self.specs = specs
ulalume3@0:         self.check_results = {}
ulalume3@0:         self.check_results['general'] = []
binietoglou@4: 
ulalume3@0:     def __enter__(self):
ulalume3@0:         self.open_file()
ulalume3@0:         return self
binietoglou@4: 
ulalume3@0:     def __exit__(self, type, value, traceback):
ulalume3@0:         if self.file:
ulalume3@0:             self.file.close()
binietoglou@4: 
ulalume3@0:     def open_file(self):
ulalume3@0:         try:
ulalume3@0:             self.file = netcdf.Dataset(self.filepath)
ulalume3@0:         except:
ulalume3@0:             self.check_results['general'].append(SpecError('Could not open file {0}.'.format(self.filename)))
binietoglou@4: 
ulalume3@0:     def close_file(self):
ulalume3@0:         self.file.close()
binietoglou@4: 
ulalume3@0:     def run_checks(self):
binietoglou@4:         if self.file:
ulalume3@0:             self.check_file()
ulalume3@0:             self.check_attributes()
ulalume3@0:             self.check_dimensions()
ulalume3@0:             self.check_variables()
ulalume3@0:         self.checks_run = True
ulalume3@0: 
ulalume3@0:     def check_file(self):
ulalume3@0:         self.check_results['file'] = []
binietoglou@4: 
ulalume3@0:         try:
ulalume3@0:             specs_file = self.specs['file']
ulalume3@0:         except:
ulalume3@0:             specs_file = []
binietoglou@4: 
ulalume3@0:         for file_spec in specs_file:
binietoglou@4:             check_passed, error = file_spec.check(self.file, self.filename)
binietoglou@4: 
ulalume3@0:             if error:
ulalume3@0:                 self.check_results['file'].extend(list(error))
binietoglou@4: 
ulalume3@0:             if not file_spec.continue_check:
binietoglou@4:                 break
binietoglou@4: 
ulalume3@0:     def check_attributes(self):
ulalume3@0:         """ Check if attributes are according to specs """
binietoglou@4: 
ulalume3@0:         self.check_results['attributes'] = []
binietoglou@4: 
ulalume3@0:         try:
ulalume3@0:             spec_attributes = self.specs['attributes'].keys()
ulalume3@0:         except:
ulalume3@0:             spec_attributes = []
binietoglou@4: 
ulalume3@0:         for attribute_name in spec_attributes:
ulalume3@0:             attribute_specs = self.specs['attributes'][attribute_name]
ulalume3@0:             for attribute_spec in attribute_specs:
ulalume3@0:                 check_passed, error = attribute_spec.check(self.file, attribute_name)
ulalume3@0: 
ulalume3@0:                 if error:
binietoglou@4:                     self.check_results['attributes'].extend(list(error))
ulalume3@0: 
ulalume3@0:                 if not attribute_spec.continue_check:
binietoglou@4:                     break  # Don't continue checking specifications if a blocking check failed.
binietoglou@4: 
ulalume3@0:         for attribute_name in self.file.ncattrs():
ulalume3@0:             if attribute_name not in spec_attributes:
binietoglou@4:                 self.check_results['attributes'].append(SpecWarning(
binietoglou@4:                     'Attribute {0} found in the file but is not defined in the specifications'.format(attribute_name)))
binietoglou@4: 
ulalume3@0:     def check_dimensions(self):
ulalume3@0:         """ Check if dimension are according to specs """
ulalume3@0:         self.check_results['dimensions'] = []
binietoglou@4: 
ulalume3@0:         try:
ulalume3@0:             spec_dimensions = self.specs['dimensions'].keys()
ulalume3@0:         except:
ulalume3@0:             spec_dimensions = []
binietoglou@4: 
ulalume3@0:         for dimension_name in spec_dimensions:
ulalume3@0:             dimension_specs = self.specs['dimensions'][dimension_name]
ulalume3@0:             for dimension_spec in dimension_specs:
ulalume3@0:                 check_passed, error = dimension_spec.check(self.file, dimension_name)
binietoglou@4: 
ulalume3@0:                 if error:
binietoglou@4:                     self.check_results['dimensions'].extend(list(error))
binietoglou@4: 
ulalume3@0:                 if not dimension_spec.continue_check:
binietoglou@4:                     break  # Don't continue checking specifications if a blocking check failed.
binietoglou@4: 
ulalume3@0:         for dimension in self.file.dimensions:
ulalume3@0:             if dimension not in spec_dimensions:
binietoglou@4:                 self.check_results['dimensions'].append(SpecWarning(
binietoglou@4:                     'Dimension {0} found in the file but is not defined in the specifications'.format(dimension)))
binietoglou@4: 
ulalume3@0:     def check_variables(self):
ulalume3@0:         """ Check if variables are according to specs """
binietoglou@4: 
ulalume3@0:         self.check_results['variables'] = []
binietoglou@4: 
ulalume3@0:         try:
ulalume3@0:             spec_variables = self.specs['variables'].keys()
ulalume3@0:         except:
ulalume3@0:             spec_variables = []
binietoglou@4: 
ulalume3@0:         for variable_name in spec_variables:
ulalume3@0:             variable_specs = self.specs['variables'][variable_name]
ulalume3@0:             for variable_spec in variable_specs:
ulalume3@0:                 check_passed, error = variable_spec.check(self.file, variable_name)
ulalume3@0: 
ulalume3@0:                 if error:
binietoglou@4:                     self.check_results['variables'].extend(list(error))
ulalume3@0: 
ulalume3@0:                 if not variable_spec.continue_check:
binietoglou@4:                     break  # Don't continue checking specifications if a blocking check failed.
binietoglou@4: 
ulalume3@0:         for variable_name in self.file.variables:
ulalume3@0:             if variable_name not in spec_variables:
binietoglou@4:                 self.check_results['variables'].append(SpecWarning(
binietoglou@4:                     'Variable {0} found in the file but is not defined in the specifications'.format(variable_name)))
ulalume3@0: 
binietoglou@4:     def file_ok(self, level='error'):
ulalume3@0:         """ Check if the file checked is ok. What ok means is defined by the level variable """
binietoglou@4: 
binietoglou@4:         status = None
ulalume3@0:         if self.checks_run:
ulalume3@0:             status = True
ulalume3@0:             for category, result_list in self.check_results.items():
ulalume3@0:                 for result in result_list:
ulalume3@0:                     if ERROR_ORDER[result.level] >= ERROR_ORDER[level]:
ulalume3@0:                         status = False
binietoglou@4: 
ulalume3@0:         return status
binietoglou@4: 
ulalume3@0:     def results_for_level(self, level):
ulalume3@0:         """ Returns all the results of a specific level """
ulalume3@0:         results = None
ulalume3@0:         if self.checks_run:
ulalume3@0:             results = []
ulalume3@0:             for category, result_list in self.check_results.items():
ulalume3@0:                 for result in result_list:
ulalume3@0:                     if ERROR_ORDER[result.level] == ERROR_ORDER[level]:
ulalume3@0:                         results.append(result)
binietoglou@4: 
ulalume3@0:         return results
binietoglou@4: 
ulalume3@0:     def results_by_level(self):
ulalume3@0:         """ Returns a dictionary with the results by level. """
binietoglou@4: 
ulalume3@0:         results = {}
ulalume3@0:         for level, order in ERROR_ORDER.items():
ulalume3@0:             results[level] = self.results_for_level(level)
binietoglou@4: 
ulalume3@0:         return results
binietoglou@4: 
ulalume3@0:     def result_count(self):
ulalume3@0:         """ Returns a dictionary with the number of results per category. """
binietoglou@4: 
ulalume3@0:         result_number = {}
ulalume3@0:         results = self.results_by_level()
binietoglou@4: 
ulalume3@0:         for category, error_list in results.items():
ulalume3@0:             if error_list is None:
ulalume3@0:                 result_number[category] = 0
ulalume3@0:             else:
ulalume3@0:                 result_number[category] = len(error_list)
ulalume3@0:         return result_number
binietoglou@4: 
ulalume3@0:     def print_report(self, level):
ulalume3@0:         """ Print a report for the given level. """
binietoglou@4: 
ulalume3@0:         print header_template.format(self, self.result_count())
binietoglou@4: 
ulalume3@0:         results = self.results_by_level()
binietoglou@4: 
ulalume3@0:         for result_level in ['error', 'warning', 'notification']:
ulalume3@0:             if ERROR_ORDER[result_level] >= ERROR_ORDER[level]:
ulalume3@0:                 print "\n{0} details".format(result_level.capitalize())
ulalume3@0:                 print "----------------"
ulalume3@0:                 for result in results[result_level]:
binietoglou@4:                     print result
binietoglou@4: 
binietoglou@4: 
binietoglou@4:                 # Sounding file specifications
binietoglou@4: 
ulalume3@0: 
binietoglou@4: sounding_specs = {'file': [FilenameShellPattern('rs_*.nc'), ],
binietoglou@4:                   'dimensions': {'points': [DimensionMandatory(True),
binietoglou@4:                                             DimensionUnlimited(False), ],
binietoglou@4:                                  },
binietoglou@4:                   'variables': {'Altitude': [VariableMandatory(True),
binietoglou@4:                                              VariableDimensions(['points', ]),
binietoglou@4:                                              VariableType(float)],
binietoglou@4:                                 'Temperature': [VariableMandatory(True),
binietoglou@4:                                                 VariableDimensions(['points', ]),
binietoglou@4:                                                 VariableType(float)],
binietoglou@4:                                 'Pressure': [VariableMandatory(True),
binietoglou@4:                                              VariableDimensions(['points', ]),
binietoglou@4:                                              VariableType(float)],
binietoglou@4:                                 'RelativeHumidity': [VariableMandatory(False),
binietoglou@4:                                                      VariableDimensions(['points', ]),
binietoglou@4:                                                      VariableType(float)],
binietoglou@4:                                 },
binietoglou@4:                   'attributes': {'Latitude_degrees_north': [AttributeMandatory(True),
binietoglou@4:                                                             AttributeType(float), ],
binietoglou@4:                                  'Longitude_degrees_east': [AttributeMandatory(True),
binietoglou@4:                                                             AttributeType(float), ],
binietoglou@4:                                  'Altitude_meter_asl': [AttributeMandatory(True),
binietoglou@4:                                                         AttributeType(float), ],
binietoglou@4:                                  'Location': [AttributeMandatory(False),
binietoglou@4:                                               AttributeType(unicode), ],
binietoglou@4:                                  'Sounding_Station_Name': [AttributeMandatory(False),
binietoglou@4:                                                            AttributeType(unicode), ],
binietoglou@4:                                  'WMO_Station_Number': [AttributeMandatory(False),
binietoglou@4:                                                         AttributeType(unicode), ],
binietoglou@4:                                  'WBAN_Station_Number': [AttributeMandatory(False),
binietoglou@4:                                                          AttributeType(unicode), ],
binietoglou@4:                                  'Sounding_Start_Date': [AttributeMandatory(True),
binietoglou@4:                                                          AttributeType(unicode, block_next=True),
binietoglou@4:                                                          AttributeStrLength(8)],
binietoglou@4:                                  'Sounding_Start_Time_UT': [AttributeMandatory(True),
binietoglou@4:                                                             AttributeType(unicode, block_next=True),
binietoglou@4:                                                             AttributeStrLength(6)],
binietoglou@4:                                  'Sounding_Stop_Time_UT': [AttributeMandatory(False),
binietoglou@4:                                                            AttributeType(unicode, block_next=True),
binietoglou@4:                                                            AttributeStrLength(6)],
binietoglou@4:                                  },
binietoglou@4:                   'name': "SCC Sounding file"
binietoglou@4:                   }
ulalume3@0: 
ulalume3@0: # Lidar ratio file specifications
binietoglou@4: lidar_ratio_specs = {'file': [FilenameShellPattern('*.nc'), ],
binietoglou@4:                      'dimensions': {'points': [DimensionMandatory(True),
binietoglou@4:                                                DimensionUnlimited(False), ],
binietoglou@4:                                     'products': [DimensionMandatory(True),
binietoglou@4:                                                  DimensionUnlimited(False), ],
binietoglou@4:                                     },
binietoglou@4:                      'variables': {'Altitude': [VariableMandatory(True),
binietoglou@4:                                                 VariableDimensions(['points', ]),
binietoglou@4:                                                 VariableType(float)],
binietoglou@4:                                    'Lidar_Ratio': [VariableMandatory(True),
binietoglou@4:                                                    VariableDimensions(['products', 'points']),
ulalume3@0:                                                    VariableType(float)],
binietoglou@4:                                    'product_ID': [VariableMandatory(True),
binietoglou@4:                                                   VariableDimensions(['products', ]),
binietoglou@4:                                                   VariableType(int)],
binietoglou@4:                                    },
binietoglou@4:                      'attributes': {'Lidar_Station_Name': [AttributeMandatory(True),
binietoglou@4:                                                            AttributeType(unicode), ],
binietoglou@4:                                     },
binietoglou@4:                      'name': "SCC Lidar ratio file"
binietoglou@4:                      }
ulalume3@0: 
ulalume3@0: # Overlap file specifications
binietoglou@4: overlap_specs = {'file': [FilenameShellPattern('ov_*.nc'), ],
binietoglou@4:                  'dimensions': {'points': [DimensionMandatory(True),
binietoglou@4:                                            DimensionUnlimited(False), ],
binietoglou@4:                                 'channels': [DimensionMandatory(True),
binietoglou@4:                                              DimensionUnlimited(False), ],
binietoglou@4:                                 },
binietoglou@4:                  'variables': {'Altitude': [VariableMandatory(True),
binietoglou@4:                                             VariableDimensions(['points', ]),
ulalume3@0:                                             VariableType(float)],
binietoglou@4:                                'Overlap_Function': [VariableMandatory(True),
binietoglou@4:                                                     VariableDimensions(['channels', 'points']),
ulalume3@0:                                                     VariableType(float)],
binietoglou@4:                                'channel_ID': [VariableMandatory(True),
binietoglou@4:                                               VariableDimensions(['channels', ]),
ulalume3@0:                                               VariableType(int)],
binietoglou@4:                                },
ulalume3@0:                  'attributes': {'Lidar_Station_Name': [AttributeMandatory(True),
binietoglou@4:                                                        AttributeType(unicode, block_next=True),
ulalume3@0:                                                        AttributeStrLength(2)],
ulalume3@0:                                 'Overlap_Measurement_Date': [AttributeMandatory(True),
binietoglou@4:                                                              AttributeType(unicode, block_next=True),
ulalume3@0:                                                              AttributeStrLength(8)],
binietoglou@4:                                 },
ulalume3@0:                  'name': "SCC Overlap file"
binietoglou@4:                  }
ulalume3@0: 
ulalume3@0: # Raw data file specifications
binietoglou@4: data_specs = {'file': [FilenameShellPattern('*.nc'), ],
binietoglou@4:               'dimensions': {'points': [DimensionMandatory(True),
binietoglou@4:                                         DimensionUnlimited(False), ],
binietoglou@4:                              'channels': [DimensionMandatory(True),
binietoglou@4:                                           DimensionUnlimited(False), ],
binietoglou@4:                              'nb_of_time_scales': [DimensionMandatory(True),
binietoglou@4:                                                    DimensionUnlimited(False), ],
binietoglou@4:                              'time': [DimensionMandatory(True),
binietoglou@4:                                       DimensionUnlimited(True), ],
binietoglou@4:                              'time_bck': [DimensionMandatory(False),
binietoglou@4:                                           DimensionUnlimited(False), ],
binietoglou@4:                              'scan_angles': [DimensionMandatory(True),
binietoglou@4:                                              DimensionUnlimited(False), ],
binietoglou@4:                              },
binietoglou@4:               'variables': {'channel_ID': [VariableMandatory(True),
binietoglou@4:                                            VariableDimensions(['channels', ]),
binietoglou@4:                                            VariableType(int)],
binietoglou@4:                             'Laser_Repetition_Rate': [VariableMandatory(False),
binietoglou@4:                                                       VariableDimensions(['channels', ]),
ulalume3@0:                                                       VariableType(int)],
binietoglou@4:                             'Laser_Pointing_Angle': [VariableMandatory(True),
binietoglou@4:                                                      VariableDimensions(['scan_angles', ]),
binietoglou@4:                                                      VariableType(float)],
binietoglou@4:                             'ID_Range': [VariableMandatory(False),
binietoglou@4:                                          VariableDimensions(['channels', ]),
binietoglou@4:                                          VariableType(int)],
binietoglou@4:                             'Scattering_Mechanism': [VariableMandatory(False),
binietoglou@4:                                                      VariableDimensions(['channels', ]),
binietoglou@4:                                                      VariableType(int)],
binietoglou@4:                             'Emitted_Wavelength': [VariableMandatory(False),
binietoglou@4:                                                    VariableDimensions(['channels', ]),
binietoglou@4:                                                    VariableType(float)],
binietoglou@4:                             'Detected_Wavelength': [VariableMandatory(False),
binietoglou@4:                                                     VariableDimensions(['channels', ]),
binietoglou@4:                                                     VariableType(float)],
binietoglou@4:                             'Raw_Data_Range_Resolution': [VariableMandatory(False),
binietoglou@4:                                                           VariableDimensions(['channels', ]),
binietoglou@4:                                                           VariableType(float)],
binietoglou@4:                             'Background_Mode': [VariableMandatory(False),
binietoglou@4:                                                 VariableDimensions(['channels', ]),
binietoglou@4:                                                 VariableType(int)],
binietoglou@4:                             'Background_Low': [VariableMandatory(True),
binietoglou@4:                                                VariableDimensions(['channels', ]),
binietoglou@4:                                                VariableType(float)],
binietoglou@4:                             'Background_High': [VariableMandatory(True),
binietoglou@4:                                                 VariableDimensions(['channels', ]),
binietoglou@4:                                                 VariableType(float)],
binietoglou@4:                             'Molecular_Calc': [VariableMandatory(True),
binietoglou@4:                                                VariableDimensions([]),
ulalume3@0:                                                VariableType(int)],
binietoglou@4:                             'id_timescale': [VariableMandatory(True),
binietoglou@4:                                              VariableDimensions(['channels', ]),
binietoglou@4:                                              VariableType(int)],
binietoglou@4:                             'Dead_Time_Corr_Type': [VariableMandatory(False),
binietoglou@4:                                                     VariableDimensions(['channels', ]),
binietoglou@4:                                                     VariableType(int)],
binietoglou@4:                             'Dead_Time': [VariableMandatory(False),
binietoglou@4:                                           VariableDimensions(['channels', ]),
binietoglou@4:                                           VariableType(float)],
binietoglou@4:                             'Acquisition_Mode': [VariableMandatory(False),
binietoglou@4:                                                  VariableDimensions(['channels', ]),
binietoglou@4:                                                  VariableType(int)],
binietoglou@4:                             'Trigger_Delay': [VariableMandatory(False),
binietoglou@4:                                               VariableDimensions(['channels', ]),
binietoglou@4:                                               VariableType(float)],
binietoglou@4:                             'Laser_Pointing_Angle_of_Profiles': [VariableMandatory(True),
binietoglou@4:                                                                  VariableDimensions(['time', 'nb_of_time_scales', ]),
binietoglou@4:                                                                  VariableType(int)],
binietoglou@4:                             'Raw_Data_Start_Time': [VariableMandatory(True),
binietoglou@4:                                                     VariableDimensions(['time', 'nb_of_time_scales', ]),
binietoglou@4:                                                     VariableType(int)],
binietoglou@4:                             'Raw_Data_Stop_Time': [VariableMandatory(True),
binietoglou@4:                                                    VariableDimensions(['time', 'nb_of_time_scales', ]),
binietoglou@4:                                                    VariableType(int)],
binietoglou@4:                             'Laser_Shots': [VariableMandatory(True),
binietoglou@4:                                             VariableDimensions(['time', 'channels', ]),
ulalume3@0:                                             VariableType(int)],
binietoglou@4:                             'Raw_Lidar_Data': [VariableMandatory(False),
binietoglou@4:                                                VariableDimensions(['time', 'channels', 'points']),
binietoglou@4:                                                VariableType(float)],
binietoglou@4:                             'Depolarization_Factor': [VariableMandatory(False),
binietoglou@4:                                                       VariableDimensions(['channels', ]),
binietoglou@4:                                                       VariableType(float)],
binietoglou@4:                             'LR_Input': [VariableMandatory(False),
binietoglou@4:                                          VariableDimensions(['channels', ]),
binietoglou@4:                                          VariableType(int)],
binietoglou@4:                             'DAQ_Range': [VariableMandatory(False),
binietoglou@4:                                           VariableDimensions(['channels', ]),
binietoglou@4:                                           VariableType(float)],
binietoglou@4:                             'Pressure_at_Lidar_Station': [VariableMandatory(False),
binietoglou@4:                                                           VariableDimensions([]),
binietoglou@4:                                                           VariableType(float)],
binietoglou@4:                             'Temperature_at_Lidar_Station': [VariableMandatory(False),
binietoglou@4:                                                              VariableDimensions([]),
ulalume3@0:                                                              VariableType(float)],
binietoglou@4:                             'Background_Profile': [VariableMandatory(False),
binietoglou@4:                                                    VariableDimensions(['time_bck', 'channels', 'points']),
binietoglou@4:                                                    VariableType(float)],
binietoglou@4:                             'Raw_Bck_Start_Time': [VariableMandatory(False),
binietoglou@4:                                                    VariableDimensions(['time_bck', 'nb_of_time_scales', ]),
binietoglou@4:                                                    VariableType(int)],
binietoglou@4:                             'Raw_Bck_Stop_Time': [VariableMandatory(False),
binietoglou@4:                                                   VariableDimensions(['time_bck', 'nb_of_time_scales', ]),
binietoglou@4:                                                   VariableType(int)],
binietoglou@4:                             'Error_On_Raw_Lidar_Data': [VariableMandatory(False),
binietoglou@4:                                                         VariableDimensions(['time', 'channels', 'points']),
binietoglou@4:                                                         VariableType(float)],
binietoglou@4:                             'First_Signal_Rangebin': [VariableMandatory(False),
binietoglou@4:                                                       VariableDimensions(['channels', ]),
ulalume3@0:                                                       VariableType(int)],
binietoglou@4:                             },
binietoglou@4:               'attributes': {'Measurement_ID': [AttributeMandatory(True),
binietoglou@4:                                                 AttributeType(unicode, block_next=True),
binietoglou@4:                                                 AttributeStrLength(12)],
binietoglou@4:                              'RawData_Start_Date': [AttributeMandatory(True),
binietoglou@4:                                                     AttributeType(unicode, block_next=True),
binietoglou@4:                                                     AttributeStrLength(8)],
binietoglou@4:                              'RawData_Start_Time_UT': [AttributeMandatory(True),
binietoglou@4:                                                        AttributeType(unicode, block_next=True),
binietoglou@4:                                                        AttributeStrLength(6)],
binietoglou@4:                              'RawData_Stop_Time_UT': [AttributeMandatory(True),
binietoglou@4:                                                       AttributeType(unicode, block_next=True),
binietoglou@4:                                                       AttributeStrLength(6)],
binietoglou@4:                              'RawBck_Start_Date': [AttributeMandatory(False),
binietoglou@4:                                                    AttributeType(unicode, block_next=True),
binietoglou@4:                                                    AttributeStrLength(8)],
binietoglou@4:                              'RawBck_Start_Time_UT': [AttributeMandatory(False),
binietoglou@4:                                                       AttributeType(unicode, block_next=True),
binietoglou@4:                                                       AttributeStrLength(6)],
binietoglou@4:                              'RawBck_Stop_Time_UT': [AttributeMandatory(False),
binietoglou@4:                                                      AttributeType(unicode, block_next=True),
binietoglou@4:                                                      AttributeStrLength(6)],
binietoglou@4:                              'Sounding_File_Name': [AttributeMandatory(False),
binietoglou@4:                                                     AttributeType(unicode), ],
binietoglou@4:                              'LR_File_Name': [AttributeMandatory(False),
binietoglou@4:                                               AttributeType(unicode), ],
binietoglou@4:                              'Overlap_File_Name': [AttributeMandatory(False),
binietoglou@4:                                                    AttributeType(unicode), ],
binietoglou@4:                              'Location': [AttributeMandatory(False),
binietoglou@4:                                           AttributeType(unicode), ],
binietoglou@4:                              'System': [AttributeMandatory(False),
binietoglou@4:                                         AttributeType(unicode), ],
binietoglou@4:                              'Latitude_degrees_north': [AttributeMandatory(False),
binietoglou@4:                                                         AttributeType(float), ],
binietoglou@4:                              'Longitude_degrees_east': [AttributeMandatory(False),
binietoglou@4:                                                         AttributeType(float), ],
binietoglou@4:                              'Altitude_meter_asl': [AttributeMandatory(False),
binietoglou@4:                                                     AttributeType(float), ],
ulalume3@0:                              },
binietoglou@4:               'name': "SCC Raw input file"
binietoglou@4:               }
ulalume3@0: 
ulalume3@0: # Used for the command line arguments
ulalume3@0: spec_shorthands = {'sounding': sounding_specs,
ulalume3@0:                    'lidar_ratio': lidar_ratio_specs,
ulalume3@0:                    'overlap': overlap_specs,
ulalume3@0:                    'data': data_specs,}
ulalume3@0: 
ulalume3@0: if __name__ == "__main__":
ulalume3@0:     # For use from a terminal
ulalume3@0:     import argparse
binietoglou@4: 
ulalume3@0:     parser = argparse.ArgumentParser()
binietoglou@4:     parser.add_argument("file", help="The path of the file to be checked")
binietoglou@4:     parser.add_argument("-s", "--specs", default='data',
binietoglou@4:                         help="The specificiations to use",
binietoglou@4:                         choices=['data', 'overlap', 'lidar_ratio', 'sounding'])
binietoglou@4:     parser.add_argument("-l", "--level", default='warning',
binietoglou@4:                         help="The output level",
binietoglou@4:                         choices=['error', 'warning', 'notification'])
binietoglou@4: 
ulalume3@0:     # Check the arguments
ulalume3@0:     args = parser.parse_args()
binietoglou@4: 
ulalume3@0:     specs = spec_shorthands[args.specs]
binietoglou@4: 
binietoglou@4:     with FileChecker(args.file, specs) as file_checker:
ulalume3@0:         file_checker.run_checks()
ulalume3@0:         file_checker.print_report(args.level)