From 0cd0b356175f619f022cee112cc7835d85c32b0d Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 5 Jan 2023 16:15:53 +0100 Subject: [PATCH 001/399] ENH: refac module init --- pyotb/__init__.py | 26 +++++++++++++++++---- pyotb/apps.py | 58 ++++++++++++++--------------------------------- 2 files changed, 39 insertions(+), 45 deletions(-) diff --git a/pyotb/__init__.py b/pyotb/__init__.py index 187b64a..e3bc23a 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -1,8 +1,26 @@ # -*- coding: utf-8 -*- """This module provides convenient python wrapping of otbApplications.""" -__version__ = "1.5.4" +__version__ = "1.6.0" + +from .helpers import find_otb, logger, set_logger_level + +otb = find_otb() from .apps import * -from .core import App, Output, Input, get_nbchannels, get_pixel_type -from .functions import * # pylint: disable=redefined-builtin -from .helpers import logger, set_logger_level + +from .core import ( + App, + Input, + Output, + get_nbchannels, + get_pixel_type +) + +from .functions import ( # pylint: disable=redefined-builtin + all, + any, + where, + clip, + run_tf_function, + define_processing_area +) diff --git a/pyotb/apps.py b/pyotb/apps.py index e5287bb..30e3363 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -4,9 +4,9 @@ import os import sys from pathlib import Path -from .helpers import logger, find_otb - -otb = find_otb() +import otbApplication as otb +from .core import App +from .helpers import logger def get_available_applications(as_subprocess=False): @@ -55,27 +55,12 @@ def get_available_applications(as_subprocess=False): if not app_list: app_list = otb.Registry.GetAvailableApplications() if not app_list: - logger.warning("Unable to load applications. Set env variable OTB_APPLICATION_PATH then try again") - return () + raise SystemExit("Unable to load applications. Set env variable OTB_APPLICATION_PATH and try again.") logger.info("Successfully loaded %s OTB applications", len(app_list)) return app_list -AVAILABLE_APPLICATIONS = get_available_applications(as_subprocess=True) - -# First core.py call (within __init__ scope) -from .core import App # pylint: disable=wrong-import-position - -# This is to enable aliases of Apps, i.e. using apps like `pyotb.AppName(...)` instead of `pyotb.App("AppName", ...)` -_CODE_TEMPLATE = """ -class {name}(App): - """ """ - def __init__(self, *args, **kwargs): - super().__init__('{name}', *args, **kwargs) -""" - - class OTBTFApp(App): """Helper for OTBTF.""" @staticmethod @@ -111,30 +96,21 @@ class OTBTFApp(App): super().__init__(app_name, *args, **kwargs) +AVAILABLE_APPLICATIONS = get_available_applications(as_subprocess=True) + +# This is to enable aliases of Apps, i.e. using apps like `pyotb.AppName(...)` instead of `pyotb.App("AppName", ...)` +_CODE_TEMPLATE = """ +class {name}(App): + """ """ + def __init__(self, *args, **kwargs): + super().__init__('{name}', *args, **kwargs) +""" + for _app in AVAILABLE_APPLICATIONS: # Customize the behavior for some OTBTF applications. The user doesn't need to set the env variable # `OTB_TF_NSOURCES`, it is handled in pyotb - if _app == 'TensorflowModelServe': - class TensorflowModelServe(OTBTFApp): - """Serve a Tensorflow model using OTBTF.""" - def __init__(self, *args, n_sources=None, **kwargs): - """Constructor for a TensorflowModelServe object.""" - super().__init__('TensorflowModelServe', *args, n_sources=n_sources, **kwargs) - - elif _app == 'PatchesExtraction': - class PatchesExtraction(OTBTFApp): - """Extract patches using OTBTF.""" - def __init__(self, *args, n_sources=None, **kwargs): - """Constructor for a PatchesExtraction object.""" - super().__init__('PatchesExtraction', *args, n_sources=n_sources, **kwargs) - - elif _app == 'TensorflowModelTrain': - class TensorflowModelTrain(OTBTFApp): - """Train a Tensorflow model using OTBTF.""" - def __init__(self, *args, n_sources=None, **kwargs): - """Constructor for a TensorflowModelTrain object.""" - super().__init__('TensorflowModelTrain', *args, n_sources=n_sources, **kwargs) - + if _app in ("PatchesExtraction", "TensorflowModelTrain", "TensorflowModelServe"): + exec(_CODE_TEMPLATE.format(name=_app).replace("(App)", "(OTBTFApp)")) # pylint: disable=exec-used # Default behavior for any OTB application else: - exec(_CODE_TEMPLATE.format(name=_app)) # pylint: disable=exec-used + exec(_CODE_TEMPLATE.format(name=_app)) # pylint: disable=exec-used \ No newline at end of file -- GitLab From b4208119179e99d12c3c88b390515cee8620cc2d Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 5 Jan 2023 19:18:31 +0100 Subject: [PATCH 002/399] ENH: big refac for 1.6 --- pyotb/core.py | 1163 ++++++++++++++++++++++++------------------- pyotb/functions.py | 18 +- tests/test_numpy.py | 2 +- 3 files changed, 660 insertions(+), 523 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 5b9a895..7992890 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -8,11 +8,58 @@ import otbApplication as otb from .helpers import logger -class otbObject: - """Base class that gathers common operations for any OTB in-memory raster.""" +class OTBObject: + """Base class that gathers common operations for any OTB application.""" _name = "" - app = None - output_param = "" + + def __init__(self, appname, *args, frozen=False, quiet=False, image_dic=None, **kwargs): + """Common constructor for OTB applications. Handles in-memory connection between apps. + + Args: + appname: name of the app, e.g. 'BandMath' + *args: used for passing application parameters. Can be : + - dictionary containing key-arguments enumeration. Useful when a key is python-reserved + (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") + - string, App or Output, useful when the user wants to specify the input "in" + - list, useful when the user wants to specify the input list 'il' + frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___ + quiet: whether to print logs of the OTB app + **kwargs: used for passing application parameters. + e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif' + + """ + self.parameters = {} + self.appname = appname + self.quiet = quiet + self.image_dic = image_dic + if quiet: + self.app = otb.Registry.CreateApplicationWithoutLogger(appname) + else: + self.app = otb.Registry.CreateApplication(appname) + self.parameters_keys = tuple(self.app.GetParametersKeys()) + self.out_param_types = dict(get_out_param_types(self)) + self.out_param_keys = tuple(self.out_param_types.keys()) + self.exports_dict = {} + if args or kwargs: + self.set_parameters(*args, **kwargs) + self.frozen = frozen + if not frozen: + self.execute() + + @property + def key_input(self): + """Get the name of first input parameter, raster > vector > file.""" + return self.key_input_image or key_input(self, "vector") or key_input(self, "file") + + @property + def key_input_image(self): + """Get the name of first output image parameter""" + return key_input(self, "raster") + + @property + def key_output_image(self): + """Get the name of first output image parameter""" + return key_output(self, "raster") @property def name(self): @@ -22,17 +69,25 @@ class otbObject: user's defined name or appname """ - return self._name or self.app.GetName() + return self._name or self.appname @name.setter - def name(self, val): + def name(self, name): """Set custom name. Args: - val: new name + name: new name """ - self._name = val + if isinstance(name, str): + self._name = name + else: + raise TypeError(f"{self.name}: bad type ({type(name)}) for application name, only str is allowed") + + @property + def outputs(self): + """List of application outputs.""" + return [getattr(self, key) for key in self.out_param_keys if key in self.parameters] @property def dtype(self): @@ -43,7 +98,7 @@ class otbObject: """ try: - enum = self.app.GetParameterOutputImagePixelType(self.output_param) + enum = self.app.GetParameterOutputImagePixelType(self.key_output_image) return self.app.ConvertPixelTypeToNumpy(enum) except RuntimeError: return None @@ -56,12 +111,114 @@ class otbObject: shape: (height, width, bands) """ - width, height = self.app.GetImageSize(self.output_param) - bands = self.app.GetImageNbBands(self.output_param) + width, height = self.app.GetImageSize(self.key_output_image) + bands = self.app.GetImageNbBands(self.key_output_image) return (height, width, bands) - def write(self, *args, filename_extension="", pixel_type=None, **kwargs): - """Trigger execution, set output pixel type and write the output. + @property + def transform(self): + """Get image affine transform, rasterio style (see https://www.perrygeo.com/python-affine-transforms.html) + + Returns: + transform: (X spacing, X offset, X origin, Y offset, Y spacing, Y origin) + """ + spacing_x, spacing_y = self.app.GetImageSpacing(self.key_output_image) + origin_x, origin_y = self.app.GetImageOrigin(self.key_output_image) + # Shift image origin since OTB is giving coordinates of pixel center instead of corners + origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2 + return (spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y) + + def set_parameters(self, *args, **kwargs): + """Set some parameters of the app. + + When useful, e.g. for images list, this function appends the parameters + instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths + + Args: + *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved + (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") + - string, App or Output, useful when the user implicitly wants to set the param "in" + - list, useful when the user implicitly wants to set the param "il" + **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif' + + Raises: + Exception: when the setting of a parameter failed + + """ + parameters = kwargs + parameters.update(self.__parse_args(args)) + # Going through all arguments + for key, obj in parameters.items(): + if key not in self.parameters_keys: + raise KeyError(f'{self.name}: unknown parameter name "{key}"') + # When the parameter expects a list, if needed, change the value to list + if is_key_list(self, key) and not isinstance(obj, (list, tuple)): + obj = [obj] + logger.info('%s: argument for parameter "%s" was converted to list', self.name, key) + try: + # This is when we actually call self.app.SetParameter* + self.__set_param(key, obj) + except (RuntimeError, TypeError, ValueError, KeyError) as e: + raise Exception( + f"{self.name}: something went wrong before execution " + f"(while setting parameter '{key}' to '{obj}')" + ) from e + # Update _parameters using values from OtbApplication object + otb_params = self.app.GetParameters().items() + otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params} + # Save parameters keys, and values as object attributes + self.parameters.update({**parameters, **otb_params}) + + def execute(self): + """Execute and write to disk if any output parameter has been set during init.""" + logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters) + try: + self.app.Execute() + except (RuntimeError, FileNotFoundError) as e: + raise Exception(f"{self.name}: error during during app execution") from e + self.frozen = False + logger.debug("%s: execution ended", self.name) + if any(key in self.parameters for key in self.out_param_keys): + self.flush() + self.save_objects() + + def flush(self): + """Flush data to disk, this is when WriteOutput is actually called. + + Args: + parameters: key value pairs like {parameter: filename} + dtypes: optional dict to pass output data types (for rasters) + + """ + try: + logger.debug("%s: flushing data to disk", self.name) + self.app.WriteOutput() + except RuntimeError: + logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name) + self.app.ExecuteAndWriteOutput() + + def save_objects(self): + """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`. + + This is useful when the key contains reserved characters such as a point eg "io.out" + """ + for key in self.parameters_keys: + if key in dir(OTBObject): + continue # skip forbidden attribute since it is already used by the class + value = self.parameters.get(key) # basic parameters + if value is None: + try: + value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results) + except RuntimeError: + continue # this is when there is no value for key + # Convert output param path to Output object + if key in self.out_param_keys: + value = Output(self, key, value) + # Save attribute + setattr(self, key, value) + + def write(self, *args, filename_extension="", pixel_type=None, preserve_dtype=False, **kwargs): + """Set output pixel type and write the output raster files. Args: *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key contains @@ -75,7 +232,9 @@ class otbObject: outputs, all outputs are written with this unique type. Valid pixel types are uint8, uint16, uint32, int16, int32, float, double, cint16, cint32, cfloat, cdouble. (Default value = None) + preserve_dtype: propagate main input pixel type to outputs, in case pixel_type is None **kwargs: keyword arguments e.g. out='output.tif' + """ # Gather all input arguments in kwargs dict for arg in args: @@ -83,73 +242,151 @@ class otbObject: kwargs.update(arg) elif isinstance(arg, str) and kwargs: logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg) - elif isinstance(arg, str): - kwargs.update({self.output_param: arg}) - - dtypes = {} - if isinstance(pixel_type, dict): - dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()} - elif pixel_type is not None: - typ = parse_pixel_type(pixel_type) - if isinstance(self, App): - dtypes = {key: typ for key in self.output_parameters_keys} - elif isinstance(self, otbObject): - dtypes = {self.output_param: typ} - + elif isinstance(arg, str) and self.key_output_image: + kwargs.update({self.key_output_image: arg}) + # Append filename extension to filenames if filename_extension: - logger.debug('%s: using extended filename for outputs: %s', self.name, filename_extension) - if not filename_extension.startswith('?'): + logger.debug("%s: using extended filename for outputs: %s", self.name, filename_extension) + if not filename_extension.startswith("?"): filename_extension = "?" + filename_extension + for key, value in kwargs.items(): + if self.out_param_types[key] == 'raster' and '?' not in value: + kwargs[key] = value + filename_extension + # Manage output pixel types + dtypes = {} + if pixel_type: + if isinstance(pixel_type, str): + type_name = self.app.ConvertPixelTypeToNumpy(parse_pixel_type(pixel_type)) + logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name) + for key in kwargs: + if self.out_param_types.get(key) == "raster": + dtypes[key] = parse_pixel_type(pixel_type) + elif isinstance(pixel_type, dict): + dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()} + elif preserve_dtype: + self.propagate_dtype() # all outputs will have the same type as the main input raster + # Apply parameters + for key, output_filename in kwargs.items(): + if key in dtypes: + self.propagate_dtype(key, dtypes[key]) + self.set_parameters({key: output_filename}) - # Case output parameter was set during App init - if not kwargs: - if self.output_param in self.parameters: - if dtypes: - self.app.SetParameterOutputImagePixelType(self.output_param, dtypes[self.output_param]) - if filename_extension: - new_val = self.parameters[self.output_param] + filename_extension - self.app.SetParameterString(self.output_param, new_val) - else: - raise ValueError(f'{self.app.GetName()}: Output parameter is missing.') + self.flush() + self.save_objects() - # Parse kwargs - for key, output_filename in kwargs.items(): - # Stop process if a bad parameter is given - if key not in self.app.GetParametersKeys(): - raise KeyError(f'{self.app.GetName()}: Unknown parameter key "{key}"') - # Check if extended filename was not provided twice - if '?' in output_filename and filename_extension: - logger.warning('%s: extended filename was provided twice. Using the one found in path.', self.name) - elif filename_extension: - output_filename += filename_extension - - logger.debug('%s: "%s" parameter is %s', self.name, key, output_filename) - self.app.SetParameterString(key, output_filename) + def propagate_dtype(self, target_key=None, dtype=None): + """Propagate a pixel type from main input to every outputs, or to a target output key only. - if key in dtypes: - self.app.SetParameterOutputImagePixelType(key, dtypes[key]) + With multiple inputs (if dtype is not provided), the type of the first input is considered. + With multiple outputs (if target_key is not provided), all outputs will be converted to the same pixel type. - logger.debug('%s: flushing data to disk', self.name) - try: - self.app.WriteOutput() - except RuntimeError: - logger.debug('%s: failed to simply write output, executing once again then writing', self.name) - self.app.ExecuteAndWriteOutput() + Args: + target_key: output param key to change pixel type + dtype: data type to use + + """ + if not dtype: + param = self.parameters.get(self.key_input_image) + if not param: + logger.warning("%s: could not propagate pixel type from inputs to output", self.name) + return + if isinstance(param, (list, tuple)): + param = param[0] # first image in "il" + try: + dtype = get_pixel_type(param) + except (TypeError, RuntimeError): + logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param) + return + + if target_key: + keys = [target_key] + else: + keys = [k for k in self.out_param_keys if self.out_param_types[k] == "raster"] + for key in keys: + # Set output pixel type + self.app.SetParameterOutputImagePixelType(key, dtype) - def to_numpy(self, preserve_dtype=True, copy=False): + def read_values_at_coords(self, col, row, bands=None): + """Get pixel value(s) at a given YX coordinates. + + Args: + col: index along X / longitude axis + row: index along Y / latitude axis + bands: band number, list or slice to fetch values from + + Returns: + single numerical value or a list of values for each band + + """ + channels = [] + app = OTBObject("PixelValue", self, coordx=col, coordy=row, frozen=False, quiet=True) + if bands is not None: + if isinstance(bands, int): + if bands < 0: + bands = self.shape[2] + bands + channels = [bands] + elif isinstance(bands, slice): + channels = self.__channels_list_from_slice(bands) + elif not isinstance(bands, list): + raise TypeError(f"{self.name}: type '{bands}' cannot be interpreted as a valid slicing") + if channels: + app.app.Execute() + app.set_parameters({"cl": [f"Channel{n+1}" for n in channels]}) + app.execute() + data = literal_eval(app.app.GetParameterString("value")) + if len(channels) == 1: + return data[0] + return data + + def summarize(self): + """Serialize an object and its pipeline into a dictionary. + + Returns: + nested dictionary summarizing the pipeline + + """ + params = self.parameters + for k, p in params.items(): + # In the following, we replace each parameter which is an OTBObject, with its summary. + if isinstance(p, OTBObject): # single parameter + params[k] = p.summarize() + elif isinstance(p, list): # parameter list + params[k] = [pi.summarize() if isinstance(pi, OTBObject) else pi for pi in p] + + return {"name": self.name, "parameters": params} + + def export(self, key=None): + """Export a specific output image as numpy array and store it in object's exports_dict. + + Args: + key: parameter key to export, if None then the default one will be used + + Returns: + the exported numpy array + + """ + if key is None: + key = key_output(self, "raster") + if key not in self.exports_dict: + self.exports_dict[key] = self.app.ExportImage(key) + return self.exports_dict[key] + + def to_numpy(self, key=None, preserve_dtype=True, copy=False): """Export a pyotb object to numpy array. Args: + key: the output parameter name to export as numpy array preserve_dtype: when set to True, the numpy array is created with the same pixel type as - the otbObject first output. Default is True. + the OTBObject first output. Default is True. copy: whether to copy the output array, default is False required to True if preserve_dtype is False and the source app reference is lost Returns: - a numpy array + a numpy array """ - array = self.app.ExportImage(self.output_param)['array'] + data = self.export(key) + array = data["array"] if preserve_dtype: return array.astype(self.dtype) if copy: @@ -166,51 +403,117 @@ class otbObject: """ array = self.to_numpy(preserve_dtype=True, copy=False) array = np.moveaxis(array, 2, 0) - proj = self.app.GetImageProjection(self.output_param) - spacing_x, spacing_y = self.app.GetImageSpacing(self.output_param) - origin_x, origin_y = self.app.GetImageOrigin(self.output_param) - # Shift image origin since OTB is giving coordinates of pixel center instead of corners - origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2 + proj = self.app.GetImageProjection(self.key_output_image) profile = { 'crs': proj, 'dtype': array.dtype, 'count': array.shape[0], 'height': array.shape[1], 'width': array.shape[2], - 'transform': (spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y) # here we force pixel rotation to 0 ! + 'transform': self.transform } return array, profile - # Special methods - def __getitem__(self, key): - """Override the default __getitem__ behaviour. + def xy_to_rowcol(self, x, y): + """Find (row, col) index using (x, y) projected coordinates, expect image CRS - This function enables 2 things : - - access attributes like that : object['any_attribute'] - - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3] - selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]] - selecting 1000x1000 subset : object[:1000, :1000] + Args: + x: longitude or projected X + y: latitude or projected Y + + Returns: + pixel index: (row, col) + """ + spacing_x, _, origin_x, _, spacing_y, origin_y = self.transform + col = int((x - origin_x) / spacing_x) + row = int((origin_y - y) / spacing_y) + return (row, col) + + # Private functions + def __parse_args(self, args): + """Gather all input arguments in kwargs dict. Args: - key: attribute key + args: the list of arguments passed to set_parameters() Returns: - attribute or Slicer + a dictionary with the right keyword depending on the object + """ - # Accessing string attributes - if isinstance(key, str): - return self.__dict__.get(key) - # Slicing - if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)): - raise ValueError(f'"{key}"cannot be interpreted as valid slicing. Slicing should be 2D or 3D.') - if isinstance(key, tuple) and len(key) == 2: - # Adding a 3rd dimension - key = key + (slice(None, None, None),) - (rows, cols, channels) = key - return Slicer(self, rows, cols, channels) + kwargs = {} + for arg in args: + if isinstance(arg, dict): + kwargs.update(arg) + elif isinstance(arg, (str, OTBObject)): + kwargs.update({self.key_input: arg}) + elif isinstance(arg, list) and is_key_list(self, self.key_input): + kwargs.update({self.key_input: arg}) + return kwargs + + def __set_param(self, key, obj): + """Set one parameter, decide which otb.Application method to use depending on target object.""" + if obj is None or (isinstance(obj, (list, tuple)) and not obj): + self.app.ClearValue(key) + return + if key not in self.parameters_keys: + raise Exception( + f"{self.name}: parameter '{key}' was not recognized. " f"Available keys are {self.parameters_keys}" + ) + # Single-parameter cases + if isinstance(obj, OTBObject): + self.app.ConnectImage(key, obj.app, obj.key_output_image) + elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB + self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0]) + elif key == "ram": # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200 + self.app.SetParameterInt("ram", int(obj)) + elif not isinstance(obj, list): # any other parameters (str, int...) + self.app.SetParameterValue(key, obj) + # Images list + elif is_key_images_list(self, key): + # To enable possible in-memory connections, we go through the list and set the parameters one by one + for inp in obj: + if isinstance(inp, OTBObject): + self.app.ConnectImage(key, inp.app, inp.key_output_image) + elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB + self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0]) + else: # here `input` should be an image filepath + # Append `input` to the list, do not overwrite any previously set element of the image list + self.app.AddParameterStringList(key, inp) + # List of any other types (str, int...) + else: + self.app.SetParameterValue(key, obj) + + def __channels_list_from_slice(self, bands): + """Get list of channels to read values at, from a slice.""" + channels = None + start, stop, step = bands.start, bands.stop, bands.step + if step is None: + step = 1 + if start is not None and stop is not None: + channels = list(range(start, stop, step)) + elif start is not None and stop is None: + channels = list(range(start, self.shape[2], step)) + elif start is None and stop is not None: + channels = list(range(0, stop, step)) + elif start is None and stop is None: + channels = list(range(0, self.shape[2], step)) + return channels + + def __hash__(self): + """Override the default behaviour of the hash function. + + Returns: + self hash + + """ + return id(self) + + def __str__(self): + """Return a nice string representation with object id.""" + return f"<pyotb.App {self.appname} object id {id(self)}>" def __getattr__(self, name): """This method is called when the default attribute access fails. We choose to access the attribute `name` of self.app. - Thus, any method of otbApplication can be used transparently on otbObject objects, + Thus, any method of otbApplication can be used transparently on OTBObject objects, e.g. SetParameterOutputImagePixelType() or ExportImage() work Args: @@ -227,7 +530,45 @@ class otbObject: res = getattr(self.app, name) return res except AttributeError as e: - raise AttributeError(f'{self.name}: could not find attribute `{name}`') from e + raise AttributeError(f"{self.name}: could not find attribute `{name}`") from e + + def __getitem__(self, key): + """Override the default __getitem__ behaviour. + + This function enables 2 things : + - access attributes like that : object['any_attribute'] + - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3] + selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]] + selecting 1000x1000 subset : object[:1000, :1000] + - access pixel value(s) at a specified row, col index + + Args: + key: attribute key + + Returns: + attribute, pixel values or Slicer + + """ + # Accessing string attributes + if isinstance(key, str): + return self.__dict__.get(key) + # Accessing pixel value(s) using Y/X coordinates + if isinstance(key, tuple) and len(key) >= 2: + row, col = key[0], key[1] + if isinstance(row, int) and isinstance(col, int): + if row < 0 or col < 0: + raise ValueError(f"{self.name}: can't read pixel value at negative coordinates ({row}, {col})") + channels = None + if len(key) == 3: + channels = key[2] + return self.read_values_at_coords(row, col, channels) + # Slicing + if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)): + raise ValueError(f'"{key}"cannot be interpreted as valid slicing. Slicing should be 2D or 3D.') + if isinstance(key, tuple) and len(key) == 2: + # Adding a 3rd dimension + key = key + (slice(None, None, None),) + return Slicer(self, *key) def __add__(self, other): """Overrides the default addition and flavours it with BandMathX. @@ -236,12 +577,12 @@ class otbObject: other: the other member of the operation Returns: - self + other + self + other """ if isinstance(other, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation('+', self, other) + return Operation("+", self, other) def __sub__(self, other): """Overrides the default subtraction and flavours it with BandMathX. @@ -250,12 +591,12 @@ class otbObject: other: the other member of the operation Returns: - self - other + self - other """ if isinstance(other, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation('-', self, other) + return Operation("-", self, other) def __mul__(self, other): """Overrides the default subtraction and flavours it with BandMathX. @@ -264,12 +605,12 @@ class otbObject: other: the other member of the operation Returns: - self * other + self * other """ if isinstance(other, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation('*', self, other) + return Operation("*", self, other) def __truediv__(self, other): """Overrides the default subtraction and flavours it with BandMathX. @@ -278,12 +619,12 @@ class otbObject: other: the other member of the operation Returns: - self / other + self / other """ if isinstance(other, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation('/', self, other) + return Operation("/", self, other) def __radd__(self, other): """Overrides the default reverse addition and flavours it with BandMathX. @@ -292,12 +633,12 @@ class otbObject: other: the other member of the operation Returns: - other + self + other + self """ if isinstance(other, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation('+', other, self) + return Operation("+", other, self) def __rsub__(self, other): """Overrides the default subtraction and flavours it with BandMathX. @@ -306,12 +647,12 @@ class otbObject: other: the other member of the operation Returns: - other - self + other - self """ if isinstance(other, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation('-', other, self) + return Operation("-", other, self) def __rmul__(self, other): """Overrides the default multiplication and flavours it with BandMathX. @@ -320,12 +661,12 @@ class otbObject: other: the other member of the operation Returns: - other * self + other * self """ if isinstance(other, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation('*', other, self) + return Operation("*", other, self) def __rtruediv__(self, other): """Overrides the default division and flavours it with BandMathX. @@ -334,12 +675,12 @@ class otbObject: other: the other member of the operation Returns: - other / self + other / self """ if isinstance(other, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation('/', other, self) + return Operation("/", other, self) def __abs__(self): """Overrides the default abs operator and flavours it with BandMathX. @@ -348,7 +689,7 @@ class otbObject: abs(self) """ - return Operation('abs', self) + return Operation("abs", self) def __ge__(self, other): """Overrides the default greater or equal and flavours it with BandMathX. @@ -357,12 +698,12 @@ class otbObject: other: the other member of the operation Returns: - self >= other + self >= other """ if isinstance(other, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return logicalOperation('>=', self, other) + return LogicalOperation(">=", self, other) def __le__(self, other): """Overrides the default less or equal and flavours it with BandMathX. @@ -371,12 +712,12 @@ class otbObject: other: the other member of the operation Returns: - self <= other + self <= other """ if isinstance(other, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return logicalOperation('<=', self, other) + return LogicalOperation("<=", self, other) def __gt__(self, other): """Overrides the default greater operator and flavours it with BandMathX. @@ -385,12 +726,12 @@ class otbObject: other: the other member of the operation Returns: - self > other + self > other """ if isinstance(other, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return logicalOperation('>', self, other) + return LogicalOperation(">", self, other) def __lt__(self, other): """Overrides the default less operator and flavours it with BandMathX. @@ -399,12 +740,12 @@ class otbObject: other: the other member of the operation Returns: - self < other + self < other """ if isinstance(other, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return logicalOperation('<', self, other) + return LogicalOperation("<", self, other) def __eq__(self, other): """Overrides the default eq operator and flavours it with BandMathX. @@ -413,12 +754,12 @@ class otbObject: other: the other member of the operation Returns: - self == other + self == other """ if isinstance(other, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return logicalOperation('==', self, other) + return LogicalOperation("==", self, other) def __ne__(self, other): """Overrides the default different operator and flavours it with BandMathX. @@ -427,12 +768,12 @@ class otbObject: other: the other member of the operation Returns: - self != other + self != other """ if isinstance(other, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return logicalOperation('!=', self, other) + return LogicalOperation("!=", self, other) def __or__(self, other): """Overrides the default or operator and flavours it with BandMathX. @@ -441,12 +782,12 @@ class otbObject: other: the other member of the operation Returns: - self || other + self || other """ if isinstance(other, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return logicalOperation('||', self, other) + return LogicalOperation("||", self, other) def __and__(self, other): """Overrides the default and operator and flavours it with BandMathX. @@ -455,25 +796,16 @@ class otbObject: other: the other member of the operation Returns: - self && other + self && other """ if isinstance(other, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return logicalOperation('&&', self, other) + return LogicalOperation("&&", self, other) # TODO: other operations ? # e.g. __pow__... cf https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types - def __hash__(self): - """Override the default behaviour of the hash function. - - Returns: - self hash - - """ - return id(self) - def __array__(self): """This is called when running np.asarray(pyotb_object). @@ -499,158 +831,43 @@ class otbObject: a pyotb object """ - if method == '__call__': + if method == "__call__": # Converting potential pyotb inputs to arrays arrays = [] image_dic = None for inp in inputs: if isinstance(inp, (float, int, np.ndarray, np.generic)): arrays.append(inp) - elif isinstance(inp, otbObject): - image_dic = inp.app.ExportImage(self.output_param) - array = image_dic['array'] + elif isinstance(inp, OTBObject): + if not inp.exports_dict: + inp.export() + image_dic = inp.exports_dict[inp.key_output_image] + array = image_dic["array"] arrays.append(array) else: - print(type(self)) + logger.debug(type(self)) return NotImplemented # Performing the numpy operation result_array = ufunc(*arrays, **kwargs) result_dic = image_dic - result_dic['array'] = result_array + result_dic["array"] = result_array - # Importing back to OTB - app = App('ExtractROI', frozen=True, image_dic=result_dic) # pass the result_dic just to keep reference + # Importing back to OTB, pass the result_dic just to keep reference + app = OTBObject("ExtractROI", image_dic=result_dic, frozen=True, quiet=True) if result_array.shape[2] == 1: - app.ImportImage('in', result_dic) + app.ImportImage("in", result_dic) else: - app.ImportVectorImage('in', result_dic) + app.ImportVectorImage("in", result_dic) app.execute() return app return NotImplemented - def summarize(self): - """Return a nested dictionary summarizing the otbObject. - Returns: - Nested dictionary summarizing the otbObject +class App(OTBObject): - """ - params = self.parameters - for k, p in params.items(): - # In the following, we replace each parameter which is an otbObject, with its summary. - if isinstance(p, otbObject): # single parameter - params[k] = p.summarize() - elif isinstance(p, list): # parameter list - params[k] = [pi.summarize() if isinstance(pi, otbObject) else pi for pi in p] - - return {"name": self.name, "parameters": params} - - -class App(otbObject): - """Class of an OTB app.""" - def __init__(self, appname, *args, frozen=False, quiet=False, - preserve_dtype=False, image_dic=None, **kwargs): - """Enables to init an OTB application as a oneliner. Handles in-memory connection between apps. - - Args: - appname: name of the app, e.g. 'Smoothing' - *args: used for passing application parameters. Can be : - - dictionary containing key-arguments enumeration. Useful when a key is python-reserved - (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") - - string, App or Output, useful when the user wants to specify the input "in" - - list, useful when the user wants to specify the input list 'il' - frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___ - quiet: whether to print logs of the OTB app - preserve_dtype: propagate the pixel type from inputs to output. If several inputs, the type of an - arbitrary input is considered. If several outputs, all will have the same type. - image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as - the result of app.ExportImage(). Use it when the app takes a numpy array as input. - See this related issue for why it is necessary to keep reference of object: - https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824 - **kwargs: used for passing application parameters. - e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif' - - """ - self.appname = appname - self.frozen = frozen - self.quiet = quiet - self.preserve_dtype = preserve_dtype - self.image_dic = image_dic - if self.quiet: - self.app = otb.Registry.CreateApplicationWithoutLogger(appname) - else: - self.app = otb.Registry.CreateApplication(appname) - self.description = self.app.GetDocLongDescription() - self.output_parameters_keys = self.__get_output_parameters_keys() - if self.output_parameters_keys: - self.output_param = self.output_parameters_keys[0] - - self.parameters = {} - if (args or kwargs): - self.set_parameters(*args, **kwargs) - if not self.frozen: - self.execute() - - def set_parameters(self, *args, **kwargs): - """Set some parameters of the app. - - When useful, e.g. for images list, this function appends the parameters - instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths - - Args: - *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved - (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") - - string, App or Output, useful when the user implicitly wants to set the param "in" - - list, useful when the user implicitly wants to set the param "il" - **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif' - - Raises: - Exception: when the setting of a parameter failed - - """ - parameters = kwargs - parameters.update(self.__parse_args(args)) - # Going through all arguments - for param, obj in parameters.items(): - if param not in self.app.GetParametersKeys(): - raise Exception(f"{self.name}: parameter '{param}' was not recognized. " - f"Available keys are {self.app.GetParametersKeys()}") - # When the parameter expects a list, if needed, change the value to list - if self.__is_key_list(param) and not isinstance(obj, (list, tuple)): - parameters[param] = [obj] - obj = [obj] - logger.warning('%s: argument for parameter "%s" was converted to list', self.name, param) - try: - # This is when we actually call self.app.SetParameter* - self.__set_param(param, obj) - except (RuntimeError, TypeError, ValueError, KeyError) as e: - raise Exception(f"{self.name}: something went wrong before execution " - f"(while setting parameter '{param}' to '{obj}')") from e - # Update _parameters using values from OtbApplication object - otb_params = self.app.GetParameters().items() - otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params} - self.parameters.update({**parameters, **otb_params}) - # Update output images pixel types - if self.preserve_dtype: - self.__propagate_pixel_type() - - def execute(self): - """Execute and write to disk if any output parameter has been set during init.""" - logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters) - try: - self.app.Execute() - except (RuntimeError, FileNotFoundError) as e: - raise Exception(f'{self.name}: error during during app execution') from e - self.frozen = False - logger.debug("%s: execution ended", self.name) - if self.__has_output_param_key(): - logger.debug('%s: flushing data to disk', self.name) - self.app.WriteOutput() - self.__save_objects() - - def find_output(self): + def find_outputs(self): """Find output files on disk using path found in parameters. Returns: @@ -659,8 +876,7 @@ class App(otbObject): """ files = [] missing = [] - outputs = [p for p in self.output_parameters_keys if p in self.parameters] - for param in outputs: + for param in self.outputs: filename = self.parameters[param] # Remove filename extension if '?' in filename: @@ -677,144 +893,11 @@ class App(otbObject): return files - # Private functions - def __get_output_parameters_keys(self): - """Get raster output parameter keys. - - Returns: - output parameters keys - """ - return [param for param in self.app.GetParametersKeys() - if self.app.GetParameterType(param) == otb.ParameterType_OutputImage] - def __has_output_param_key(self): - """Check if App has any output parameter key.""" - if not self.output_param: - return True # apps like ReadImageInfo with no filetype output param still needs to WriteOutput - types = (otb.ParameterType_OutputFilename, otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData) - outfile_params = [param for param in self.app.GetParametersKeys() if self.app.GetParameterType(param) in types] - return any(key in self.parameters for key in outfile_params) - - @staticmethod - def __parse_args(args): - """Gather all input arguments in kwargs dict. - - Returns: - a dictionary with the right keyword depending on the object - - """ - kwargs = {} - for arg in args: - if isinstance(arg, dict): - kwargs.update(arg) - elif isinstance(arg, (str, otbObject)): - kwargs.update({'in': arg}) - elif isinstance(arg, list): - kwargs.update({'il': arg}) - return kwargs - - def __set_param(self, param, obj): - """Set one parameter, decide which otb.Application method to use depending on target object.""" - if obj is not None: - # Single-parameter cases - if isinstance(obj, otbObject): - self.app.ConnectImage(param, obj.app, obj.output_param) - elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB - outparamkey = [param for param in obj.GetParametersKeys() - if obj.GetParameterType(param) == otb.ParameterType_OutputImage][0] - self.app.ConnectImage(param, obj, outparamkey) - elif param == 'ram': # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200 - self.app.SetParameterInt('ram', int(obj)) - elif not isinstance(obj, list): # any other parameters (str, int...) - self.app.SetParameterValue(param, obj) - # Images list - elif self.__is_key_images_list(param): - # To enable possible in-memory connections, we go through the list and set the parameters one by one - for inp in obj: - if isinstance(inp, otbObject): - self.app.ConnectImage(param, inp.app, inp.output_param) - elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB - outparamkey = [param for param in inp.GetParametersKeys() if - inp.GetParameterType(param) == otb.ParameterType_OutputImage][0] - self.app.ConnectImage(param, inp, outparamkey) - else: # here `input` should be an image filepath - # Append `input` to the list, do not overwrite any previously set element of the image list - self.app.AddParameterStringList(param, inp) - # List of any other types (str, int...) - else: - self.app.SetParameterValue(param, obj) - - def __propagate_pixel_type(self): - """Propagate the pixel type from inputs to output. - - For several inputs, or with an image list, the type of the first input is considered. - If several outputs, all outputs will have the same type. - - """ - pixel_type = None - for key, param in self.parameters.items(): - if self.__is_key_input_image(key): - if not param: - continue - if isinstance(param, list): - param = param[0] # first image in "il" - try: - pixel_type = get_pixel_type(param) - type_name = self.app.ConvertPixelTypeToNumpy(pixel_type) - logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name) - for out_key in self.output_parameters_keys: - self.app.SetParameterOutputImagePixelType(out_key, pixel_type) - return - except TypeError: - pass - - logger.warning("%s: could not propagate pixel type from inputs to output, no valid input found", self.name) - - def __save_objects(self): - """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`. - - This is useful when the key contains reserved characters such as a point eg "io.out" - """ - for key in self.app.GetParametersKeys(): - if key == 'parameters': # skip forbidden attribute since it is already used by the App class - continue - value = None - if key in self.output_parameters_keys: # raster outputs - value = Output(self, key) - elif key in self.parameters: # user or default app parameters - value = self.parameters[key] - else: # any other app attribute (e.g. ReadImageInfo results) - try: - value = self.app.GetParameterValue(key) - except RuntimeError: - pass # this is when there is no value for key - if value is not None: - setattr(self, key, value) - - def __is_key_input_image(self, key): - """Check if a key of the App is an input parameter image list.""" - return self.app.GetParameterType(key) in (otb.ParameterType_InputImage, otb.ParameterType_InputImageList) - - def __is_key_list(self, key): - """Check if a key of the App is an input parameter list.""" - return self.app.GetParameterType(key) in (otb.ParameterType_InputImageList, otb.ParameterType_StringList, - otb.ParameterType_InputFilenameList, otb.ParameterType_ListView, - otb.ParameterType_InputVectorDataList) - - def __is_key_images_list(self, key): - """Check if a key of the App is an input parameter image list.""" - return self.app.GetParameterType(key) in (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList) - - # Special methods - def __str__(self): - """Return a nice string representation with object id.""" - return f'<pyotb.App {self.appname} object id {id(self)}>' - - -class Slicer(App): +class Slicer(OTBObject): """Slicer objects i.e. when we call something like raster[:, :, 2] from Python.""" - def __init__(self, x, rows, cols, channels): + def __init__(self, obj, rows, cols, channels): """Create a slicer object, that can be used directly for writing or inside a BandMath. It contains : @@ -822,22 +905,20 @@ class Slicer(App): - in case the user only wants to extract one band, an expression such as "im1b#" Args: - x: input + obj: input rows: slice along Y / Latitude axis cols: slice along X / Longitude axis channels: channels, can be slicing, list or int """ - # Initialize the app that will be used for writing the slicer - self.name = 'Slicer' - - self.output_parameter_key = 'out' - parameters = {'in': x, 'mode': 'extent'} - super().__init__('ExtractROI', parameters, preserve_dtype=True, frozen=True) + super().__init__("ExtractROI", {"in": obj, "mode": "extent"}, quiet=True, frozen=True) + self.name = "Slicer" + self.rows, self.cols = rows, cols + parameters = {} # Channel slicing if channels != slice(None, None, None): # Trigger source app execution if needed - nb_channels = get_nbchannels(x) + nb_channels = get_nbchannels(obj) self.app.Execute() # this is needed by ExtractROI for setting the `cl` parameter # if needed, converting int to list if isinstance(channels, int): @@ -853,90 +934,39 @@ class Slicer(App): elif isinstance(channels, tuple): channels = list(channels) elif not isinstance(channels, list): - raise ValueError(f'Invalid type for channels, should be int, slice or list of bands. : {channels}') + raise ValueError(f"Invalid type for channels, should be int, slice or list of bands. : {channels}") # Change the potential negative index values to reverse index channels = [c if c >= 0 else nb_channels + c for c in channels] - parameters.update({'cl': [f'Channel{i + 1}' for i in channels]}) + parameters.update({"cl": [f"Channel{i + 1}" for i in channels]}) # Spatial slicing spatial_slicing = False - # TODO: handle PixelValue app so that accessing value is possible, e.g. raster[120, 200, 0] # TODO TBD: handle the step value in the slice so that NN undersampling is possible ? e.g. raster[::2, ::2] if rows.start is not None: - parameters.update({'mode.extent.uly': rows.start}) + parameters.update({"mode.extent.uly": rows.start}) spatial_slicing = True if rows.stop is not None and rows.stop != -1: - parameters.update( - {'mode.extent.lry': rows.stop - 1}) # subtract 1 to be compliant with python convention + parameters.update({"mode.extent.lry": rows.stop - 1}) # subtract 1 to respect python convention spatial_slicing = True if cols.start is not None: - parameters.update({'mode.extent.ulx': cols.start}) + parameters.update({"mode.extent.ulx": cols.start}) spatial_slicing = True if cols.stop is not None and cols.stop != -1: - parameters.update( - {'mode.extent.lrx': cols.stop - 1}) # subtract 1 to be compliant with python convention + parameters.update({"mode.extent.lrx": cols.stop - 1}) # subtract 1 to respect python convention spatial_slicing = True + # Execute app - self.set_parameters(**parameters) + self.set_parameters(parameters) self.execute() - # These are some attributes when the user simply wants to extract *one* band to be used in an Operation if not spatial_slicing and isinstance(channels, list) and len(channels) == 1: self.one_band_sliced = channels[0] + 1 # OTB convention: channels start at 1 - self.input = x - - -class Input(App): - """Class for transforming a filepath to pyOTB object.""" - - def __init__(self, filepath): - """Constructor for an Input object. - - Args: - filepath: raster file path - - """ - self.filepath = filepath - super().__init__('ExtractROI', {'in': self.filepath}, preserve_dtype=True) - - def __str__(self): - """Return a nice string representation with input file path.""" - return f'<pyotb.Input object from {self.filepath}>' + self.input = obj + self.propagate_dtype() -class Output(otbObject): - """Class for output of an app.""" - - def __init__(self, app, output_parameter_key): - """Constructor for an Output object. - - Args: - app: The pyotb App - output_parameter_key: Output parameter key - - """ - # Keeping the OTB app and the pyotb app - self.pyotb_app, self.app = app, app.app - self.parameters = self.pyotb_app.parameters - self.output_param = output_parameter_key - self.name = f'Output {output_parameter_key} from {self.app.GetName()}' - - def summarize(self): - """Return the summary of the pipeline that generates the Output object. - - Returns: - Nested dictionary summarizing the pipeline that generates the Output object. - - """ - return self.pyotb_app.summarize() - - def __str__(self): - """Return a nice string representation with object id.""" - return f'<pyotb.Output {self.app.GetName()} object, id {id(self)}>' - - -class Operation(App): +class Operation(OTBObject): """Class for arithmetic/math operations done in Python. Example: @@ -976,9 +1006,7 @@ class Operation(App): self.nb_channels = {} self.fake_exp_bands = [] self.logical_fake_exp_bands = [] - self.create_fake_exp(operator, inputs, nb_bands=nb_bands) - # Transforming images to the adequate im#, e.g. `input1` to "im1" # creating a dictionary that is like {str(input1): 'im1', 'image2.tif': 'im2', ...}. # NB: the keys of the dictionary are strings-only, instead of 'complex' objects, to enable easy serialization @@ -988,20 +1016,18 @@ class Operation(App): for inp in self.inputs: if not isinstance(inp, (int, float)): if str(inp) not in self.im_dic: - self.im_dic[str(inp)] = f'im{self.im_count}' + self.im_dic[str(inp)] = f"im{self.im_count}" mapping_str_to_input[str(inp)] = inp self.im_count += 1 - # getting unique image inputs, in the order im1, im2, im3 ... + # Getting unique image inputs, in the order im1, im2, im3 ... self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)] - self.output_param = 'out' - # Computing the BandMath or BandMathX app self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands) + # Init app self.name = f'Operation exp="{self.exp}"' - - appname = 'BandMath' if len(self.exp_bands) == 1 else 'BandMathX' - super().__init__(appname, il=self.unique_inputs, exp=self.exp) + appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX" + super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True) def create_fake_exp(self, operator, inputs, nb_bands=None): """Create a 'fake' expression. @@ -1019,18 +1045,18 @@ class Operation(App): logger.debug("%s, %s", operator, inputs) # this is when we use the ternary operator with `pyotb.where` function. The output nb of bands is already known - if operator == '?' and nb_bands: + if operator == "?" and nb_bands: pass # For any other operations, the output number of bands is the same as inputs else: - if any(isinstance(inp, Slicer) and hasattr(inp, 'one_band_sliced') for inp in inputs): + if any(isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced") for inp in inputs): nb_bands = 1 else: nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))] # check that all inputs have the same nb of bands if len(nb_bands_list) > 1: if not all(x == nb_bands_list[0] for x in nb_bands_list): - raise Exception('All images do not have the same number of bands') + raise Exception("All images do not have the same number of bands") nb_bands = nb_bands_list[0] # Create a list of fake expressions, each item of the list corresponding to one band @@ -1046,12 +1072,14 @@ class Operation(App): cond_band = 1 else: cond_band = band - fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp(inp, cond_band, - keep_logical=True) + fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp( + inp, cond_band, keep_logical=True + ) # any other input else: - fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp(inp, band, - keep_logical=False) + fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp( + inp, band, keep_logical=False + ) fake_exps.append(fake_exp) # Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates) if i == 0 and corresponding_inputs and nb_channels: @@ -1060,13 +1088,13 @@ class Operation(App): # Generating the fake expression of the whole operation if len(inputs) == 1: # this is only for 'abs' - fake_exp = f'({operator}({fake_exps[0]}))' + fake_exp = f"({operator}({fake_exps[0]}))" elif len(inputs) == 2: # We create here the "fake" expression. For example, for a BandMathX expression such as '2 * im1 + im2', # the false expression stores the expression 2 * str(input1) + str(input2) - fake_exp = f'({fake_exps[0]} {operator} {fake_exps[1]})' - elif len(inputs) == 3 and operator == '?': # this is only for ternary expression - fake_exp = f'({fake_exps[0]} ? {fake_exps[1]} : {fake_exps[2]})' + fake_exp = f"({fake_exps[0]} {operator} {fake_exps[1]})" + elif len(inputs) == 3 and operator == "?": # this is only for ternary expression + fake_exp = f"({fake_exps[0]} ? {fake_exps[1]} : {fake_exps[2]})" self.fake_exp_bands.append(fake_exp) @@ -1091,7 +1119,7 @@ class Operation(App): exp_bands.append(one_band_exp) # Form the final expression (e.g. 'im1b1 + 1; im1b2 + 1') - exp = ';'.join(exp_bands) + exp = ";".join(exp_bands) return exp_bands, exp @@ -1116,8 +1144,8 @@ class Operation(App): """ # Special case for one-band slicer - if isinstance(x, Slicer) and hasattr(x, 'one_band_sliced'): - if keep_logical and isinstance(x.input, logicalOperation): + if isinstance(x, Slicer) and hasattr(x, "one_band_sliced"): + if keep_logical and isinstance(x.input, LogicalOperation): fake_exp = x.input.logical_fake_exp_bands[x.one_band_sliced - 1] inputs = x.input.inputs nb_channels = x.input.nb_channels @@ -1128,11 +1156,11 @@ class Operation(App): nb_channels = x.input.nb_channels else: # Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1') - fake_exp = str(x.input) + f'b{x.one_band_sliced}' + fake_exp = str(x.input) + f"b{x.one_band_sliced}" inputs = [x.input] nb_channels = {x.input: 1} - # For logicalOperation, we save almost the same attributes as an Operation - elif keep_logical and isinstance(x, logicalOperation): + # For LogicalOperation, we save almost the same attributes as an Operation + elif keep_logical and isinstance(x, LogicalOperation): fake_exp = x.logical_fake_exp_bands[band - 1] inputs = x.inputs nb_channels = x.nb_channels @@ -1150,25 +1178,24 @@ class Operation(App): nb_channels = {x: get_nbchannels(x)} inputs = [x] # Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1') - fake_exp = str(x) + f'b{band}' + fake_exp = str(x) + f"b{band}" return fake_exp, inputs, nb_channels def __str__(self): - """Return a nice string representation with object id.""" - return f'<pyotb.Operation `{self.operator}` object, id {id(self)}>' + """Return a nice string representation with operator and object id.""" + return f"<pyotb.Operation `{self.operator}` object, id {id(self)}>" -class logicalOperation(Operation): +class LogicalOperation(Operation): """A specialization of Operation class for boolean logical operations i.e. >, <, >=, <=, ==, !=, `&` and `|`. The only difference is that not only the BandMath expression is saved (e.g. "im1b1 > 0 ? 1 : 0"), but also the logical expression (e.g. "im1b1 > 0") """ - def __init__(self, operator, *inputs, nb_bands=None): - """Constructor for a logicalOperation object. + """Constructor for a LogicalOperation object. Args: operator: string operator (one of >, <, >=, <=, ==, !=, &, |) @@ -1192,39 +1219,89 @@ class logicalOperation(Operation): """ # For any other operations, the output number of bands is the same as inputs - if any(isinstance(inp, Slicer) and hasattr(inp, 'one_band_sliced') for inp in inputs): + if any(isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced") for inp in inputs): nb_bands = 1 else: nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))] # check that all inputs have the same nb of bands if len(nb_bands_list) > 1: if not all(x == nb_bands_list[0] for x in nb_bands_list): - raise Exception('All images do not have the same number of bands') + raise Exception("All images do not have the same number of bands") nb_bands = nb_bands_list[0] # Create a list of fake exp, each item of the list corresponding to one band for i, band in enumerate(range(1, nb_bands + 1)): fake_exps = [] for inp in inputs: - fake_exp, corresponding_inputs, nb_channels = super().create_one_input_fake_exp(inp, band, - keep_logical=True) + fake_exp, corresp_inputs, nb_channels = super().create_one_input_fake_exp(inp, band, keep_logical=True) fake_exps.append(fake_exp) # Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates) - if i == 0 and corresponding_inputs and nb_channels: - self.inputs.extend(corresponding_inputs) + if i == 0 and corresp_inputs and nb_channels: + self.inputs.extend(corresp_inputs) self.nb_channels.update(nb_channels) # We create here the "fake" expression. For example, for a BandMathX expression such as 'im1 > im2', # the logical fake expression stores the expression "str(input1) > str(input2)" - logical_fake_exp = f'({fake_exps[0]} {operator} {fake_exps[1]})' + logical_fake_exp = f"({fake_exps[0]} {operator} {fake_exps[1]})" # We keep the logical expression, useful if later combined with other logical operations self.logical_fake_exp_bands.append(logical_fake_exp) # We create a valid BandMath expression, e.g. "str(input1) > str(input2) ? 1 : 0" - fake_exp = f'({logical_fake_exp} ? 1 : 0)' + fake_exp = f"({logical_fake_exp} ? 1 : 0)" self.fake_exp_bands.append(fake_exp) +class FileIO: + """Base class of an IO file object.""" + # TODO: check file exists, create missing directories, ..? + + +class Input(OTBObject, FileIO): + """Class for transforming a filepath to pyOTB object.""" + + def __init__(self, filepath): + """Default constructor. + + Args: + filepath: the path of an input image + + """ + super().__init__("ExtractROI", {"in": str(filepath)}) + self._name = f"Input from {filepath}" + self.filepath = Path(filepath) + self.propagate_dtype() + + def __str__(self): + """Return a nice string representation with file path.""" + return f"<pyotb.Input object from {self.filepath}>" + + +class Output(FileIO): + """Object that behave like a pointer to a specific application output file.""" + + def __init__(self, source_app, param_key, filepath=None): + """Constructor for an Output object. + + Args: + source_app: The pyotb App to store reference from + param_key: Output parameter key of the target app + filepath: path of the output file (if not in memory) + + """ + self.source_app = source_app + self.param_key = param_key + self.filepath = None + if filepath: + if '?' in filepath: + filepath = filepath.split('?')[0] + self.filepath = Path(filepath) + self.name = f"Output {param_key} from {self.source_app.name}" + + def __str__(self): + """Return a nice string representation with source app name and object id.""" + return f"<pyotb.Output {self.source_app.name} object, id {id(self)}>" + + def get_nbchannels(inp): """Get the nb of bands of input image. @@ -1235,7 +1312,7 @@ def get_nbchannels(inp): number of bands in image """ - if isinstance(inp, otbObject): + if isinstance(inp, OTBObject): nb_channels = inp.shape[-1] else: # Executing the app, without printing its log @@ -1272,8 +1349,8 @@ def get_pixel_type(inp): 'float': 'float', 'double': 'double'} pixel_type = datatype_to_pixeltype[datatype] pixel_type = getattr(otb, f'ImagePixelType_{pixel_type}') - elif isinstance(inp, (otbObject)): - pixel_type = inp.GetParameterOutputImagePixelType(inp.output_param) + elif isinstance(inp, (OTBObject)): + pixel_type = inp.GetParameterOutputImagePixelType(inp.key_output_image) else: raise TypeError(f'Could not get the pixel type. Not supported type: {inp}') @@ -1295,3 +1372,63 @@ def parse_pixel_type(pixel_type): if isinstance(pixel_type, int): return pixel_type raise ValueError(f'Bad pixel type specification ({pixel_type})') + + +def is_key_list(pyotb_app, key): + """Check if a key of the App is an input parameter list.""" + return pyotb_app.app.GetParameterType(key) in ( + otb.ParameterType_InputImageList, + otb.ParameterType_StringList, + otb.ParameterType_InputFilenameList, + otb.ParameterType_ListView, + otb.ParameterType_InputVectorDataList, + ) + + +def is_key_images_list(pyotb_app, key): + """Check if a key of the App is an input parameter image list.""" + return pyotb_app.app.GetParameterType(key) in (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList) + + +def get_out_param_types(pyotb_app): + """Get output parameter data type (raster, vector, file).""" + outfile_types = { + otb.ParameterType_OutputImage: "raster", + otb.ParameterType_OutputVectorData: "vector", + otb.ParameterType_OutputFilename: "file", + } + for k in pyotb_app.parameters_keys: + t = pyotb_app.app.GetParameterType(k) + if t in outfile_types: + yield k, outfile_types[t] + + +def get_out_images_param_keys(app): + """Return every output parameter keys of an OTB app.""" + return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage] + + +def key_input(pyotb_app, file_type): + """Get the first input param key for a specific file type.""" + types = { + "raster": (otb.ParameterType_InputImage, otb.ParameterType_InputImageList), + "vector": (otb.ParameterType_InputVectorData, otb.ParameterType_InputVectorDataList), + "file": (otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList) + } + for key in pyotb_app.parameters_keys: + if pyotb_app.app.GetParameterType(key) in types[file_type]: + return key + return None + + +def key_output(pyotb_app, file_type): + """Get the first output param key for a specific file type.""" + types = { + "raster": otb.ParameterType_OutputImage, + "vector": otb.ParameterType_OutputVectorData, + "file": otb.ParameterType_OutputFilename + } + for key in pyotb_app.parameters_keys: + if pyotb_app.app.GetParameterType(key) == types[file_type]: + return key + return None \ No newline at end of file diff --git a/pyotb/functions.py b/pyotb/functions.py index eee401a..942a760 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -7,7 +7,7 @@ import textwrap import uuid from collections import Counter -from .core import (otbObject, App, Input, Operation, logicalOperation, get_nbchannels) +from .core import (OTBObject, App, Input, Operation, LogicalOperation, get_nbchannels) from .helpers import logger @@ -111,25 +111,25 @@ def all(*inputs): # pylint: disable=redefined-builtin # Checking that all bands of the single image are True if len(inputs) == 1: inp = inputs[0] - if isinstance(inp, logicalOperation): + if isinstance(inp, LogicalOperation): res = inp[:, :, 0] else: res = (inp[:, :, 0] != 0) for band in range(1, inp.shape[-1]): - if isinstance(inp, logicalOperation): + if isinstance(inp, LogicalOperation): res = res & inp[:, :, band] else: res = res & (inp[:, :, band] != 0) # Checking that all images are True else: - if isinstance(inputs[0], logicalOperation): + if isinstance(inputs[0], LogicalOperation): res = inputs[0] else: res = (inputs[0] != 0) for inp in inputs[1:]: - if isinstance(inp, logicalOperation): + if isinstance(inp, LogicalOperation): res = res & inp else: res = res & (inp != 0) @@ -166,25 +166,25 @@ def any(*inputs): # pylint: disable=redefined-builtin # Checking that at least one band of the image is True if len(inputs) == 1: inp = inputs[0] - if isinstance(inp, logicalOperation): + if isinstance(inp, LogicalOperation): res = inp[:, :, 0] else: res = (inp[:, :, 0] != 0) for band in range(1, inp.shape[-1]): - if isinstance(inp, logicalOperation): + if isinstance(inp, LogicalOperation): res = res | inp[:, :, band] else: res = res | (inp[:, :, band] != 0) # Checking that at least one image is True else: - if isinstance(inputs[0], logicalOperation): + if isinstance(inputs[0], LogicalOperation): res = inputs[0] else: res = (inputs[0] != 0) for inp in inputs[1:]: - if isinstance(inp, logicalOperation): + if isinstance(inp, LogicalOperation): res = res | inp else: res = res | (inp != 0) diff --git a/tests/test_numpy.py b/tests/test_numpy.py index 5b1dd04..8780473 100644 --- a/tests/test_numpy.py +++ b/tests/test_numpy.py @@ -31,7 +31,7 @@ def test_convert_to_array(): def test_add_noise_array(): white_noise = np.random.normal(0, 50, size=INPUT.shape) noisy_image = INPUT + white_noise - assert isinstance(noisy_image, pyotb.otbObject) + assert isinstance(noisy_image, pyotb.core.OTBObject) assert noisy_image.shape == INPUT.shape -- GitLab From 4123e0377d6327eafd02e2cdfa694e6b80f4232e Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 5 Jan 2023 19:33:02 +0100 Subject: [PATCH 003/399] ENH: move App class def to apps.py --- pyotb/__init__.py | 1 - pyotb/apps.py | 31 ++++++++++++++++++++++++++++++- pyotb/core.py | 33 ++------------------------------- pyotb/functions.py | 8 ++++---- 4 files changed, 36 insertions(+), 37 deletions(-) diff --git a/pyotb/__init__.py b/pyotb/__init__.py index e3bc23a..6164870 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -9,7 +9,6 @@ otb = find_otb() from .apps import * from .core import ( - App, Input, Output, get_nbchannels, diff --git a/pyotb/apps.py b/pyotb/apps.py index 30e3363..6d0ff91 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -5,8 +5,8 @@ import sys from pathlib import Path import otbApplication as otb -from .core import App from .helpers import logger +from .core import OTBObject def get_available_applications(as_subprocess=False): @@ -61,6 +61,35 @@ def get_available_applications(as_subprocess=False): return app_list +class App(OTBObject): + + def find_outputs(self): + """Find output files on disk using path found in parameters. + + Returns: + list of files found on disk + + """ + files = [] + missing = [] + for param in self.outputs: + filename = self.parameters[param] + # Remove filename extension + if '?' in filename: + filename = filename.split('?')[0] + path = Path(filename) + if path.exists(): + files.append(str(path.absolute())) + else: + missing.append(str(path.absolute())) + if missing: + missing = tuple(missing) + for filename in missing: + logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) + + return files + + class OTBTFApp(App): """Helper for OTBTF.""" @staticmethod diff --git a/pyotb/core.py b/pyotb/core.py index 7992890..4df41df 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -865,35 +865,6 @@ class OTBObject: return NotImplemented -class App(OTBObject): - - def find_outputs(self): - """Find output files on disk using path found in parameters. - - Returns: - list of files found on disk - - """ - files = [] - missing = [] - for param in self.outputs: - filename = self.parameters[param] - # Remove filename extension - if '?' in filename: - filename = filename.split('?')[0] - path = Path(filename) - if path.exists(): - files.append(str(path.absolute())) - else: - missing.append(str(path.absolute())) - if missing: - missing = tuple(missing) - for filename in missing: - logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) - - return files - - class Slicer(OTBObject): """Slicer objects i.e. when we call something like raster[:, :, 2] from Python.""" @@ -1317,7 +1288,7 @@ def get_nbchannels(inp): else: # Executing the app, without printing its log try: - info = App("ReadImageInfo", inp, quiet=True) + info = OTBObject("ReadImageInfo", inp, quiet=True) nb_channels = info.GetParameterInt("numberbands") except Exception as e: # this happens when we pass a str that is not a filepath raise TypeError(f'Could not get the number of channels of `{inp}`. Not a filepath or wrong filepath') from e @@ -1338,7 +1309,7 @@ def get_pixel_type(inp): if isinstance(inp, str): # Executing the app, without printing its log try: - info = App("ReadImageInfo", inp, quiet=True) + info = OTBObject("ReadImageInfo", inp, quiet=True) except Exception as info_err: # this happens when we pass a str that is not a filepath raise TypeError(f'Could not get the pixel type of `{inp}`. Not a filepath or wrong filepath') from info_err datatype = info.GetParameterString("datatype") # which is such as short, float... diff --git a/pyotb/functions.py b/pyotb/functions.py index 942a760..0ead43d 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -7,7 +7,7 @@ import textwrap import uuid from collections import Counter -from .core import (OTBObject, App, Input, Operation, LogicalOperation, get_nbchannels) +from .core import (OTBObject, Input, Operation, LogicalOperation, get_nbchannels) from .helpers import logger @@ -412,7 +412,7 @@ def define_processing_area(*args, window_rule='intersection', pixel_size_rule='m 'mode.extent.ulx': ulx, 'mode.extent.uly': lry, # bug in OTB <= 7.3 : 'mode.extent.lrx': lrx, 'mode.extent.lry': uly, # ULY/LRY are inverted } - new_input = App('ExtractROI', params) + new_input = OTBObject('ExtractROI', params) # TODO: OTB 7.4 fixes this bug, how to handle different versions of OTB? new_inputs.append(new_input) # Potentially update the reference inputs for later resampling @@ -453,7 +453,7 @@ def define_processing_area(*args, window_rule='intersection', pixel_size_rule='m new_inputs = [] for inp in inputs: if metadatas[inp]['GeoTransform'][1] != pixel_size: - superimposed = App('Superimpose', inr=reference_input, inm=inp, interpolator=interpolator) + superimposed = OTBObject('Superimpose', inr=reference_input, inm=inp, interpolator=interpolator) new_inputs.append(superimposed) else: new_inputs.append(inp) @@ -478,7 +478,7 @@ def define_processing_area(*args, window_rule='intersection', pixel_size_rule='m new_inputs = [] for inp in inputs: if image_sizes[inp] != most_common_image_size: - superimposed = App('Superimpose', inr=same_size_images[0], inm=inp, interpolator=interpolator) + superimposed = OTBObject('Superimpose', inr=same_size_images[0], inm=inp, interpolator=interpolator) new_inputs.append(superimposed) else: new_inputs.append(inp) -- GitLab From 6b53f8ae364dea699820a375b360369056b36ec2 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 5 Jan 2023 19:56:34 +0100 Subject: [PATCH 004/399] ENH: update flush() docstring --- pyotb/core.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 4df41df..fabc38b 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -183,13 +183,7 @@ class OTBObject: self.save_objects() def flush(self): - """Flush data to disk, this is when WriteOutput is actually called. - - Args: - parameters: key value pairs like {parameter: filename} - dtypes: optional dict to pass output data types (for rasters) - - """ + """Flush data to disk, this is when WriteOutput is actually called.""" try: logger.debug("%s: flushing data to disk", self.name) self.app.WriteOutput() -- GitLab From e29539ed4f2f4bcf46dae1760a7669adba684908 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 5 Jan 2023 20:47:08 +0100 Subject: [PATCH 005/399] STYLE: linting --- pyotb/apps.py | 4 +++- pyotb/core.py | 13 +++++++++---- pyotb/functions.py | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pyotb/apps.py b/pyotb/apps.py index 6d0ff91..0302ebc 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -62,6 +62,7 @@ def get_available_applications(as_subprocess=False): class App(OTBObject): + """Base class for UI related functions, will be subclassed using app name as class name, see CODE_TEMPLATE.""" def find_outputs(self): """Find output files on disk using path found in parameters. @@ -120,6 +121,7 @@ class OTBTFApp(App): n_sources: number of sources. Default is None (resolves the number of sources based on the content of the dict passed in args, where some 'source' str is found) **kwargs: kwargs + """ self.set_nb_sources(*args, n_sources=n_sources) super().__init__(app_name, *args, **kwargs) @@ -142,4 +144,4 @@ for _app in AVAILABLE_APPLICATIONS: exec(_CODE_TEMPLATE.format(name=_app).replace("(App)", "(OTBTFApp)")) # pylint: disable=exec-used # Default behavior for any OTB application else: - exec(_CODE_TEMPLATE.format(name=_app)) # pylint: disable=exec-used \ No newline at end of file + exec(_CODE_TEMPLATE.format(name=_app)) # pylint: disable=exec-used diff --git a/pyotb/core.py b/pyotb/core.py index fabc38b..7204767 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -24,6 +24,11 @@ class OTBObject: - list, useful when the user wants to specify the input list 'il' frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___ quiet: whether to print logs of the OTB app + image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as + the result of app.ExportImage(). Use it when the app takes a numpy array as input. + See this related issue for why it is necessary to keep reference of object: + https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824 + **kwargs: used for passing application parameters. e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif' @@ -53,12 +58,12 @@ class OTBObject: @property def key_input_image(self): - """Get the name of first output image parameter""" + """Get the name of first input image parameter.""" return key_input(self, "raster") @property def key_output_image(self): - """Get the name of first output image parameter""" + """Get the name of first output image parameter.""" return key_output(self, "raster") @property @@ -117,7 +122,7 @@ class OTBObject: @property def transform(self): - """Get image affine transform, rasterio style (see https://www.perrygeo.com/python-affine-transforms.html) + """Get image affine transform, rasterio style (see https://www.perrygeo.com/python-affine-transforms.html). Returns: transform: (X spacing, X offset, X origin, Y offset, Y spacing, Y origin) @@ -406,7 +411,7 @@ class OTBObject: return array, profile def xy_to_rowcol(self, x, y): - """Find (row, col) index using (x, y) projected coordinates, expect image CRS + """Find (row, col) index using (x, y) projected coordinates - image CRS is exepcted. Args: x: longitude or projected X diff --git a/pyotb/functions.py b/pyotb/functions.py index 0ead43d..669039a 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -352,7 +352,7 @@ def define_processing_area(*args, window_rule='intersection', pixel_size_rule='m for inp in inputs: if isinstance(inp, str): # this is for filepaths metadata = Input(inp).GetImageMetaData('out') - elif isinstance(inp, otbObject): + elif isinstance(inp, OTBObject): metadata = inp.GetImageMetaData(inp.output_param) else: raise TypeError(f"Wrong input : {inp}") -- GitLab From 9988d74385b1636e049f9b93cb4ed2287d257f88 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 5 Jan 2023 20:00:06 +0000 Subject: [PATCH 006/399] STYLE: more linting --- pyotb/core.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 7204767..81e0409 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """This module is the core of pyotb.""" from pathlib import Path +from ast import literal_eval import numpy as np import otbApplication as otb @@ -55,7 +56,7 @@ class OTBObject: def key_input(self): """Get the name of first input parameter, raster > vector > file.""" return self.key_input_image or key_input(self, "vector") or key_input(self, "file") - + @property def key_input_image(self): """Get the name of first input image parameter.""" @@ -92,7 +93,7 @@ class OTBObject: @property def outputs(self): """List of application outputs.""" - return [getattr(self, key) for key in self.out_param_keys if key in self.parameters] + return [getattr(self, key) for key in self.out_param_keys if key in self.parameters] @property def dtype(self): @@ -411,7 +412,7 @@ class OTBObject: return array, profile def xy_to_rowcol(self, x, y): - """Find (row, col) index using (x, y) projected coordinates - image CRS is exepcted. + """Find (row, col) index using (x, y) projected coordinates - image CRS is expected. Args: x: longitude or projected X @@ -1357,7 +1358,9 @@ def is_key_list(pyotb_app, key): def is_key_images_list(pyotb_app, key): """Check if a key of the App is an input parameter image list.""" - return pyotb_app.app.GetParameterType(key) in (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList) + return pyotb_app.app.GetParameterType(key) in ( + otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList + ) def get_out_param_types(pyotb_app): @@ -1401,4 +1404,4 @@ def key_output(pyotb_app, file_type): for key in pyotb_app.parameters_keys: if pyotb_app.app.GetParameterType(key) == types[file_type]: return key - return None \ No newline at end of file + return None -- GitLab From a682b9efdb40093d5ea9d63ccca2a245f5692dea Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 6 Jan 2023 18:06:03 +0100 Subject: [PATCH 007/399] FIX: make sure to call execute *after* propagate_dtype --- pyotb/core.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 81e0409..56a9786 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -926,15 +926,15 @@ class Slicer(OTBObject): if cols.stop is not None and cols.stop != -1: parameters.update({"mode.extent.lrx": cols.stop - 1}) # subtract 1 to respect python convention spatial_slicing = True - - # Execute app - self.set_parameters(parameters) - self.execute() # These are some attributes when the user simply wants to extract *one* band to be used in an Operation if not spatial_slicing and isinstance(channels, list) and len(channels) == 1: self.one_band_sliced = channels[0] + 1 # OTB convention: channels start at 1 self.input = obj + + # Execute app + self.set_parameters(parameters) self.propagate_dtype() + self.execute() class Operation(OTBObject): @@ -1237,10 +1237,11 @@ class Input(OTBObject, FileIO): filepath: the path of an input image """ - super().__init__("ExtractROI", {"in": str(filepath)}) + super().__init__("ExtractROI", {"in": str(filepath)}, frozen=True) self._name = f"Input from {filepath}" self.filepath = Path(filepath) self.propagate_dtype() + self.execute() def __str__(self): """Return a nice string representation with file path.""" -- GitLab From 4ab1fc77df0e90ad9e56a3ec64d5c713a4f8e2de Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 9 Jan 2023 10:48:10 +0100 Subject: [PATCH 008/399] STYLE: exports_dict -> exports_dic to align with image_dic --- pyotb/core.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 56a9786..a6aaa7a 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -38,6 +38,7 @@ class OTBObject: self.appname = appname self.quiet = quiet self.image_dic = image_dic + self.exports_dic = {} if quiet: self.app = otb.Registry.CreateApplicationWithoutLogger(appname) else: @@ -45,7 +46,6 @@ class OTBObject: self.parameters_keys = tuple(self.app.GetParametersKeys()) self.out_param_types = dict(get_out_param_types(self)) self.out_param_keys = tuple(self.out_param_types.keys()) - self.exports_dict = {} if args or kwargs: self.set_parameters(*args, **kwargs) self.frozen = frozen @@ -356,7 +356,7 @@ class OTBObject: return {"name": self.name, "parameters": params} def export(self, key=None): - """Export a specific output image as numpy array and store it in object's exports_dict. + """Export a specific output image as numpy array and store it in object's exports_dic. Args: key: parameter key to export, if None then the default one will be used @@ -367,9 +367,9 @@ class OTBObject: """ if key is None: key = key_output(self, "raster") - if key not in self.exports_dict: - self.exports_dict[key] = self.app.ExportImage(key) - return self.exports_dict[key] + if key not in self.exports_dic: + self.exports_dic[key] = self.app.ExportImage(key) + return self.exports_dic[key] def to_numpy(self, key=None, preserve_dtype=True, copy=False): """Export a pyotb object to numpy array. @@ -839,9 +839,9 @@ class OTBObject: if isinstance(inp, (float, int, np.ndarray, np.generic)): arrays.append(inp) elif isinstance(inp, OTBObject): - if not inp.exports_dict: + if not inp.exports_dic: inp.export() - image_dic = inp.exports_dict[inp.key_output_image] + image_dic = inp.exports_dic[inp.key_output_image] array = image_dic["array"] arrays.append(array) else: -- GitLab From f8e72333096da9e2c8b84971ae7ec895f75080d3 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 9 Jan 2023 15:04:44 +0100 Subject: [PATCH 009/399] ENH: add preserve_dtype to np export function --- pyotb/core.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index a6aaa7a..795e4e7 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -355,7 +355,7 @@ class OTBObject: return {"name": self.name, "parameters": params} - def export(self, key=None): + def export(self, key=None, preserve_dtype=True): """Export a specific output image as numpy array and store it in object's exports_dic. Args: @@ -366,9 +366,11 @@ class OTBObject: """ if key is None: - key = key_output(self, "raster") + key = self.key_output_image if key not in self.exports_dic: self.exports_dic[key] = self.app.ExportImage(key) + if preserve_dtype: + self.exports_dic[key]["array"] = self.exports_dic[key]["array"].astype(self.dtype) return self.exports_dic[key] def to_numpy(self, key=None, preserve_dtype=True, copy=False): @@ -376,7 +378,7 @@ class OTBObject: Args: key: the output parameter name to export as numpy array - preserve_dtype: when set to True, the numpy array is created with the same pixel type as + preserve_dtype: when set to True, the numpy array is converted to the same pixel type as the OTBObject first output. Default is True. copy: whether to copy the output array, default is False required to True if preserve_dtype is False and the source app reference is lost @@ -385,10 +387,8 @@ class OTBObject: a numpy array """ - data = self.export(key) + data = self.export(key, preserve_dtype) array = data["array"] - if preserve_dtype: - return array.astype(self.dtype) if copy: return array.copy() return array -- GitLab From ba47461d86508615e588f2c8a66c61466f48721e Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 9 Jan 2023 15:05:56 +0100 Subject: [PATCH 010/399] CI: enhance existing tests + add new ones --- tests/test_core.py | 63 ++++++++++++++++++++++++++++++++++++------ tests/test_numpy.py | 11 +++++++- tests/test_pipeline.py | 2 +- 3 files changed, 66 insertions(+), 10 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index abefc80..7cbff55 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -8,7 +8,26 @@ FILEPATH = os.environ["TEST_INPUT_IMAGE"] INPUT = pyotb.Input(FILEPATH) -# Basic tests +# Test ExtractROI app's parameters were set for the Input object +def test_parameters(): + assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304) + + +# Test OTB objects properties +def test_name(): + assert INPUT.name == "Input from tests/image.tif" + INPUT.name = "Test input" + assert INPUT.name == "Test input" + + +def test_key_input(): + assert INPUT.key_input == INPUT.key_input_image == "in" + + +def test_key_output(): + assert INPUT.key_output_image == "out" + + def test_dtype(): assert INPUT.dtype == "uint8" @@ -17,9 +36,14 @@ def test_shape(): assert INPUT.shape == (304, 251, 4) +def test_transform(): + assert INPUT.transform == (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0) + + def test_slicer_shape(): extract = INPUT[:50, :60, :3] assert extract.shape == (50, 60, 3) + assert extract.parameters["cl"] == ("Channel1", "Channel2", "Channel3") def test_slicer_preserve_dtype(): @@ -27,10 +51,18 @@ def test_slicer_preserve_dtype(): assert extract.dtype == "uint8" -# More complex tests +# Arithmetic tests def test_operation(): op = INPUT / 255 * 128 - assert op.exp == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)" + assert ( + op.exp + == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)" + ) + assert op.dtype == "float32" + + +def test_func_abs_expression(): + assert abs(INPUT).exp == "(abs(im1b1));(abs(im1b2));(abs(im1b3));(abs(im1b4))" def test_sum_bands(): @@ -43,7 +75,10 @@ def test_binary_mask_where(): # Create binary mask based on several possible values values = [1, 2, 3, 4] res = pyotb.where(pyotb.any(INPUT[:, :, 0] == value for value in values), 255, 0) - assert res.exp == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)" + assert ( + res.exp + == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)" + ) # Apps @@ -64,9 +99,11 @@ def test_app_computeimagestats_sliced(): assert slicer_stats["out.min"] == "[180]" -# NDVI +# BandMath NDVI == RadiometricIndices NDVI ? def test_ndvi_comparison(): - ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (INPUT[:, :, -1] + INPUT[:, :, 0]) + ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / ( + INPUT[:, :, -1] + INPUT[:, :, 0] + ) ndvi_indices = pyotb.RadiometricIndices( {"in": INPUT, "list": "Vegetation:NDVI", "channels.red": 1, "channels.nir": 4} ) @@ -77,7 +114,9 @@ def test_ndvi_comparison(): ndvi_indices.write("/tmp/ndvi_indices.tif", pixel_type="float") assert Path("/tmp/ndvi_indices.tif").exists() - compared = pyotb.CompareImages({"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"}) + compared = pyotb.CompareImages( + {"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"} + ) assert compared.count == 0 assert compared.mse == 0 @@ -85,4 +124,12 @@ def test_ndvi_comparison(): assert thresholded_indices.exp == "((im1b1 >= 0.3) ? 1 : 0)" thresholded_bandmath = pyotb.where(ndvi_bandmath >= 0.3, 1, 0) - assert thresholded_bandmath.exp == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)" + assert ( + thresholded_bandmath.exp + == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)" + ) + + +# XY => RowCol +def test_xy_to_rowcol(): + assert INPUT.xy_to_rowcol(760100, 6946210) == (19, 7) diff --git a/tests/test_numpy.py b/tests/test_numpy.py index 8780473..4240cb9 100644 --- a/tests/test_numpy.py +++ b/tests/test_numpy.py @@ -7,6 +7,14 @@ FILEPATH = os.environ["TEST_INPUT_IMAGE"] INPUT = pyotb.Input(FILEPATH) +def test_export(): + INPUT.export() + assert "out" in INPUT.exports_dic + array = INPUT.exports_dic["out"]["array"] + assert isinstance(array, np.ndarray) + assert array.dtype == "uint8" + + def test_to_numpy(): array = INPUT.to_numpy() assert array.dtype == np.uint8 @@ -42,8 +50,9 @@ def test_to_rasterio(): assert profile["transform"] == (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0) # CRS test requires GDAL python bindings - try: + try: from osgeo import osr + crs = osr.SpatialReference() crs.ImportFromEPSG(2154) dest_crs = osr.SpatialReference() diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index c2d8f0e..3c88153 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -17,7 +17,7 @@ OTBAPPS_BLOCKS = [ ] PYOTB_BLOCKS = [ - lambda inp: 1 + abs(inp) * 2, + lambda inp: 1 / (1 + abs(inp) * 2), lambda inp: inp[:80, 10:60, :], ] PIPELINES_LENGTH = [1, 2, 3] -- GitLab From 5dc70553851882edd1916ef1776d54475e915395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr> Date: Mon, 9 Jan 2023 14:15:19 +0000 Subject: [PATCH 011/399] Apply 7 suggestion(s) to 2 file(s) --- pyotb/apps.py | 6 ++---- pyotb/core.py | 14 ++++++-------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/pyotb/apps.py b/pyotb/apps.py index 0302ebc..2ef6a90 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -79,10 +79,8 @@ class App(OTBObject): if '?' in filename: filename = filename.split('?')[0] path = Path(filename) - if path.exists(): - files.append(str(path.absolute())) - else: - missing.append(str(path.absolute())) + dest = files if path.exists() else missing + dest.append(str(path.absolute())) if missing: missing = tuple(missing) for filename in missing: diff --git a/pyotb/core.py b/pyotb/core.py index 795e4e7..4de1440 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -441,9 +441,7 @@ class OTBObject: for arg in args: if isinstance(arg, dict): kwargs.update(arg) - elif isinstance(arg, (str, OTBObject)): - kwargs.update({self.key_input: arg}) - elif isinstance(arg, list) and is_key_list(self, self.key_input): + elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.key_input): kwargs.update({self.key_input: arg}) return kwargs @@ -453,7 +451,7 @@ class OTBObject: self.app.ClearValue(key) return if key not in self.parameters_keys: - raise Exception( + raise KeyError( f"{self.name}: parameter '{key}' was not recognized. " f"Available keys are {self.parameters_keys}" ) # Single-parameter cases @@ -1027,7 +1025,7 @@ class Operation(OTBObject): # check that all inputs have the same nb of bands if len(nb_bands_list) > 1: if not all(x == nb_bands_list[0] for x in nb_bands_list): - raise Exception("All images do not have the same number of bands") + raise ValueError("All images do not have the same number of bands") nb_bands = nb_bands_list[0] # Create a list of fake expressions, each item of the list corresponding to one band @@ -1127,7 +1125,7 @@ class Operation(OTBObject): nb_channels = x.input.nb_channels else: # Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1') - fake_exp = str(x.input) + f"b{x.one_band_sliced}" + fake_exp = f"{x.input}b{x.one_band_sliced}" inputs = [x.input] nb_channels = {x.input: 1} # For LogicalOperation, we save almost the same attributes as an Operation @@ -1149,7 +1147,7 @@ class Operation(OTBObject): nb_channels = {x: get_nbchannels(x)} inputs = [x] # Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1') - fake_exp = str(x) + f"b{band}" + fake_exp = f"{x}b{band}" return fake_exp, inputs, nb_channels @@ -1197,7 +1195,7 @@ class LogicalOperation(Operation): # check that all inputs have the same nb of bands if len(nb_bands_list) > 1: if not all(x == nb_bands_list[0] for x in nb_bands_list): - raise Exception("All images do not have the same number of bands") + raise ValueError("All images do not have the same number of bands") nb_bands = nb_bands_list[0] # Create a list of fake exp, each item of the list corresponding to one band -- GitLab From 33681e964aa0ea59229e5f5e3e9556c555253ba0 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 9 Jan 2023 15:31:23 +0100 Subject: [PATCH 012/399] CI: temporarily mark ndvi_test as expected failure --- tests/test_core.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index 7cbff55..5539245 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -3,6 +3,8 @@ import pyotb from ast import literal_eval from pathlib import Path +import pytest + FILEPATH = os.environ["TEST_INPUT_IMAGE"] INPUT = pyotb.Input(FILEPATH) @@ -100,6 +102,7 @@ def test_app_computeimagestats_sliced(): # BandMath NDVI == RadiometricIndices NDVI ? +@pytest.mark.xfail(reason="Regression in OTB 8.2, waiting for Rémi's patch to be merged.") def test_ndvi_comparison(): ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / ( INPUT[:, :, -1] + INPUT[:, :, 0] -- GitLab From a1eae249bc0a50732825b00b3d8b90ab0486a6b2 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 10 Jan 2023 15:56:25 +0100 Subject: [PATCH 013/399] ENH: better Input and Output classes, check path before exec --- pyotb/core.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 4de1440..4cd7ae0 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -172,8 +172,9 @@ class OTBObject: # Update _parameters using values from OtbApplication object otb_params = self.app.GetParameters().items() otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params} - # Save parameters keys, and values as object attributes + # Update param dict and save values as object attributes self.parameters.update({**parameters, **otb_params}) + self.save_objects() def execute(self): """Execute and write to disk if any output parameter has been set during init.""" @@ -272,7 +273,6 @@ class OTBObject: self.set_parameters({key: output_filename}) self.flush() - self.save_objects() def propagate_dtype(self, target_key=None, dtype=None): """Propagate a pixel type from main input to every outputs, or to a target output key only. @@ -1220,12 +1220,7 @@ class LogicalOperation(Operation): self.fake_exp_bands.append(fake_exp) -class FileIO: - """Base class of an IO file object.""" - # TODO: check file exists, create missing directories, ..? - - -class Input(OTBObject, FileIO): +class Input(OTBObject): """Class for transforming a filepath to pyOTB object.""" def __init__(self, filepath): @@ -1235,9 +1230,11 @@ class Input(OTBObject, FileIO): filepath: the path of an input image """ + self.filepath = Path(filepath) + if not self.filepath.exists(): + raise FileNotFoundError(filepath) super().__init__("ExtractROI", {"in": str(filepath)}, frozen=True) self._name = f"Input from {filepath}" - self.filepath = Path(filepath) self.propagate_dtype() self.execute() @@ -1246,10 +1243,10 @@ class Input(OTBObject, FileIO): return f"<pyotb.Input object from {self.filepath}>" -class Output(FileIO): +class Output: """Object that behave like a pointer to a specific application output file.""" - def __init__(self, source_app, param_key, filepath=None): + def __init__(self, source_app, param_key, filepath=None, mkdir=True): """Constructor for an Output object. Args: @@ -1261,12 +1258,18 @@ class Output(FileIO): self.source_app = source_app self.param_key = param_key self.filepath = None - if filepath: + if filepath is not None: if '?' in filepath: filepath = filepath.split('?')[0] self.filepath = Path(filepath) + if mkdir: + self.make_parent_dirs() self.name = f"Output {param_key} from {self.source_app.name}" + def make_parent_dirs(self): + if not self.filepath.parent.exists(): + self.filepath.parent.mkdir(parents=True) + def __str__(self): """Return a nice string representation with source app name and object id.""" return f"<pyotb.Output {self.source_app.name} object, id {id(self)}>" -- GitLab From 959deab9fdab2acbcdefd96e24c6f51e47f852c7 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 10 Jan 2023 15:56:43 +0100 Subject: [PATCH 014/399] ENH: more tests for the core module --- tests/test_core.py | 76 ++++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 5539245..7d231bb 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -10,12 +10,29 @@ FILEPATH = os.environ["TEST_INPUT_IMAGE"] INPUT = pyotb.Input(FILEPATH) -# Test ExtractROI app's parameters were set for the Input object +# Input settings def test_parameters(): assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304) -# Test OTB objects properties +# Catch errors before exec +def test_missing_input_file(): + with pytest.raises(FileNotFoundError): + pyotb.Input("missing_file.tif") + + +def test_wrong_key(): + with pytest.raises(KeyError): + app = pyotb.BandMath(INPUT, expression="im1b1") + + +# Create dir before write +def test_write(): + INPUT.write("/tmp/missing_dir/test_write.tif") + assert INPUT.out.filepath.exists() + + +# OTBObject's properties def test_name(): assert INPUT.name == "Input from tests/image.tif" INPUT.name = "Test input" @@ -53,13 +70,10 @@ def test_slicer_preserve_dtype(): assert extract.dtype == "uint8" -# Arithmetic tests +# Arithmetics def test_operation(): op = INPUT / 255 * 128 - assert ( - op.exp - == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)" - ) + assert op.exp == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)" assert op.dtype == "float32" @@ -68,7 +82,6 @@ def test_func_abs_expression(): def test_sum_bands(): - # Sum of bands summed = sum(INPUT[:, :, b] for b in range(INPUT.shape[-1])) assert summed.exp == "((((0 + im1b1) + im1b2) + im1b3) + im1b4)" @@ -77,17 +90,13 @@ def test_binary_mask_where(): # Create binary mask based on several possible values values = [1, 2, 3, 4] res = pyotb.where(pyotb.any(INPUT[:, :, 0] == value for value in values), 255, 0) - assert ( - res.exp - == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)" - ) + assert res.exp == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)" # Apps def test_app_readimageinfo(): info = pyotb.ReadImageInfo(INPUT, quiet=True) - assert info.sizex == 251 - assert info.sizey == 304 + assert (info.sizex, info.sizey) == (251, 304) assert info["numberbands"] == info.numberbands == 4 @@ -101,38 +110,33 @@ def test_app_computeimagestats_sliced(): assert slicer_stats["out.min"] == "[180]" +def test_read_values_at_coords(): + assert INPUT[0, 0, 0] == 180 + assert INPUT[10, 20, :] == [196, 188, 172, 255] + + +# XY => RowCol +def test_xy_to_rowcol(): + assert INPUT.xy_to_rowcol(760100, 6946210) == (19, 7) + + # BandMath NDVI == RadiometricIndices NDVI ? @pytest.mark.xfail(reason="Regression in OTB 8.2, waiting for Rémi's patch to be merged.") def test_ndvi_comparison(): - ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / ( - INPUT[:, :, -1] + INPUT[:, :, 0] - ) - ndvi_indices = pyotb.RadiometricIndices( - {"in": INPUT, "list": "Vegetation:NDVI", "channels.red": 1, "channels.nir": 4} - ) + ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (INPUT[:, :, -1] + INPUT[:, :, 0]) + ndvi_indices = pyotb.RadiometricIndices(INPUT, {"list": "Vegetation:NDVI", "channels.red": 1, "channels.nir": 4}) assert ndvi_bandmath.exp == "((im1b4 - im1b1) / (im1b4 + im1b1))" ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", pixel_type="float") - assert Path("/tmp/ndvi_bandmath.tif").exists() + assert ndvi_bandmath.out.filepath.exists() ndvi_indices.write("/tmp/ndvi_indices.tif", pixel_type="float") - assert Path("/tmp/ndvi_indices.tif").exists() + assert ndvi_indices.out.filepath.exists() - compared = pyotb.CompareImages( - {"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"} - ) - assert compared.count == 0 - assert compared.mse == 0 + compared = pyotb.CompareImages({"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"}) + assert (compared.count, compared.mse) == (0, 0) thresholded_indices = pyotb.where(ndvi_indices >= 0.3, 1, 0) assert thresholded_indices.exp == "((im1b1 >= 0.3) ? 1 : 0)" thresholded_bandmath = pyotb.where(ndvi_bandmath >= 0.3, 1, 0) - assert ( - thresholded_bandmath.exp - == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)" - ) - - -# XY => RowCol -def test_xy_to_rowcol(): - assert INPUT.xy_to_rowcol(760100, 6946210) == (19, 7) + assert thresholded_bandmath.exp == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)" -- GitLab From 07bda7ce7606834a7a98fbd3c74977ecf1cf424f Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 10 Jan 2023 16:24:06 +0100 Subject: [PATCH 015/399] CI: linter and test order --- pyotb/core.py | 8 +++++++- tests/test_core.py | 16 ++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 4cd7ae0..94b26d8 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1253,12 +1253,13 @@ class Output: source_app: The pyotb App to store reference from param_key: Output parameter key of the target app filepath: path of the output file (if not in memory) + mkdir: create missing parent directories """ self.source_app = source_app self.param_key = param_key self.filepath = None - if filepath is not None: + if filepath: if '?' in filepath: filepath = filepath.split('?')[0] self.filepath = Path(filepath) @@ -1266,7 +1267,12 @@ class Output: self.make_parent_dirs() self.name = f"Output {param_key} from {self.source_app.name}" + def exists(self): + """Check file exist.""" + return self.filepath.exists() + def make_parent_dirs(self): + """Create missing parent directories.""" if not self.filepath.parent.exists(): self.filepath.parent.mkdir(parents=True) diff --git a/tests/test_core.py b/tests/test_core.py index 7d231bb..128e75f 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -23,13 +23,7 @@ def test_missing_input_file(): def test_wrong_key(): with pytest.raises(KeyError): - app = pyotb.BandMath(INPUT, expression="im1b1") - - -# Create dir before write -def test_write(): - INPUT.write("/tmp/missing_dir/test_write.tif") - assert INPUT.out.filepath.exists() + pyotb.BandMath(INPUT, expression="im1b1") # OTBObject's properties @@ -70,7 +64,7 @@ def test_slicer_preserve_dtype(): assert extract.dtype == "uint8" -# Arithmetics +# Arithmetic def test_operation(): op = INPUT / 255 * 128 assert op.exp == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)" @@ -120,6 +114,12 @@ def test_xy_to_rowcol(): assert INPUT.xy_to_rowcol(760100, 6946210) == (19, 7) +# Create dir before write +def test_write(): + INPUT.write("/tmp/missing_dir/test_write.tif") + assert INPUT.out.filepath.exists() + + # BandMath NDVI == RadiometricIndices NDVI ? @pytest.mark.xfail(reason="Regression in OTB 8.2, waiting for Rémi's patch to be merged.") def test_ndvi_comparison(): -- GitLab From edc7c6047ca225c1977e9986b3cbafc31f33d034 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 10 Jan 2023 16:31:02 +0100 Subject: [PATCH 016/399] ENH: use new Output helper functions --- pyotb/apps.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/pyotb/apps.py b/pyotb/apps.py index 2ef6a90..72b3942 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -73,20 +73,13 @@ class App(OTBObject): """ files = [] missing = [] - for param in self.outputs: - filename = self.parameters[param] - # Remove filename extension - if '?' in filename: - filename = filename.split('?')[0] - path = Path(filename) - dest = files if path.exists() else missing - dest.append(str(path.absolute())) - if missing: - missing = tuple(missing) - for filename in missing: - logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) - - return files + for out in self.outputs: + dest = files if out.exists() else missing + dest.append(str(out.filepath.absolute())) + for filename in missing: + logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) + + return tuple(files) class OTBTFApp(App): @@ -136,8 +129,7 @@ class {name}(App): """ for _app in AVAILABLE_APPLICATIONS: - # Customize the behavior for some OTBTF applications. The user doesn't need to set the env variable - # `OTB_TF_NSOURCES`, it is handled in pyotb + # Customize the behavior for some OTBTF applications. `OTB_TF_NSOURCES` is now handled by pyotb if _app in ("PatchesExtraction", "TensorflowModelTrain", "TensorflowModelServe"): exec(_CODE_TEMPLATE.format(name=_app).replace("(App)", "(OTBTFApp)")) # pylint: disable=exec-used # Default behavior for any OTB application -- GitLab From e93a9db047a2f8934e482aeccbccb48167b41cad Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 10 Jan 2023 17:11:24 +0100 Subject: [PATCH 017/399] ENH: move UI code to App def --- pyotb/apps.py | 34 ++++++++++++++++++ pyotb/core.py | 88 ++++++++++++++++------------------------------ tests/test_core.py | 4 --- 3 files changed, 64 insertions(+), 62 deletions(-) diff --git a/pyotb/apps.py b/pyotb/apps.py index 72b3942..bb15f1c 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -63,6 +63,40 @@ def get_available_applications(as_subprocess=False): class App(OTBObject): """Base class for UI related functions, will be subclassed using app name as class name, see CODE_TEMPLATE.""" + _name = "" + + def __init__(self, *args, **kwargs): + """Default App constructor, adds UI specific attributes and functions.""" + super().__init__(*args, **kwargs) + self.description = self.app.GetDocLongDescription() + + @property + def name(self): + """Application name that will be printed in logs. + + Returns: + user's defined name or appname + + """ + return self._name or self.appname + + @name.setter + def name(self, name): + """Set custom name. + + Args: + name: new name + + """ + if isinstance(name, str): + self._name = name + else: + raise TypeError(f"{self.name}: bad type ({type(name)}) for application name, only str is allowed") + + @property + def outputs(self): + """List of application outputs.""" + return [getattr(self, key) for key in self.out_param_keys if key in self.parameters] def find_outputs(self): """Find output files on disk using path found in parameters. diff --git a/pyotb/core.py b/pyotb/core.py index 94b26d8..0943996 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -11,7 +11,6 @@ from .helpers import logger class OTBObject: """Base class that gathers common operations for any OTB application.""" - _name = "" def __init__(self, appname, *args, frozen=False, quiet=False, image_dic=None, **kwargs): """Common constructor for OTB applications. Handles in-memory connection between apps. @@ -35,7 +34,7 @@ class OTBObject: """ self.parameters = {} - self.appname = appname + self.name = self.appname = appname self.quiet = quiet self.image_dic = image_dic self.exports_dic = {} @@ -46,6 +45,7 @@ class OTBObject: self.parameters_keys = tuple(self.app.GetParametersKeys()) self.out_param_types = dict(get_out_param_types(self)) self.out_param_keys = tuple(self.out_param_types.keys()) + if args or kwargs: self.set_parameters(*args, **kwargs) self.frozen = frozen @@ -67,34 +67,6 @@ class OTBObject: """Get the name of first output image parameter.""" return key_output(self, "raster") - @property - def name(self): - """Application name that will be printed in logs. - - Returns: - user's defined name or appname - - """ - return self._name or self.appname - - @name.setter - def name(self, name): - """Set custom name. - - Args: - name: new name - - """ - if isinstance(name, str): - self._name = name - else: - raise TypeError(f"{self.name}: bad type ({type(name)}) for application name, only str is allowed") - - @property - def outputs(self): - """List of application outputs.""" - return [getattr(self, key) for key in self.out_param_keys if key in self.parameters] - @property def dtype(self): """Expose the pixel type of an output image using numpy convention. @@ -176,6 +148,26 @@ class OTBObject: self.parameters.update({**parameters, **otb_params}) self.save_objects() + def save_objects(self): + """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`. + + This is useful when the key contains reserved characters such as a point eg "io.out" + """ + for key in self.parameters_keys: + if key in dir(self.__class__): + continue # skip forbidden attribute since it is already used by the class + value = self.parameters.get(key) # basic parameters + if value is None: + try: + value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results) + except RuntimeError: + continue # this is when there is no value for key + # Convert output param path to Output object + if key in self.out_param_keys: + value = Output(self, key, value) + # Save attribute + setattr(self, key, value) + def execute(self): """Execute and write to disk if any output parameter has been set during init.""" logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters) @@ -185,9 +177,9 @@ class OTBObject: raise Exception(f"{self.name}: error during during app execution") from e self.frozen = False logger.debug("%s: execution ended", self.name) + self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics if any(key in self.parameters for key in self.out_param_keys): - self.flush() - self.save_objects() + self.flush() # auto flush if any output param was provided during app init def flush(self): """Flush data to disk, this is when WriteOutput is actually called.""" @@ -198,26 +190,6 @@ class OTBObject: logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name) self.app.ExecuteAndWriteOutput() - def save_objects(self): - """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`. - - This is useful when the key contains reserved characters such as a point eg "io.out" - """ - for key in self.parameters_keys: - if key in dir(OTBObject): - continue # skip forbidden attribute since it is already used by the class - value = self.parameters.get(key) # basic parameters - if value is None: - try: - value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results) - except RuntimeError: - continue # this is when there is no value for key - # Convert output param path to Output object - if key in self.out_param_keys: - value = Output(self, key, value) - # Save attribute - setattr(self, key, value) - def write(self, *args, filename_extension="", pixel_type=None, preserve_dtype=False, **kwargs): """Set output pixel type and write the output raster files. @@ -328,7 +300,7 @@ class OTBObject: elif isinstance(bands, slice): channels = self.__channels_list_from_slice(bands) elif not isinstance(bands, list): - raise TypeError(f"{self.name}: type '{bands}' cannot be interpreted as a valid slicing") + raise TypeError(f"{self.name}: type '{type(bands)}' cannot be interpreted as a valid slicing") if channels: app.app.Execute() app.set_parameters({"cl": [f"Channel{n+1}" for n in channels]}) @@ -422,9 +394,8 @@ class OTBObject: pixel index: (row, col) """ spacing_x, _, origin_x, _, spacing_y, origin_y = self.transform - col = int((x - origin_x) / spacing_x) - row = int((origin_y - y) / spacing_y) - return (row, col) + row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x + return (int(row), int(col)) # Private functions def __parse_args(self, args): @@ -494,6 +465,7 @@ class OTBObject: channels = list(range(0, self.shape[2], step)) return channels + # Special functions def __hash__(self): """Override the default behaviour of the hash function. @@ -994,9 +966,9 @@ class Operation(OTBObject): # Computing the BandMath or BandMathX app self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands) # Init app - self.name = f'Operation exp="{self.exp}"' appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX" super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True) + self.name = f'Operation exp="{self.exp}"' def create_fake_exp(self, operator, inputs, nb_bands=None): """Create a 'fake' expression. @@ -1234,7 +1206,7 @@ class Input(OTBObject): if not self.filepath.exists(): raise FileNotFoundError(filepath) super().__init__("ExtractROI", {"in": str(filepath)}, frozen=True) - self._name = f"Input from {filepath}" + self.name = f"Input from {filepath}" self.propagate_dtype() self.execute() diff --git a/tests/test_core.py b/tests/test_core.py index 128e75f..bd59d70 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -27,10 +27,6 @@ def test_wrong_key(): # OTBObject's properties -def test_name(): - assert INPUT.name == "Input from tests/image.tif" - INPUT.name = "Test input" - assert INPUT.name == "Test input" def test_key_input(): -- GitLab From d0125cda7de9e1aed0dfc43adbcfc121a12b6500 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 10 Jan 2023 17:39:25 +0100 Subject: [PATCH 018/399] FIX: tests for xy_to_rowcol and read_values_at_coords --- pyotb/core.py | 6 +++--- tests/test_core.py | 8 ++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 0943996..e3b9254 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -278,12 +278,12 @@ class OTBObject: # Set output pixel type self.app.SetParameterOutputImagePixelType(key, dtype) - def read_values_at_coords(self, col, row, bands=None): + def read_values_at_coords(self, row, col, bands=None): """Get pixel value(s) at a given YX coordinates. Args: - col: index along X / longitude axis row: index along Y / latitude axis + col: index along X / longitude axis bands: band number, list or slice to fetch values from Returns: @@ -395,7 +395,7 @@ class OTBObject: """ spacing_x, _, origin_x, _, spacing_y, origin_y = self.transform row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x - return (int(row), int(col)) + return (abs(int(row)), int(col)) # Private functions def __parse_args(self, args): diff --git a/tests/test_core.py b/tests/test_core.py index bd59d70..10d8ac5 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -102,12 +102,16 @@ def test_app_computeimagestats_sliced(): def test_read_values_at_coords(): assert INPUT[0, 0, 0] == 180 - assert INPUT[10, 20, :] == [196, 188, 172, 255] + assert INPUT[10, 20, :] == [207, 192, 172, 255] # XY => RowCol def test_xy_to_rowcol(): - assert INPUT.xy_to_rowcol(760100, 6946210) == (19, 7) + assert INPUT.xy_to_rowcol(760101, 6945977) == (19, 7) + + +def test_pixel_coords_numpy_equals_otb(): + assert INPUT[19,7] == list(INPUT.to_numpy()[19,7]) # Create dir before write -- GitLab From 663963109aeefe9cca01c6cb289bc078b950e6af Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 10 Jan 2023 18:04:54 +0100 Subject: [PATCH 019/399] ENH: prevent access image properties of app without raster output --- pyotb/core.py | 17 ++++++++++------- tests/test_core.py | 8 ++++++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index e3b9254..5d2ed01 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -55,7 +55,7 @@ class OTBObject: @property def key_input(self): """Get the name of first input parameter, raster > vector > file.""" - return self.key_input_image or key_input(self, "vector") or key_input(self, "file") + return key_input(self, "raster") or key_input(self, "vector") or key_input(self, "file") @property def key_input_image(self): @@ -75,11 +75,10 @@ class OTBObject: dtype: pixel type of the output image """ - try: - enum = self.app.GetParameterOutputImagePixelType(self.key_output_image) - return self.app.ConvertPixelTypeToNumpy(enum) - except RuntimeError: - return None + if not self.key_output_image: + raise TypeError(f"{self.name}: this application has no raster output") + enum = self.app.GetParameterOutputImagePixelType(self.key_output_image) + return self.app.ConvertPixelTypeToNumpy(enum) @property def shape(self): @@ -89,6 +88,8 @@ class OTBObject: shape: (height, width, bands) """ + if not self.key_output_image: + raise TypeError(f"{self.name}: this application has no raster output") width, height = self.app.GetImageSize(self.key_output_image) bands = self.app.GetImageNbBands(self.key_output_image) return (height, width, bands) @@ -100,6 +101,8 @@ class OTBObject: Returns: transform: (X spacing, X offset, X origin, Y offset, Y spacing, Y origin) """ + if not self.key_output_image: + raise TypeError(f"{self.name}: this application has no raster output") spacing_x, spacing_y = self.app.GetImageSpacing(self.key_output_image) origin_x, origin_y = self.app.GetImageOrigin(self.key_output_image) # Shift image origin since OTB is giving coordinates of pixel center instead of corners @@ -1303,7 +1306,7 @@ def get_pixel_type(inp): elif isinstance(inp, (OTBObject)): pixel_type = inp.GetParameterOutputImagePixelType(inp.key_output_image) else: - raise TypeError(f'Could not get the pixel type. Not supported type: {inp}') + raise TypeError(f'Could not get the pixel type of {type(inp)} object {inp}') return pixel_type diff --git a/tests/test_core.py b/tests/test_core.py index 10d8ac5..adf74f3 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -27,8 +27,6 @@ def test_wrong_key(): # OTBObject's properties - - def test_key_input(): assert INPUT.key_input == INPUT.key_input_image == "in" @@ -49,6 +47,12 @@ def test_transform(): assert INPUT.transform == (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0) +def test_nonraster_property(): + with pytest.raises(TypeError): + pyotb.ReadImageInfo(INPUT).dtype + + +# Slicer def test_slicer_shape(): extract = INPUT[:50, :60, :3] assert extract.shape == (50, 60, 3) -- GitLab From 0801bdef187174bd1c4c239bc1e610352a81f08e Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 10 Jan 2023 18:23:33 +0100 Subject: [PATCH 020/399] ENH: use function channels_list_from_slice in Slicer --- pyotb/core.py | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 5d2ed01..a913512 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -301,7 +301,7 @@ class OTBObject: bands = self.shape[2] + bands channels = [bands] elif isinstance(bands, slice): - channels = self.__channels_list_from_slice(bands) + channels = self.channels_list_from_slice(bands) elif not isinstance(bands, list): raise TypeError(f"{self.name}: type '{type(bands)}' cannot be interpreted as a valid slicing") if channels: @@ -313,6 +313,22 @@ class OTBObject: return data[0] return data + def channels_list_from_slice(self, bands): + """Get list of channels to read values at, from a slice.""" + channels = None + start, stop, step = bands.start, bands.stop, bands.step + if step is None: + step = 1 + if start is not None and stop is not None: + channels = list(range(start, stop, step)) + elif start is not None and stop is None: + channels = list(range(start, self.shape[2], step)) + elif start is None and stop is not None: + channels = list(range(0, stop, step)) + elif start is None and stop is None: + channels = list(range(0, self.shape[2], step)) + return channels + def summarize(self): """Serialize an object and its pipeline into a dictionary. @@ -452,22 +468,6 @@ class OTBObject: else: self.app.SetParameterValue(key, obj) - def __channels_list_from_slice(self, bands): - """Get list of channels to read values at, from a slice.""" - channels = None - start, stop, step = bands.start, bands.stop, bands.step - if step is None: - step = 1 - if start is not None and stop is not None: - channels = list(range(start, stop, step)) - elif start is not None and stop is None: - channels = list(range(start, self.shape[2], step)) - elif start is None and stop is not None: - channels = list(range(0, stop, step)) - elif start is None and stop is None: - channels = list(range(0, self.shape[2], step)) - return channels - # Special functions def __hash__(self): """Override the default behaviour of the hash function. @@ -869,12 +869,7 @@ class Slicer(OTBObject): channels = [channels] # if needed, converting slice to list elif isinstance(channels, slice): - channels_start = channels.start if channels.start is not None else 0 - channels_start = channels_start if channels_start >= 0 else nb_channels + channels_start - channels_end = channels.stop if channels.stop is not None else nb_channels - channels_end = channels_end if channels_end >= 0 else nb_channels + channels_end - channels_step = channels.step if channels.step is not None else 1 - channels = range(channels_start, channels_end, channels_step) + channels = self.channels_list_from_slice(channels) elif isinstance(channels, tuple): channels = list(channels) elif not isinstance(channels, list): -- GitLab From aa810bec48a616b89d31a64099cfcae5b795137f Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 10 Jan 2023 18:42:35 +0100 Subject: [PATCH 021/399] FIX: add missing negative band index to channels_list_from_slice --- pyotb/core.py | 22 +++++++++++----------- tests/test_core.py | 4 ++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index a913512..ff7b6ef 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -315,19 +315,19 @@ class OTBObject: def channels_list_from_slice(self, bands): """Get list of channels to read values at, from a slice.""" - channels = None + channels, nb_channels = None, self.shape[2] start, stop, step = bands.start, bands.stop, bands.step - if step is None: - step = 1 + start = nb_channels + start if isinstance(start, int) and start < 0 else start + stop = nb_channels + stop if isinstance(stop, int) and stop < 0 else stop + step = 1 if step is None else step if start is not None and stop is not None: - channels = list(range(start, stop, step)) - elif start is not None and stop is None: - channels = list(range(start, self.shape[2], step)) - elif start is None and stop is not None: - channels = list(range(0, stop, step)) - elif start is None and stop is None: - channels = list(range(0, self.shape[2], step)) - return channels + return list(range(start, stop, step)) + if start is not None and stop is None: + return list(range(start, nb_channels, step)) + if start is None and stop is not None: + return list(range(0, stop, step)) + if start is None and stop is None: + return list(range(0, nb_channels, step)) def summarize(self): """Serialize an object and its pipeline into a dictionary. diff --git a/tests/test_core.py b/tests/test_core.py index adf74f3..2d99bd2 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -64,6 +64,10 @@ def test_slicer_preserve_dtype(): assert extract.dtype == "uint8" +def test_slicer_negative_band_index(): + assert INPUT[:50, :60, :-2].shape == (50, 60, 2) + + # Arithmetic def test_operation(): op = INPUT / 255 * 128 -- GitLab From 7bb9aad4c13aa895285bfc362826986dcfa63023 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 10 Jan 2023 18:50:46 +0100 Subject: [PATCH 022/399] STYLE: linting --- pyotb/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index ff7b6ef..65a05f6 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -315,7 +315,7 @@ class OTBObject: def channels_list_from_slice(self, bands): """Get list of channels to read values at, from a slice.""" - channels, nb_channels = None, self.shape[2] + nb_channels = self.shape[2] start, stop, step = bands.start, bands.stop, bands.step start = nb_channels + start if isinstance(start, int) and start < 0 else start stop = nb_channels + stop if isinstance(stop, int) and stop < 0 else stop @@ -328,6 +328,7 @@ class OTBObject: return list(range(0, stop, step)) if start is None and stop is None: return list(range(0, nb_channels, step)) + raise ValueError(f"{self.name}: '{bands}' cannot be interpreted as valid slicing.") def summarize(self): """Serialize an object and its pipeline into a dictionary. -- GitLab From 5d4be4d77a3bef9c2d4c1a0953e6d402fa64d21e Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 11 Jan 2023 11:58:17 +0100 Subject: [PATCH 023/399] ENH: autoflush during app init if any output param was provided --- pyotb/core.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 65a05f6..e9e2018 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -35,6 +35,7 @@ class OTBObject: """ self.parameters = {} self.name = self.appname = appname + self.frozen = frozen self.quiet = quiet self.image_dic = image_dic self.exports_dic = {} @@ -45,12 +46,12 @@ class OTBObject: self.parameters_keys = tuple(self.app.GetParametersKeys()) self.out_param_types = dict(get_out_param_types(self)) self.out_param_keys = tuple(self.out_param_types.keys()) - if args or kwargs: self.set_parameters(*args, **kwargs) - self.frozen = frozen - if not frozen: + if not self.frozen: self.execute() + if any(key in self.parameters for key in self.out_param_keys): + self.flush() # auto flush if any output param was provided during app init @property def key_input(self): @@ -181,8 +182,6 @@ class OTBObject: self.frozen = False logger.debug("%s: execution ended", self.name) self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics - if any(key in self.parameters for key in self.out_param_keys): - self.flush() # auto flush if any output param was provided during app init def flush(self): """Flush data to disk, this is when WriteOutput is actually called.""" @@ -246,7 +245,6 @@ class OTBObject: if key in dtypes: self.propagate_dtype(key, dtypes[key]) self.set_parameters({key: output_filename}) - self.flush() def propagate_dtype(self, target_key=None, dtype=None): -- GitLab From af002a4502234208b0b11d613804b3ca39c323de Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 11 Jan 2023 12:17:05 +0100 Subject: [PATCH 024/399] STYLE: whitespaces and comments --- pyotb/core.py | 42 ++++++++++++++---------------------------- 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index e9e2018..cf13497 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -219,6 +219,7 @@ class OTBObject: logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg) elif isinstance(arg, str) and self.key_output_image: kwargs.update({self.key_output_image: arg}) + # Append filename extension to filenames if filename_extension: logger.debug("%s: using extended filename for outputs: %s", self.name, filename_extension) @@ -227,6 +228,7 @@ class OTBObject: for key, value in kwargs.items(): if self.out_param_types[key] == 'raster' and '?' not in value: kwargs[key] = value + filename_extension + # Manage output pixel types dtypes = {} if pixel_type: @@ -240,7 +242,8 @@ class OTBObject: dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()} elif preserve_dtype: self.propagate_dtype() # all outputs will have the same type as the main input raster - # Apply parameters + + # Set parameters and flush to disk for key, output_filename in kwargs.items(): if key in dtypes: self.propagate_dtype(key, dtypes[key]) @@ -270,13 +273,11 @@ class OTBObject: except (TypeError, RuntimeError): logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param) return - if target_key: keys = [target_key] else: keys = [k for k in self.out_param_keys if self.out_param_types[k] == "raster"] for key in keys: - # Set output pixel type self.app.SetParameterOutputImagePixelType(key, dtype) def read_values_at_coords(self, row, col, bands=None): @@ -292,7 +293,7 @@ class OTBObject: """ channels = [] - app = OTBObject("PixelValue", self, coordx=col, coordy=row, frozen=False, quiet=True) + app = OTBObject("PixelValue", self, coordx=col, coordy=row, frozen=True, quiet=True) if bands is not None: if isinstance(bands, int): if bands < 0: @@ -342,7 +343,6 @@ class OTBObject: params[k] = p.summarize() elif isinstance(p, list): # parameter list params[k] = [pi.summarize() if isinstance(pi, OTBObject) else pi for pi in p] - return {"name": self.name, "parameters": params} def export(self, key=None, preserve_dtype=True): @@ -441,7 +441,7 @@ class OTBObject: return if key not in self.parameters_keys: raise KeyError( - f"{self.name}: parameter '{key}' was not recognized. " f"Available keys are {self.parameters_keys}" + f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}" ) # Single-parameter cases if isinstance(obj, OTBObject): @@ -819,12 +819,10 @@ class OTBObject: else: logger.debug(type(self)) return NotImplemented - # Performing the numpy operation result_array = ufunc(*arrays, **kwargs) result_dic = image_dic result_dic["array"] = result_array - # Importing back to OTB, pass the result_dic just to keep reference app = OTBObject("ExtractROI", image_dic=result_dic, frozen=True, quiet=True) if result_array.shape[2] == 1: @@ -833,7 +831,6 @@ class OTBObject: app.ImportVectorImage("in", result_dic) app.execute() return app - return NotImplemented @@ -858,6 +855,7 @@ class Slicer(OTBObject): self.name = "Slicer" self.rows, self.cols = rows, cols parameters = {} + # Channel slicing if channels != slice(None, None, None): # Trigger source app execution if needed @@ -873,7 +871,6 @@ class Slicer(OTBObject): channels = list(channels) elif not isinstance(channels, list): raise ValueError(f"Invalid type for channels, should be int, slice or list of bands. : {channels}") - # Change the potential negative index values to reverse index channels = [c if c >= 0 else nb_channels + c for c in channels] parameters.update({"cl": [f"Channel{i + 1}" for i in channels]}) @@ -957,12 +954,10 @@ class Operation(OTBObject): self.im_dic[str(inp)] = f"im{self.im_count}" mapping_str_to_input[str(inp)] = inp self.im_count += 1 - # Getting unique image inputs, in the order im1, im2, im3 ... self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)] - # Computing the BandMath or BandMathX app self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands) - # Init app + # Execute app appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX" super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True) self.name = f'Operation exp="{self.exp}"' @@ -980,9 +975,8 @@ class Operation(OTBObject): """ self.inputs.clear() self.nb_channels.clear() - logger.debug("%s, %s", operator, inputs) - # this is when we use the ternary operator with `pyotb.where` function. The output nb of bands is already known + # This is when we use the ternary operator with `pyotb.where` function. The output nb of bands is already known if operator == "?" and nb_bands: pass # For any other operations, the output number of bands is the same as inputs @@ -1002,10 +996,10 @@ class Operation(OTBObject): for i, band in enumerate(range(1, nb_bands + 1)): fake_exps = [] for k, inp in enumerate(inputs): - # Generating the fake expression of the current input + # Generating the fake expression of the current input, # this is a special case for the condition of the ternary operator `cond ? x : y` if len(inputs) == 3 and k == 0: - # when cond is monoband whereas the result is multiband, we expand the cond to multiband + # When cond is monoband whereas the result is multiband, we expand the cond to multiband if nb_bands != inp.shape[2]: cond_band = 1 else: @@ -1013,8 +1007,8 @@ class Operation(OTBObject): fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp( inp, cond_band, keep_logical=True ) - # any other input else: + # Any other input fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp( inp, band, keep_logical=False ) @@ -1033,7 +1027,6 @@ class Operation(OTBObject): fake_exp = f"({fake_exps[0]} {operator} {fake_exps[1]})" elif len(inputs) == 3 and operator == "?": # this is only for ternary expression fake_exp = f"({fake_exps[0]} ? {fake_exps[1]} : {fake_exps[2]})" - self.fake_exp_bands.append(fake_exp) def get_real_exp(self, fake_exp_bands): @@ -1052,13 +1045,11 @@ class Operation(OTBObject): for one_band_fake_exp in fake_exp_bands: one_band_exp = one_band_fake_exp for inp in self.inputs: - # replace the name of in-memory object (e.g. '<pyotb.App object>b1' by 'im1b1') + # Replace the name of in-memory object (e.g. '<pyotb.App object>b1' by 'im1b1') one_band_exp = one_band_exp.replace(str(inp), self.im_dic[str(inp)]) exp_bands.append(one_band_exp) - # Form the final expression (e.g. 'im1b1 + 1; im1b2 + 1') exp = ";".join(exp_bands) - return exp_bands, exp @staticmethod @@ -1088,7 +1079,7 @@ class Operation(OTBObject): inputs = x.input.inputs nb_channels = x.input.nb_channels elif isinstance(x.input, Operation): - # keep only one band of the expression + # Keep only one band of the expression fake_exp = x.input.fake_exp_bands[x.one_band_sliced - 1] inputs = x.input.inputs nb_channels = x.input.nb_channels @@ -1117,7 +1108,6 @@ class Operation(OTBObject): inputs = [x] # Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1') fake_exp = f"{x}b{band}" - return fake_exp, inputs, nb_channels def __str__(self): @@ -1177,11 +1167,9 @@ class LogicalOperation(Operation): if i == 0 and corresp_inputs and nb_channels: self.inputs.extend(corresp_inputs) self.nb_channels.update(nb_channels) - # We create here the "fake" expression. For example, for a BandMathX expression such as 'im1 > im2', # the logical fake expression stores the expression "str(input1) > str(input2)" logical_fake_exp = f"({fake_exps[0]} {operator} {fake_exps[1]})" - # We keep the logical expression, useful if later combined with other logical operations self.logical_fake_exp_bands.append(logical_fake_exp) # We create a valid BandMath expression, e.g. "str(input1) > str(input2) ? 1 : 0" @@ -1284,7 +1272,6 @@ def get_pixel_type(inp): """ if isinstance(inp, str): - # Executing the app, without printing its log try: info = OTBObject("ReadImageInfo", inp, quiet=True) except Exception as info_err: # this happens when we pass a str that is not a filepath @@ -1301,7 +1288,6 @@ def get_pixel_type(inp): pixel_type = inp.GetParameterOutputImagePixelType(inp.key_output_image) else: raise TypeError(f'Could not get the pixel type of {type(inp)} object {inp}') - return pixel_type -- GitLab From 712014ed5b691e413a4ff2c7e89985fce501b2f7 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 11 Jan 2023 12:32:42 +0100 Subject: [PATCH 025/399] STYLE: whitespaces --- pyotb/apps.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/pyotb/apps.py b/pyotb/apps.py index bb15f1c..2a85fc6 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -5,8 +5,8 @@ import sys from pathlib import Path import otbApplication as otb -from .helpers import logger from .core import OTBObject +from .helpers import logger def get_available_applications(as_subprocess=False): @@ -32,7 +32,6 @@ def get_available_applications(as_subprocess=False): env["OTB_LOGGER_LEVEL"] = "CRITICAL" # in order to suppress warnings while listing applications pycmd = "import otbApplication; print(otbApplication.Registry.GetAvailableApplications())" cmd_args = [sys.executable, "-c", pycmd] - try: import subprocess # pylint: disable=import-outside-toplevel params = {"env": env, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE} @@ -48,7 +47,6 @@ def get_available_applications(as_subprocess=False): logger.debug("Failed to call subprocess") except (ValueError, SyntaxError, AssertionError): logger.debug("Failed to decode output or convert to tuple:\nstdout=%s\nstderr=%s", stdout, stderr) - if not app_list: logger.info("Failed to list applications in an independent process. Falling back to local python import") # Find applications using the normal way @@ -56,7 +54,6 @@ def get_available_applications(as_subprocess=False): app_list = otb.Registry.GetAvailableApplications() if not app_list: raise SystemExit("Unable to load applications. Set env variable OTB_APPLICATION_PATH and try again.") - logger.info("Successfully loaded %s OTB applications", len(app_list)) return app_list @@ -88,10 +85,9 @@ class App(OTBObject): name: new name """ - if isinstance(name, str): - self._name = name - else: + if not isinstance(name, str): raise TypeError(f"{self.name}: bad type ({type(name)}) for application name, only str is allowed") + self._name = name @property def outputs(self): @@ -105,14 +101,12 @@ class App(OTBObject): list of files found on disk """ - files = [] - missing = [] + files, missing = [], [] for out in self.outputs: dest = files if out.exists() else missing dest.append(str(out.filepath.absolute())) for filename in missing: logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) - return tuple(files) -- GitLab From 7e2a2c1a89569175e8914082cdc7e4cd6d4fc0bc Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 11 Jan 2023 14:56:44 +0100 Subject: [PATCH 026/399] WIP: proposition to reduce code complexity, simplify app naming --- pyotb/apps.py | 22 ---------------------- pyotb/core.py | 18 ++++++++---------- 2 files changed, 8 insertions(+), 32 deletions(-) diff --git a/pyotb/apps.py b/pyotb/apps.py index 2a85fc6..b3f9fd7 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -67,28 +67,6 @@ class App(OTBObject): super().__init__(*args, **kwargs) self.description = self.app.GetDocLongDescription() - @property - def name(self): - """Application name that will be printed in logs. - - Returns: - user's defined name or appname - - """ - return self._name or self.appname - - @name.setter - def name(self, name): - """Set custom name. - - Args: - name: new name - - """ - if not isinstance(name, str): - raise TypeError(f"{self.name}: bad type ({type(name)}) for application name, only str is allowed") - self._name = name - @property def outputs(self): """List of application outputs.""" diff --git a/pyotb/core.py b/pyotb/core.py index cf13497..380602d 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -12,11 +12,11 @@ from .helpers import logger class OTBObject: """Base class that gathers common operations for any OTB application.""" - def __init__(self, appname, *args, frozen=False, quiet=False, image_dic=None, **kwargs): + def __init__(self, name, *args, frozen=False, quiet=False, image_dic=None, **kwargs): """Common constructor for OTB applications. Handles in-memory connection between apps. Args: - appname: name of the app, e.g. 'BandMath' + name: name of the app, e.g. 'BandMath' *args: used for passing application parameters. Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") @@ -34,15 +34,13 @@ class OTBObject: """ self.parameters = {} - self.name = self.appname = appname + self.name = name self.frozen = frozen self.quiet = quiet self.image_dic = image_dic self.exports_dic = {} - if quiet: - self.app = otb.Registry.CreateApplicationWithoutLogger(appname) - else: - self.app = otb.Registry.CreateApplication(appname) + self.app = otb.Registry.CreateApplicationWithoutLogger(name) if quiet \ + else otb.Registry.CreateApplication(name) self.parameters_keys = tuple(self.app.GetParametersKeys()) self.out_param_types = dict(get_out_param_types(self)) self.out_param_keys = tuple(self.out_param_types.keys()) @@ -479,7 +477,7 @@ class OTBObject: def __str__(self): """Return a nice string representation with object id.""" - return f"<pyotb.App {self.appname} object id {id(self)}>" + return f"<pyotb.App {self.name} object id {id(self)}>" def __getattr__(self, name): """This method is called when the default attribute access fails. @@ -958,8 +956,8 @@ class Operation(OTBObject): self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)] self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands) # Execute app - appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX" - super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True) + name = "BandMath" if len(self.exp_bands) == 1 else "BandMathX" + super().__init__(name, il=self.unique_inputs, exp=self.exp, quiet=True) self.name = f'Operation exp="{self.exp}"' def create_fake_exp(self, operator, inputs, nb_bands=None): -- GitLab From 277579963121dfd0b9bc979a4228836618958809 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 11 Jan 2023 15:34:25 +0100 Subject: [PATCH 027/399] FIX: Input back working with /vsiXX/ resources --- pyotb/core.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 380602d..532c8f3 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -39,8 +39,8 @@ class OTBObject: self.quiet = quiet self.image_dic = image_dic self.exports_dic = {} - self.app = otb.Registry.CreateApplicationWithoutLogger(name) if quiet \ - else otb.Registry.CreateApplication(name) + create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication + self.app = create(name) self.parameters_keys = tuple(self.app.GetParametersKeys()) self.out_param_types = dict(get_out_param_types(self)) self.out_param_keys = tuple(self.out_param_types.keys()) @@ -68,7 +68,7 @@ class OTBObject: @property def dtype(self): - """Expose the pixel type of an output image using numpy convention. + """Expose the pixel type of output image using numpy convention. Returns: dtype: pixel type of the output image @@ -91,7 +91,7 @@ class OTBObject: raise TypeError(f"{self.name}: this application has no raster output") width, height = self.app.GetImageSize(self.key_output_image) bands = self.app.GetImageNbBands(self.key_output_image) - return (height, width, bands) + return height, width, bands @property def transform(self): @@ -106,7 +106,7 @@ class OTBObject: origin_x, origin_y = self.app.GetImageOrigin(self.key_output_image) # Shift image origin since OTB is giving coordinates of pixel center instead of corners origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2 - return (spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y) + return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y def set_parameters(self, *args, **kwargs): """Set some parameters of the app. @@ -344,10 +344,12 @@ class OTBObject: return {"name": self.name, "parameters": params} def export(self, key=None, preserve_dtype=True): - """Export a specific output image as numpy array and store it in object's exports_dic. + """Export a specific output image as numpy array and store it in object exports_dic. Args: key: parameter key to export, if None then the default one will be used + preserve_dtype: when set to True, the numpy array is converted to the same pixel type as + the OTBObject first output. Default is True Returns: the exported numpy array @@ -367,7 +369,7 @@ class OTBObject: Args: key: the output parameter name to export as numpy array preserve_dtype: when set to True, the numpy array is converted to the same pixel type as - the OTBObject first output. Default is True. + the OTBObject first output. Default is True copy: whether to copy the output array, default is False required to True if preserve_dtype is False and the source app reference is lost @@ -1178,24 +1180,22 @@ class LogicalOperation(Operation): class Input(OTBObject): """Class for transforming a filepath to pyOTB object.""" - def __init__(self, filepath): + def __init__(self, path): """Default constructor. Args: - filepath: the path of an input image + path: Anything supported by GDAL (local file on the filesystem, remote resource e.g. /vsicurl/.., etc.) """ - self.filepath = Path(filepath) - if not self.filepath.exists(): - raise FileNotFoundError(filepath) - super().__init__("ExtractROI", {"in": str(filepath)}, frozen=True) - self.name = f"Input from {filepath}" + self.path = path + super().__init__("ExtractROI", {"in": path}, frozen=True) + self.name = f"Input from {path}" self.propagate_dtype() self.execute() def __str__(self): """Return a nice string representation with file path.""" - return f"<pyotb.Input object from {self.filepath}>" + return f"<pyotb.Input object from {self.path}>" class Output: -- GitLab From 68ef3d21b3752ee3bf472b92d8adab2cb5cac80b Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 11 Jan 2023 15:34:45 +0100 Subject: [PATCH 028/399] FIX: remove file missing test --- tests/test_core.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 2d99bd2..f053b2f 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -17,7 +17,7 @@ def test_parameters(): # Catch errors before exec def test_missing_input_file(): - with pytest.raises(FileNotFoundError): + with pytest.raises(RuntimeError): pyotb.Input("missing_file.tif") @@ -129,7 +129,6 @@ def test_write(): # BandMath NDVI == RadiometricIndices NDVI ? -@pytest.mark.xfail(reason="Regression in OTB 8.2, waiting for Rémi's patch to be merged.") def test_ndvi_comparison(): ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (INPUT[:, :, -1] + INPUT[:, :, 0]) ndvi_indices = pyotb.RadiometricIndices(INPUT, {"list": "Vegetation:NDVI", "channels.red": 1, "channels.nir": 4}) -- GitLab From 45e32e849579391020da1e9ef05d496f4a216071 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 11 Jan 2023 15:35:14 +0100 Subject: [PATCH 029/399] ENH: test_serialization can work with any input image path --- tests/test_serialization.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_serialization.py b/tests/test_serialization.py index 2e0ce7f..e64f2f1 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -12,7 +12,7 @@ def test_pipeline_simple(): summary = app3.summarize() reference = {'name': 'ManageNoData', 'parameters': {'in': { 'name': 'OrthoRectification', 'parameters': {'io.in': { - 'name': 'BandMath', 'parameters': {'il': ('tests/image.tif',), 'exp': 'im1b1'}}, + 'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}}, 'map': 'utm', 'outputs.isotropic': True}}, 'mode': 'buildmask'}} @@ -28,12 +28,12 @@ def test_pipeline_diamond(): summary = app4.summarize() reference = {'name': 'BandMathX', 'parameters': {'il': [ {'name': 'OrthoRectification', 'parameters': {'io.in': { - 'name': 'BandMath', 'parameters': {'il': ('tests/image.tif',), 'exp': 'im1b1'}}, + 'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}}, 'map': 'utm', 'outputs.isotropic': True}}, {'name': 'ManageNoData', 'parameters': {'in': { 'name': 'OrthoRectification', 'parameters': { - 'io.in': {'name': 'BandMath', 'parameters': {'il': ('tests/image.tif',), 'exp': 'im1b1'}}, + 'io.in': {'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}}, 'map': 'utm', 'outputs.isotropic': True}}, 'mode': 'buildmask'}} -- GitLab From 93eb844b398c46142a4d75756c699d79916eb8c6 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 11 Jan 2023 15:47:31 +0100 Subject: [PATCH 030/399] FIX: remove file missing test --- tests/test_core.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index f053b2f..2eadef7 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -15,12 +15,6 @@ def test_parameters(): assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304) -# Catch errors before exec -def test_missing_input_file(): - with pytest.raises(RuntimeError): - pyotb.Input("missing_file.tif") - - def test_wrong_key(): with pytest.raises(KeyError): pyotb.BandMath(INPUT, expression="im1b1") -- GitLab From 210b3f70fb9732d0b22b0d222c5b2b26031c005e Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 11 Jan 2023 17:20:06 +0100 Subject: [PATCH 031/399] REFAC: simplify input/output keys accessors --- pyotb/apps.py | 8 ++-- pyotb/core.py | 105 ++++++++++++++++++++++---------------------------- 2 files changed, 50 insertions(+), 63 deletions(-) diff --git a/pyotb/apps.py b/pyotb/apps.py index b3f9fd7..5e30d0e 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -68,9 +68,9 @@ class App(OTBObject): self.description = self.app.GetDocLongDescription() @property - def outputs(self): - """List of application outputs.""" - return [getattr(self, key) for key in self.out_param_keys if key in self.parameters] + def used_outputs(self): + """List of used application outputs.""" + return [getattr(self, key) for key in self.out_param_types if key in self.parameters] def find_outputs(self): """Find output files on disk using path found in parameters. @@ -80,7 +80,7 @@ class App(OTBObject): """ files, missing = [], [] - for out in self.outputs: + for out in self.used_outputs: dest = files if out.exists() else missing dest.append(str(out.filepath.absolute())) for filename in missing: diff --git a/pyotb/core.py b/pyotb/core.py index 532c8f3..bbd681d 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -42,29 +42,45 @@ class OTBObject: create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication self.app = create(name) self.parameters_keys = tuple(self.app.GetParametersKeys()) - self.out_param_types = dict(get_out_param_types(self)) - self.out_param_keys = tuple(self.out_param_types.keys()) + # Output parameters types + self.all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys} + self.out_param_types = {k: v for k, v in self.all_param_types.items() + if v in (otb.ParameterType_OutputImage, + otb.ParameterType_OutputVectorData, + otb.ParameterType_OutputFilename)} if args or kwargs: self.set_parameters(*args, **kwargs) if not self.frozen: self.execute() - if any(key in self.parameters for key in self.out_param_keys): + if any(key in self.parameters for key in self.parameters_keys): self.flush() # auto flush if any output param was provided during app init + def get_first_key(self, param_types): + """Get the first output param key for specific file types.""" + for key, param_type in sorted(self.all_param_types.items()): + if param_type in param_types: + return key + return None + @property def key_input(self): """Get the name of first input parameter, raster > vector > file.""" - return key_input(self, "raster") or key_input(self, "vector") or key_input(self, "file") + return self.get_first_key(param_types=[otb.ParameterType_InputImage, + otb.ParameterType_InputImageList]) \ + or self.get_first_key(param_types=[otb.ParameterType_InputVectorData, + otb.ParameterType_InputVectorDataList]) \ + or self.get_first_key(param_types=[otb.ParameterType_InputFilename, + otb.ParameterType_InputFilenameList]) @property def key_input_image(self): """Get the name of first input image parameter.""" - return key_input(self, "raster") + return self.get_first_key(param_types=[otb.ParameterType_InputImage, otb.ParameterType_InputImageList]) @property def key_output_image(self): """Get the name of first output image parameter.""" - return key_output(self, "raster") + return self.get_first_key(param_types=[otb.ParameterType_OutputImage]) @property def dtype(self): @@ -165,7 +181,7 @@ class OTBObject: except RuntimeError: continue # this is when there is no value for key # Convert output param path to Output object - if key in self.out_param_keys: + if key in self.out_param_types: value = Output(self, key, value) # Save attribute setattr(self, key, value) @@ -224,7 +240,7 @@ class OTBObject: if not filename_extension.startswith("?"): filename_extension = "?" + filename_extension for key, value in kwargs.items(): - if self.out_param_types[key] == 'raster' and '?' not in value: + if self.out_param_types[key] == otb.ParameterType_OutputImage and '?' not in value: kwargs[key] = value + filename_extension # Manage output pixel types @@ -234,7 +250,7 @@ class OTBObject: type_name = self.app.ConvertPixelTypeToNumpy(parse_pixel_type(pixel_type)) logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name) for key in kwargs: - if self.out_param_types.get(key) == "raster": + if self.out_param_types[key] == otb.ParameterType_OutputImage: dtypes[key] = parse_pixel_type(pixel_type) elif isinstance(pixel_type, dict): dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()} @@ -274,7 +290,7 @@ class OTBObject: if target_key: keys = [target_key] else: - keys = [k for k in self.out_param_keys if self.out_param_types[k] == "raster"] + keys = [k for k, v in self.out_param_types.items() if v == otb.ParameterType_OutputImage] for key in keys: self.app.SetParameterOutputImagePixelType(key, dtype) @@ -303,7 +319,7 @@ class OTBObject: raise TypeError(f"{self.name}: type '{type(bands)}' cannot be interpreted as a valid slicing") if channels: app.app.Execute() - app.set_parameters({"cl": [f"Channel{n+1}" for n in channels]}) + app.set_parameters({"cl": [f"Channel{n + 1}" for n in channels]}) app.execute() data = literal_eval(app.app.GetParameterString("value")) if len(channels) == 1: @@ -413,7 +429,7 @@ class OTBObject: """ spacing_x, _, origin_x, _, spacing_y, origin_y = self.transform row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x - return (abs(int(row)), int(col)) + return abs(int(row)), int(col) # Private functions def __parse_args(self, args): @@ -1122,6 +1138,7 @@ class LogicalOperation(Operation): logical expression (e.g. "im1b1 > 0") """ + def __init__(self, operator, *inputs, nb_bands=None): """Constructor for a LogicalOperation object. @@ -1273,16 +1290,25 @@ def get_pixel_type(inp): try: info = OTBObject("ReadImageInfo", inp, quiet=True) except Exception as info_err: # this happens when we pass a str that is not a filepath - raise TypeError(f'Could not get the pixel type of `{inp}`. Not a filepath or wrong filepath') from info_err + raise TypeError(f"Could not get the pixel type of `{inp}`. Not a filepath or wrong filepath") from info_err datatype = info.GetParameterString("datatype") # which is such as short, float... if not datatype: - raise Exception(f'Unable to read pixel type of image {inp}') - datatype_to_pixeltype = {'unsigned_char': 'uint8', 'short': 'int16', 'unsigned_short': 'uint16', - 'int': 'int32', 'unsigned_int': 'uint32', 'long': 'int32', 'ulong': 'uint32', - 'float': 'float', 'double': 'double'} - pixel_type = datatype_to_pixeltype[datatype] - pixel_type = getattr(otb, f'ImagePixelType_{pixel_type}') - elif isinstance(inp, (OTBObject)): + raise TypeError(f"Unable to read pixel type of image {inp}") + datatype_to_pixeltype = { + 'unsigned_char': 'uint8', + 'short': 'int16', + 'unsigned_short': 'uint16', + 'int': 'int32', + 'unsigned_int': 'uint32', + 'long': 'int32', + 'ulong': 'uint32', + 'float': 'float', + 'double': 'double' + } + if datatype not in datatype_to_pixeltype: + raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}") + pixel_type = getattr(otb, f'ImagePixelType_{datatype_to_pixeltype[datatype]}') + elif isinstance(inp, OTBObject): pixel_type = inp.GetParameterOutputImagePixelType(inp.key_output_image) else: raise TypeError(f'Could not get the pixel type of {type(inp)} object {inp}') @@ -1324,45 +1350,6 @@ def is_key_images_list(pyotb_app, key): ) -def get_out_param_types(pyotb_app): - """Get output parameter data type (raster, vector, file).""" - outfile_types = { - otb.ParameterType_OutputImage: "raster", - otb.ParameterType_OutputVectorData: "vector", - otb.ParameterType_OutputFilename: "file", - } - for k in pyotb_app.parameters_keys: - t = pyotb_app.app.GetParameterType(k) - if t in outfile_types: - yield k, outfile_types[t] - - def get_out_images_param_keys(app): """Return every output parameter keys of an OTB app.""" return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage] - - -def key_input(pyotb_app, file_type): - """Get the first input param key for a specific file type.""" - types = { - "raster": (otb.ParameterType_InputImage, otb.ParameterType_InputImageList), - "vector": (otb.ParameterType_InputVectorData, otb.ParameterType_InputVectorDataList), - "file": (otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList) - } - for key in pyotb_app.parameters_keys: - if pyotb_app.app.GetParameterType(key) in types[file_type]: - return key - return None - - -def key_output(pyotb_app, file_type): - """Get the first output param key for a specific file type.""" - types = { - "raster": otb.ParameterType_OutputImage, - "vector": otb.ParameterType_OutputVectorData, - "file": otb.ParameterType_OutputFilename - } - for key in pyotb_app.parameters_keys: - if pyotb_app.app.GetParameterType(key) == types[file_type]: - return key - return None -- GitLab From 4f8d363cb84e2e26ab72102c61723c6bb786066f Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 11 Jan 2023 17:05:44 +0000 Subject: [PATCH 032/399] Apply 1 suggestion(s) to 1 file(s) --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index bbd681d..86bfb9c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -52,7 +52,7 @@ class OTBObject: self.set_parameters(*args, **kwargs) if not self.frozen: self.execute() - if any(key in self.parameters for key in self.parameters_keys): + if any(key in self.parameters for key in self.out_param_types): self.flush() # auto flush if any output param was provided during app init def get_first_key(self, param_types): -- GitLab From 9638ef14dbff9a4afc842bbc831394aff1a0d5ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr> Date: Fri, 13 Jan 2023 10:40:27 +0000 Subject: [PATCH 033/399] Typing hints in functions --- pyotb/apps.py | 15 +++--- pyotb/core.py | 123 ++++++++++++++++++++++++--------------------- pyotb/functions.py | 18 ++++--- pyotb/helpers.py | 16 +++--- 4 files changed, 92 insertions(+), 80 deletions(-) diff --git a/pyotb/apps.py b/pyotb/apps.py index 5e30d0e..a2af7ed 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- """Search for OTB (set env if necessary), subclass core.App for each available application.""" +from __future__ import annotations import os import sys from pathlib import Path @@ -9,7 +10,7 @@ from .core import OTBObject from .helpers import logger -def get_available_applications(as_subprocess=False): +def get_available_applications(as_subprocess: bool = False) -> list[str]: """Find available OTB applications. Args: @@ -68,11 +69,11 @@ class App(OTBObject): self.description = self.app.GetDocLongDescription() @property - def used_outputs(self): + def used_outputs(self) -> list[str]: """List of used application outputs.""" return [getattr(self, key) for key in self.out_param_types if key in self.parameters] - def find_outputs(self): + def find_outputs(self) -> tuple[str]: """Find output files on disk using path found in parameters. Returns: @@ -91,7 +92,7 @@ class App(OTBObject): class OTBTFApp(App): """Helper for OTBTF.""" @staticmethod - def set_nb_sources(*args, n_sources=None): + def set_nb_sources(*args, n_sources: int = None): """Set the number of sources of TensorflowModelServe. Can be either user-defined or deduced from the args. Args: @@ -109,11 +110,11 @@ class OTBTFApp(App): if n_sources >= 1: os.environ['OTB_TF_NSOURCES'] = str(n_sources) - def __init__(self, app_name, *args, n_sources=None, **kwargs): + def __init__(self, name: str, *args, n_sources: int = None, **kwargs): """Constructor for an OTBTFApp object. Args: - app_name: name of the OTBTF app + name: name of the OTBTF app *args: arguments (dict). NB: we don't need kwargs because it cannot contain source#.il n_sources: number of sources. Default is None (resolves the number of sources based on the content of the dict passed in args, where some 'source' str is found) @@ -121,7 +122,7 @@ class OTBTFApp(App): """ self.set_nb_sources(*args, n_sources=n_sources) - super().__init__(app_name, *args, **kwargs) + super().__init__(name, *args, **kwargs) AVAILABLE_APPLICATIONS = get_available_applications(as_subprocess=True) diff --git a/pyotb/core.py b/pyotb/core.py index 86bfb9c..88496b6 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- """This module is the core of pyotb.""" +from __future__ import annotations +from typing import Any from pathlib import Path from ast import literal_eval @@ -12,7 +14,7 @@ from .helpers import logger class OTBObject: """Base class that gathers common operations for any OTB application.""" - def __init__(self, name, *args, frozen=False, quiet=False, image_dic=None, **kwargs): + def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, **kwargs): """Common constructor for OTB applications. Handles in-memory connection between apps. Args: @@ -55,7 +57,7 @@ class OTBObject: if any(key in self.parameters for key in self.out_param_types): self.flush() # auto flush if any output param was provided during app init - def get_first_key(self, param_types): + def get_first_key(self, param_types: list[str]) -> str: """Get the first output param key for specific file types.""" for key, param_type in sorted(self.all_param_types.items()): if param_type in param_types: @@ -63,7 +65,7 @@ class OTBObject: return None @property - def key_input(self): + def key_input(self) -> str: """Get the name of first input parameter, raster > vector > file.""" return self.get_first_key(param_types=[otb.ParameterType_InputImage, otb.ParameterType_InputImageList]) \ @@ -73,17 +75,17 @@ class OTBObject: otb.ParameterType_InputFilenameList]) @property - def key_input_image(self): + def key_input_image(self) -> str: """Get the name of first input image parameter.""" return self.get_first_key(param_types=[otb.ParameterType_InputImage, otb.ParameterType_InputImageList]) @property - def key_output_image(self): + def key_output_image(self) -> str: """Get the name of first output image parameter.""" return self.get_first_key(param_types=[otb.ParameterType_OutputImage]) @property - def dtype(self): + def dtype(self) -> np.dtype: """Expose the pixel type of output image using numpy convention. Returns: @@ -96,7 +98,7 @@ class OTBObject: return self.app.ConvertPixelTypeToNumpy(enum) @property - def shape(self): + def shape(self) -> tuple(int): """Enables to retrieve the shape of a pyotb object using numpy convention. Returns: @@ -110,7 +112,7 @@ class OTBObject: return height, width, bands @property - def transform(self): + def transform(self) -> tuple(int): """Get image affine transform, rasterio style (see https://www.perrygeo.com/python-affine-transforms.html). Returns: @@ -206,7 +208,8 @@ class OTBObject: logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name) self.app.ExecuteAndWriteOutput() - def write(self, *args, filename_extension="", pixel_type=None, preserve_dtype=False, **kwargs): + def write(self, *args, filename_extension: str = "", pixel_type: dict[str, str] | str = None, + preserve_dtype: bool = False, **kwargs): """Set output pixel type and write the output raster files. Args: @@ -264,7 +267,7 @@ class OTBObject: self.set_parameters({key: output_filename}) self.flush() - def propagate_dtype(self, target_key=None, dtype=None): + def propagate_dtype(self, target_key: str = None, dtype: int = None): """Propagate a pixel type from main input to every outputs, or to a target output key only. With multiple inputs (if dtype is not provided), the type of the first input is considered. @@ -294,7 +297,7 @@ class OTBObject: for key in keys: self.app.SetParameterOutputImagePixelType(key, dtype) - def read_values_at_coords(self, row, col, bands=None): + def read_values_at_coords(self, row: int, col: int, bands: int = None) -> list[int | float] | int | float: """Get pixel value(s) at a given YX coordinates. Args: @@ -326,7 +329,7 @@ class OTBObject: return data[0] return data - def channels_list_from_slice(self, bands): + def channels_list_from_slice(self, bands: int) -> list[int]: """Get list of channels to read values at, from a slice.""" nb_channels = self.shape[2] start, stop, step = bands.start, bands.stop, bands.step @@ -343,7 +346,7 @@ class OTBObject: return list(range(0, nb_channels, step)) raise ValueError(f"{self.name}: '{bands}' cannot be interpreted as valid slicing.") - def summarize(self): + def summarize(self) -> dict: """Serialize an object and its pipeline into a dictionary. Returns: @@ -359,7 +362,7 @@ class OTBObject: params[k] = [pi.summarize() if isinstance(pi, OTBObject) else pi for pi in p] return {"name": self.name, "parameters": params} - def export(self, key=None, preserve_dtype=True): + def export(self, key: str = None, preserve_dtype: bool = True) -> dict[str, dict[str, np.ndarray]]: """Export a specific output image as numpy array and store it in object exports_dic. Args: @@ -379,7 +382,7 @@ class OTBObject: self.exports_dic[key]["array"] = self.exports_dic[key]["array"].astype(self.dtype) return self.exports_dic[key] - def to_numpy(self, key=None, preserve_dtype=True, copy=False): + def to_numpy(self, key: str = None, preserve_dtype: bool = True, copy: bool = False) -> np.ndarray: """Export a pyotb object to numpy array. Args: @@ -399,7 +402,7 @@ class OTBObject: return array.copy() return array - def to_rasterio(self): + def to_rasterio(self) -> tuple[np.ndarray, dict[str, Any]]: """Export image as a numpy array and its metadata compatible with rasterio. Returns: @@ -417,7 +420,7 @@ class OTBObject: } return array, profile - def xy_to_rowcol(self, x, y): + def xy_to_rowcol(self, x: float, y: float) -> tuple[int, int]: """Find (row, col) index using (x, y) projected coordinates - image CRS is expected. Args: @@ -432,7 +435,7 @@ class OTBObject: return abs(int(row)), int(col) # Private functions - def __parse_args(self, args): + def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]: """Gather all input arguments in kwargs dict. Args: @@ -450,7 +453,7 @@ class OTBObject: kwargs.update({self.key_input: arg}) return kwargs - def __set_param(self, key, obj): + def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]): """Set one parameter, decide which otb.Application method to use depending on target object.""" if obj is None or (isinstance(obj, (list, tuple)) and not obj): self.app.ClearValue(key) @@ -558,7 +561,7 @@ class OTBObject: key = key + (slice(None, None, None),) return Slicer(self, *key) - def __add__(self, other): + def __add__(self, other: OTBObject | Output | str | int | float) -> Operation: """Overrides the default addition and flavours it with BandMathX. Args: @@ -572,7 +575,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("+", self, other) - def __sub__(self, other): + def __sub__(self, other: OTBObject | Output | str | int | float) -> Operation: """Overrides the default subtraction and flavours it with BandMathX. Args: @@ -586,7 +589,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("-", self, other) - def __mul__(self, other): + def __mul__(self, other: OTBObject | Output | str | int | float) -> Operation: """Overrides the default subtraction and flavours it with BandMathX. Args: @@ -600,7 +603,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("*", self, other) - def __truediv__(self, other): + def __truediv__(self, other: OTBObject | Output | str | int | float) -> Operation: """Overrides the default subtraction and flavours it with BandMathX. Args: @@ -614,7 +617,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("/", self, other) - def __radd__(self, other): + def __radd__(self, other: OTBObject | Output | str | int | float) -> Operation: """Overrides the default reverse addition and flavours it with BandMathX. Args: @@ -628,7 +631,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("+", other, self) - def __rsub__(self, other): + def __rsub__(self, other: OTBObject | Output | str | int | float) -> Operation: """Overrides the default subtraction and flavours it with BandMathX. Args: @@ -642,7 +645,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("-", other, self) - def __rmul__(self, other): + def __rmul__(self, other: OTBObject | Output | str | int | float) -> Operation: """Overrides the default multiplication and flavours it with BandMathX. Args: @@ -656,7 +659,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("*", other, self) - def __rtruediv__(self, other): + def __rtruediv__(self, other: OTBObject | Output | str | int | float) -> Operation: """Overrides the default division and flavours it with BandMathX. Args: @@ -670,7 +673,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("/", other, self) - def __abs__(self): + def __abs__(self) -> Operation: """Overrides the default abs operator and flavours it with BandMathX. Returns: @@ -679,7 +682,7 @@ class OTBObject: """ return Operation("abs", self) - def __ge__(self, other): + def __ge__(self, other: OTBObject | Output | str | int | float) -> Operation: """Overrides the default greater or equal and flavours it with BandMathX. Args: @@ -693,7 +696,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation(">=", self, other) - def __le__(self, other): + def __le__(self, other: OTBObject | Output | str | int | float) -> Operation: """Overrides the default less or equal and flavours it with BandMathX. Args: @@ -707,7 +710,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation("<=", self, other) - def __gt__(self, other): + def __gt__(self, other: OTBObject | Output | str | int | float) -> Operation: """Overrides the default greater operator and flavours it with BandMathX. Args: @@ -721,7 +724,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation(">", self, other) - def __lt__(self, other): + def __lt__(self, other: OTBObject | Output | str | int | float) -> Operation: """Overrides the default less operator and flavours it with BandMathX. Args: @@ -735,7 +738,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation("<", self, other) - def __eq__(self, other): + def __eq__(self, other: OTBObject | Output | str | int | float) -> Operation: """Overrides the default eq operator and flavours it with BandMathX. Args: @@ -749,7 +752,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation("==", self, other) - def __ne__(self, other): + def __ne__(self, other: OTBObject | Output | str | int | float) -> Operation: """Overrides the default different operator and flavours it with BandMathX. Args: @@ -763,7 +766,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation("!=", self, other) - def __or__(self, other): + def __or__(self, other: OTBObject | Output | str | int | float) -> Operation: """Overrides the default or operator and flavours it with BandMathX. Args: @@ -777,7 +780,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation("||", self, other) - def __and__(self, other): + def __and__(self, other: OTBObject | Output | str | int | float) -> Operation: """Overrides the default and operator and flavours it with BandMathX. Args: @@ -794,7 +797,7 @@ class OTBObject: # TODO: other operations ? # e.g. __pow__... cf https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types - def __array__(self): + def __array__(self) -> np.ndarray: """This is called when running np.asarray(pyotb_object). Returns: @@ -803,7 +806,7 @@ class OTBObject: """ return self.to_numpy() - def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs) -> OTBObject: """This is called whenever a numpy function is called on a pyotb object. Operation is performed in numpy, then imported back to pyotb with the same georeference as input. @@ -853,7 +856,7 @@ class OTBObject: class Slicer(OTBObject): """Slicer objects i.e. when we call something like raster[:, :, 2] from Python.""" - def __init__(self, obj, rows, cols, channels): + def __init__(self, obj: OTBObject | Output | str, rows: int, cols: int, channels: int): """Create a slicer object, that can be used directly for writing or inside a BandMath. It contains : @@ -938,7 +941,7 @@ class Operation(OTBObject): """ - def __init__(self, operator, *inputs, nb_bands=None): + def __init__(self, operator: str, *inputs, nb_bands: int = None): """Given some inputs and an operator, this function enables to transform this into an OTB application. Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator. @@ -978,7 +981,8 @@ class Operation(OTBObject): super().__init__(name, il=self.unique_inputs, exp=self.exp, quiet=True) self.name = f'Operation exp="{self.exp}"' - def create_fake_exp(self, operator, inputs, nb_bands=None): + def create_fake_exp(self, operator: str, inputs: list[OTBObject | Output | str | int | float], + nb_bands: int = None): """Create a 'fake' expression. E.g for the operation input1 + input2, we create a fake expression that is like "str(input1) + str(input2)" @@ -1045,7 +1049,7 @@ class Operation(OTBObject): fake_exp = f"({fake_exps[0]} ? {fake_exps[1]} : {fake_exps[2]})" self.fake_exp_bands.append(fake_exp) - def get_real_exp(self, fake_exp_bands): + def get_real_exp(self, fake_exp_bands: str) -> tuple(list[str], str): """Generates the BandMathX expression. Args: @@ -1069,7 +1073,8 @@ class Operation(OTBObject): return exp_bands, exp @staticmethod - def create_one_input_fake_exp(x, band, keep_logical=False): + def create_one_input_fake_exp(x: OTBObject | Output | str, + band: int, keep_logical: bool = False) -> tuple(str, list[OTBObject], int): """This an internal function, only to be used by `create_fake_exp`. Enable to create a fake expression just for one input and one band. @@ -1126,7 +1131,7 @@ class Operation(OTBObject): fake_exp = f"{x}b{band}" return fake_exp, inputs, nb_channels - def __str__(self): + def __str__(self) -> str: """Return a nice string representation with operator and object id.""" return f"<pyotb.Operation `{self.operator}` object, id {id(self)}>" @@ -1139,7 +1144,7 @@ class LogicalOperation(Operation): """ - def __init__(self, operator, *inputs, nb_bands=None): + def __init__(self, operator: str, *inputs, nb_bands: int = None): """Constructor for a LogicalOperation object. Args: @@ -1151,7 +1156,8 @@ class LogicalOperation(Operation): super().__init__(operator, *inputs, nb_bands=nb_bands) self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands) - def create_fake_exp(self, operator, inputs, nb_bands=None): + def create_fake_exp(self, operator: str, inputs: list[OTBObject | Output | str | int | float], + nb_bands: int = None): """Create a 'fake' expression. E.g for the operation input1 > input2, we create a fake expression that is like @@ -1197,7 +1203,7 @@ class LogicalOperation(Operation): class Input(OTBObject): """Class for transforming a filepath to pyOTB object.""" - def __init__(self, path): + def __init__(self, path: str): """Default constructor. Args: @@ -1210,7 +1216,7 @@ class Input(OTBObject): self.propagate_dtype() self.execute() - def __str__(self): + def __str__(self) -> str: """Return a nice string representation with file path.""" return f"<pyotb.Input object from {self.path}>" @@ -1218,7 +1224,7 @@ class Input(OTBObject): class Output: """Object that behave like a pointer to a specific application output file.""" - def __init__(self, source_app, param_key, filepath=None, mkdir=True): + def __init__(self, source_app: OTBObject, param_key: str, filepath: str = None, mkdir: bool = True): """Constructor for an Output object. Args: @@ -1239,7 +1245,7 @@ class Output: self.make_parent_dirs() self.name = f"Output {param_key} from {self.source_app.name}" - def exists(self): + def exists(self) -> bool: """Check file exist.""" return self.filepath.exists() @@ -1248,12 +1254,12 @@ class Output: if not self.filepath.parent.exists(): self.filepath.parent.mkdir(parents=True) - def __str__(self): + def __str__(self) -> str: """Return a nice string representation with source app name and object id.""" return f"<pyotb.Output {self.source_app.name} object, id {id(self)}>" -def get_nbchannels(inp): +def get_nbchannels(inp: str | OTBObject) -> int: """Get the nb of bands of input image. Args: @@ -1275,7 +1281,7 @@ def get_nbchannels(inp): return nb_channels -def get_pixel_type(inp): +def get_pixel_type(inp: str | OTBObject) -> str: """Get the encoding of input image pixels. Args: @@ -1315,7 +1321,7 @@ def get_pixel_type(inp): return pixel_type -def parse_pixel_type(pixel_type): +def parse_pixel_type(pixel_type: str | int) -> int: """Convert one str pixel type to OTB integer enum if necessary. Args: @@ -1332,7 +1338,7 @@ def parse_pixel_type(pixel_type): raise ValueError(f'Bad pixel type specification ({pixel_type})') -def is_key_list(pyotb_app, key): +def is_key_list(pyotb_app: OTBObject, key: str) -> bool: """Check if a key of the App is an input parameter list.""" return pyotb_app.app.GetParameterType(key) in ( otb.ParameterType_InputImageList, @@ -1343,13 +1349,14 @@ def is_key_list(pyotb_app, key): ) -def is_key_images_list(pyotb_app, key): +def is_key_images_list(pyotb_app: OTBObject, key: str) -> bool: """Check if a key of the App is an input parameter image list.""" return pyotb_app.app.GetParameterType(key) in ( - otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList + otb.ParameterType_InputImageList, + otb.ParameterType_InputFilenameList ) -def get_out_images_param_keys(app): +def get_out_images_param_keys(app: OTBObject) -> list[str]: """Return every output parameter keys of an OTB app.""" return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage] diff --git a/pyotb/functions.py b/pyotb/functions.py index 669039a..aa161e6 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- """This module provides a set of functions for pyotb.""" +from __future__ import annotations import inspect import os import sys @@ -7,18 +8,19 @@ import textwrap import uuid from collections import Counter -from .core import (OTBObject, Input, Operation, LogicalOperation, get_nbchannels) +from .core import OTBObject, Input, Operation, LogicalOperation, get_nbchannels, Output from .helpers import logger -def where(cond, x, y): +def where(cond: OTBObject | Output | str, x: OTBObject | Output | str | int | float, + y: OTBObject | Output | str | int | float) -> Operation: """Functionally similar to numpy.where. Where cond is True (!=0), returns x. Else returns y. Args: cond: condition, must be a raster (filepath, App, Operation...). If cond is monoband whereas x or y are multiband, cond channels are expanded to match x & y ones. - x: value if cond is True. Can be float, int, App, filepath, Operation... - y: value if cond is False. Can be float, int, App, filepath, Operation... + x: value if cond is True. Can be: float, int, App, filepath, Operation... + y: value if cond is False. Can be: float, int, App, filepath, Operation... Returns: an output where pixels are x if cond is True, else y @@ -61,7 +63,8 @@ def where(cond, x, y): return operation -def clip(a, a_min, a_max): +def clip(a: OTBObject | Output | str, a_min: OTBObject | Output | str | int | float, + a_max: OTBObject | Output | str | int | float): """Clip values of image in a range of values. Args: @@ -321,8 +324,9 @@ def run_tf_function(func): return wrapper -def define_processing_area(*args, window_rule='intersection', pixel_size_rule='minimal', interpolator='nn', - reference_window_input=None, reference_pixel_size_input=None): +def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_rule: str = 'minimal', + interpolator: str = 'nn', reference_window_input: dict = None, + reference_pixel_size_input: str = None) -> list[OTBObject]: """Given several inputs, this function handles the potential resampling and cropping to same extent. WARNING: Not fully implemented / tested diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 742fee1..d38dfb3 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -26,7 +26,7 @@ logger_handler.setLevel(getattr(logging, LOG_LEVEL)) logger.addHandler(logger_handler) -def set_logger_level(level): +def set_logger_level(level: str): """Allow user to change the current logging level. Args: @@ -36,7 +36,7 @@ def set_logger_level(level): logger_handler.setLevel(getattr(logging, level)) -def find_otb(prefix=OTB_ROOT, scan=True, scan_userdir=True): +def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = True): """Try to load OTB bindings or scan system, help user in case of failure, set env variables. Path precedence : OTB_ROOT > python bindings directory @@ -98,7 +98,7 @@ def find_otb(prefix=OTB_ROOT, scan=True, scan_userdir=True): raise SystemExit("Failed to import OTB. Exiting.") from e -def set_environment(prefix): +def set_environment(prefix: str): """Set environment variables (before OTB import), raise error if anything is wrong. Args: @@ -156,7 +156,7 @@ def set_environment(prefix): os.environ["PROJ_LIB"] = proj_lib -def __find_lib(prefix=None, otb_module=None): +def __find_lib(prefix: str = None, otb_module=None): """Try to find OTB external libraries directory. Args: @@ -187,7 +187,7 @@ def __find_lib(prefix=None, otb_module=None): return None -def __find_python_api(lib_dir): +def __find_python_api(lib_dir: Path): """Try to find the python path. Args: @@ -206,7 +206,7 @@ def __find_python_api(lib_dir): return None -def __find_apps_path(lib_dir): +def __find_apps_path(lib_dir: Path): """Try to find the OTB applications path. Args: @@ -225,7 +225,7 @@ def __find_apps_path(lib_dir): return "" -def __find_otb_root(scan_userdir=False): +def __find_otb_root(scan_userdir: bool = False): """Search for OTB root directory in well known locations. Args: @@ -271,7 +271,7 @@ def __find_otb_root(scan_userdir=False): return prefix -def __suggest_fix_import(error_message, prefix): +def __suggest_fix_import(error_message: str, prefix: str): """Help user to fix the OTB installation with appropriate log messages.""" logger.critical("An error occurred while importing OTB Python API") logger.critical("OTB error message was '%s'", error_message) -- GitLab From 071ed7308ebc75e86651117ffe3de49d143491bf Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 24 Jan 2023 09:38:20 +0000 Subject: [PATCH 034/399] Resolve "Update pyproject.toml to PEP621" --- .gitlab-ci.yml | 6 ++--- pyproject.toml | 68 ++++++++++++++++++++++++++++++++++++++++++++++---- setup.cfg | 3 --- setup.py | 32 ++---------------------- 4 files changed, 68 insertions(+), 41 deletions(-) delete mode 100644 setup.cfg diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 203b789..a38149e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,16 +36,16 @@ flake8: pydocstyle: extends: .static_analysis before_script: - - pip install pydocstyle + - pip install pydocstyle tomli script: - - pydocstyle $PWD/pyotb --convention=google + - pydocstyle $PWD/pyotb pylint: extends: .static_analysis before_script: - pip install pylint script: - - pylint --max-line-length=120 $PWD/pyotb --disable=too-many-nested-blocks,too-many-locals,too-many-statements,too-few-public-methods,too-many-instance-attributes,too-many-arguments,invalid-name,fixme,too-many-return-statements,too-many-lines,too-many-branches,import-outside-toplevel,wrong-import-position,wrong-import-order,import-error,missing-class-docstring + - pylint $PWD/pyotb # ---------------------------------- Documentation ---------------------------------- diff --git a/pyproject.toml b/pyproject.toml index ddb1465..631be88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,65 @@ [build-system] -requires = [ - "setuptools>=42", - "wheel", - "numpy>=1.13,<1.23" -] +requires = ["setuptools >= 61.0", "wheel"] build-backend = "setuptools.build_meta" + +[project] +name = "pyotb" +description = "Library to enable easy use of the Orfeo ToolBox (OTB) in Python" +authors = [{name = "Rémi Cresson", email = "remi.cresson@inrae.fr"}, {name = "Nicolas Narçon"}, {name = "Vincent Delbar"}] +requires-python = ">=3.7" +keywords = ["gis", "remote sensing", "otb", "orfeotoolbox", "orfeo toolbox"] +dependencies = ["numpy>=1.16"] +readme = "README.md" +license = { file = "LICENSE" } +dynamic = ["version"] +classifiers=[ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Scientific/Engineering :: GIS", + "Topic :: Scientific/Engineering :: Image Processing", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] + +[project.optional-dependencies] +dev = ["pytest", "pylint", "codespell", "pydocstyle", "tomli"] + +[project.urls] +documentation = "https://pyotb.readthedocs.io" +homepage = "https://github.com/orfeotoolbox/pyotb" +repository = "https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb" + +[tool.setuptools] +packages = ["pyotb"] + +[tool.setuptools.dynamic] +version = {attr = "pyotb.__version__"} + +[tool.pylint] +max-line-length = 120 +disable = [ + "fixme", + "import-error", + "import-outside-toplevel", + "wrong-import-position", + "wrong-import-order", + "invalid-name", + "too-many-nested-blocks", + "too-many-locals", + "too-many-statements", + "too-many-instance-attributes", + "too-many-arguments", + "too-many-return-statements", + "too-many-lines", + "too-many-branches", +] + +[tool.pydocstyle] +convention = "google" + +[tool.black] +line-length = 120 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 778d274..0000000 --- a/setup.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[metadata] -description-file=README.md - diff --git a/setup.py b/setup.py index 519ba5d..6068493 100644 --- a/setup.py +++ b/setup.py @@ -1,31 +1,3 @@ -# -*- coding: utf-8 -*- -import setuptools +from setuptools import setup -with open("README.md", "r", encoding="utf-8") as fh: - long_description = fh.read() - -setuptools.setup( - name="pyotb", - version="1.5.4", - author="Nicolas Narçon", - author_email="nicolas.narcon@gmail.com", - description="Library to enable easy use of the Orfeo Tool Box (OTB) in Python", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/", - classifiers=[ - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Topic :: Scientific/Engineering :: GIS", - "Topic :: Scientific/Engineering :: Image Processing", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - ], - packages=setuptools.find_packages(), - python_requires=">=3.6", - keywords="remote sensing, otb, orfeotoolbox, orfeo toolbox", -) -#package_dir={"": "src"}, +setup() -- GitLab From 9733ddbae069f30c6be5eb3c93999d0e18d9ba16 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 13 Jan 2023 14:27:45 +0100 Subject: [PATCH 035/399] ENH: add property elapsed time --- pyotb/apps.py | 8 ++++++-- pyotb/core.py | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pyotb/apps.py b/pyotb/apps.py index a2af7ed..b63da8a 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -29,7 +29,7 @@ def get_available_applications(as_subprocess: bool = False) -> list[str]: env = os.environ.copy() if "PYTHONPATH" not in env: env["PYTHONPATH"] = "" - env["PYTHONPATH"] = ":" + str(Path(otb.__file__).parent) + env["PYTHONPATH"] += ":" + str(Path(otb.__file__).parent) env["OTB_LOGGER_LEVEL"] = "CRITICAL" # in order to suppress warnings while listing applications pycmd = "import otbApplication; print(otbApplication.Registry.GetAvailableApplications())" cmd_args = [sys.executable, "-c", pycmd] @@ -49,7 +49,7 @@ def get_available_applications(as_subprocess: bool = False) -> list[str]: except (ValueError, SyntaxError, AssertionError): logger.debug("Failed to decode output or convert to tuple:\nstdout=%s\nstderr=%s", stdout, stderr) if not app_list: - logger.info("Failed to list applications in an independent process. Falling back to local python import") + logger.debug("Failed to list applications in an independent process. Falling back to local python import") # Find applications using the normal way if not app_list: app_list = otb.Registry.GetAvailableApplications() @@ -68,6 +68,10 @@ class App(OTBObject): super().__init__(*args, **kwargs) self.description = self.app.GetDocLongDescription() + def elapsed_time(self): + """Get elapsed time between app init and end of exec or file writing.""" + return self.time_end - self.time_start + @property def used_outputs(self) -> list[str]: """List of used application outputs.""" diff --git a/pyotb/core.py b/pyotb/core.py index 88496b6..c3649b5 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -4,6 +4,7 @@ from __future__ import annotations from typing import Any from pathlib import Path from ast import literal_eval +from time import perf_counter import numpy as np import otbApplication as otb @@ -44,7 +45,7 @@ class OTBObject: create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication self.app = create(name) self.parameters_keys = tuple(self.app.GetParametersKeys()) - # Output parameters types + self.time_start, self.time_end = 0, 0 self.all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys} self.out_param_types = {k: v for k, v in self.all_param_types.items() if v in (otb.ParameterType_OutputImage, @@ -191,11 +192,13 @@ class OTBObject: def execute(self): """Execute and write to disk if any output parameter has been set during init.""" logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters) + self.time_start = perf_counter() try: self.app.Execute() except (RuntimeError, FileNotFoundError) as e: raise Exception(f"{self.name}: error during during app execution") from e self.frozen = False + self.time_end = perf_counter() logger.debug("%s: execution ended", self.name) self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics @@ -207,6 +210,7 @@ class OTBObject: except RuntimeError: logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name) self.app.ExecuteAndWriteOutput() + self.time_end = perf_counter() def write(self, *args, filename_extension: str = "", pixel_type: dict[str, str] | str = None, preserve_dtype: bool = False, **kwargs): -- GitLab From a56158153fac1865a3e6d0c40a0fd9e040952331 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 13 Jan 2023 14:27:55 +0100 Subject: [PATCH 036/399] FIX: import error help message --- pyotb/helpers.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index d38dfb3..ebccc48 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -294,16 +294,12 @@ def __suggest_fix_import(error_message: str, prefix: str): logger.critical("You may need to install cmake in order to recompile python bindings") else: logger.critical("Unable to automatically locate python dynamic library of %s", sys.executable) - return elif sys.platform == "win32": if error_message.startswith("DLL load failed"): if sys.version_info.minor != 7: logger.critical("You need Python 3.5 (OTB releases 6.4 to 7.4) or Python 3.7 (since OTB 8)") - issue_link = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2010" - logger.critical("Another workaround is to recompile Python bindings with cmake, see %s", issue_link) else: logger.critical("It seems that your env variables aren't properly set," " first use 'call otbenv.bat' then try to import pyotb once again") - return docs_link = "https://www.orfeo-toolbox.org/CookBook/Installation.html" logger.critical("You can verify installation requirements for your OS at %s", docs_link) -- GitLab From 06c983d4be574cb6d0c6f0ffe65a94cb059d90d7 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 13 Jan 2023 14:28:37 +0100 Subject: [PATCH 037/399] ENH: add properties data and metadata --- pyotb/core.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pyotb/core.py b/pyotb/core.py index c3649b5..f2fe4a8 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -85,6 +85,22 @@ class OTBObject: """Get the name of first output image parameter.""" return self.get_first_key(param_types=[otb.ParameterType_OutputImage]) + @property + def data(self): + """Expose app's output data values in a dictionary.""" + skip_keys = tuple(self.out_param_types) + tuple(self.parameters) + ("ram", "elev.default") + keys = (k for k in self.parameters_keys if k not in skip_keys) + def _check(v): + return not isinstance(v, otb.ApplicationProxy) and v not in ("", None, [], ()) + return {str(k): self[k] for k in keys if _check(self[k])} + + @property + def metadata(self): + if not self.key_output_image: + raise TypeError(f"{self.name}: this application has no raster output") + return dict(self.app.GetMetadataDictionary(self.key_output_image)) + + @property def dtype(self) -> np.dtype: """Expose the pixel type of output image using numpy convention. @@ -186,6 +202,11 @@ class OTBObject: # Convert output param path to Output object if key in self.out_param_types: value = Output(self, key, value) + elif isinstance(key, str): + try: + value = literal_eval(value) + except (ValueError, SyntaxError): + pass # Save attribute setattr(self, key, value) -- GitLab From f066af5d22464f98f68930538aa5074de9f7eae6 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 13 Jan 2023 14:28:54 +0100 Subject: [PATCH 038/399] ENH: add functions get_infos and get_statistics --- pyotb/core.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pyotb/core.py b/pyotb/core.py index f2fe4a8..dc02141 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -322,6 +322,16 @@ class OTBObject: for key in keys: self.app.SetParameterOutputImagePixelType(key, dtype) + def get_infos(self): + if not self.key_output_image: + raise TypeError(f"{self.name}: this application has no raster output") + return OTBObject("ReadImageInfo", self, quiet=True).data + + def get_statistics(self): + if not self.key_output_image: + raise TypeError(f"{self.name}: this application has no raster output") + return OTBObject("ComputeImagesStatistics", self, quiet=True).data + def read_values_at_coords(self, row: int, col: int, bands: int = None) -> list[int | float] | int | float: """Get pixel value(s) at a given YX coordinates. -- GitLab From 408d20d7acf0c54b84566a62db24426cba410c98 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 13 Jan 2023 14:44:31 +0100 Subject: [PATCH 039/399] ENH: add tests for new features + reorganize --- tests/test_core.py | 58 ++++++++++++++++++++++++++++++--------------- tests/test_numpy.py | 4 ++++ 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 2eadef7..436f99e 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -8,6 +8,12 @@ import pytest FILEPATH = os.environ["TEST_INPUT_IMAGE"] INPUT = pyotb.Input(FILEPATH) +TEST_IMAGE_STATS = { + 'out.mean': [79.5505, 109.225, 115.456, 249.349], + 'out.min': [33, 64, 91, 47], + 'out.max': [255, 255, 230, 255], + 'out.std': [51.0754, 35.3152, 23.4514, 20.3827] +} # Input settings @@ -20,7 +26,7 @@ def test_wrong_key(): pyotb.BandMath(INPUT, expression="im1b1") -# OTBObject's properties +# OTBObject properties def test_key_input(): assert INPUT.key_input == INPUT.key_input_image == "in" @@ -41,10 +47,39 @@ def test_transform(): assert INPUT.transform == (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0) +def test_data(): + assert pyotb.ComputeImagesStatistics(INPUT).data == TEST_IMAGE_STATS + + +def test_metadata(): + assert INPUT.metadata["Metadata_1"] == "TIFFTAG_SOFTWARE=CSinG - 13 SEPTEMBRE 2012" + + def test_nonraster_property(): with pytest.raises(TypeError): pyotb.ReadImageInfo(INPUT).dtype +def test_elapsed_time(): + assert pyotb.ReadImageInfo(INPUT).elapsed_time < 1 + +# Other functions +def test_get_infos(): + infos = INPUT.get_infos() + assert (infos["sizex"], infos["sizey"]) == (251, 304) + + +def test_get_statistics(): + assert INPUT.get_statistics() == TEST_IMAGE_STATS + + +def test_xy_to_rowcol(): + assert INPUT.xy_to_rowcol(760101, 6945977) == (19, 7) + + +def test_write(): + INPUT.write("/tmp/missing_dir/test_write.tif") + assert INPUT.out.exists() + # Slicer def test_slicer_shape(): @@ -85,7 +120,7 @@ def test_binary_mask_where(): assert res.exp == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)" -# Apps +# Essential apps def test_app_readimageinfo(): info = pyotb.ReadImageInfo(INPUT, quiet=True) assert (info.sizex, info.sizey) == (251, 304) @@ -94,12 +129,12 @@ def test_app_readimageinfo(): def test_app_computeimagestats(): stats = pyotb.ComputeImagesStatistics([INPUT], quiet=True) - assert stats["out.min"] == "[33, 64, 91, 47]" + assert stats["out.min"] == TEST_IMAGE_STATS["out.min"] def test_app_computeimagestats_sliced(): slicer_stats = pyotb.ComputeImagesStatistics(il=[INPUT[:10, :10, 0]], quiet=True) - assert slicer_stats["out.min"] == "[180]" + assert slicer_stats["out.min"] == [180] def test_read_values_at_coords(): @@ -107,21 +142,6 @@ def test_read_values_at_coords(): assert INPUT[10, 20, :] == [207, 192, 172, 255] -# XY => RowCol -def test_xy_to_rowcol(): - assert INPUT.xy_to_rowcol(760101, 6945977) == (19, 7) - - -def test_pixel_coords_numpy_equals_otb(): - assert INPUT[19,7] == list(INPUT.to_numpy()[19,7]) - - -# Create dir before write -def test_write(): - INPUT.write("/tmp/missing_dir/test_write.tif") - assert INPUT.out.filepath.exists() - - # BandMath NDVI == RadiometricIndices NDVI ? def test_ndvi_comparison(): ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (INPUT[:, :, -1] + INPUT[:, :, 0]) diff --git a/tests/test_numpy.py b/tests/test_numpy.py index 4240cb9..dd33425 100644 --- a/tests/test_numpy.py +++ b/tests/test_numpy.py @@ -36,6 +36,10 @@ def test_convert_to_array(): assert INPUT.shape == array.shape +def test_pixel_coords_otb_equals_numpy(): + assert INPUT[19,7] == list(INPUT.to_numpy()[19,7]) + + def test_add_noise_array(): white_noise = np.random.normal(0, 50, size=INPUT.shape) noisy_image = INPUT + white_noise -- GitLab From 8390bd43aa8387ef28cdd8bda9fc0726d39b56c3 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 13 Jan 2023 14:49:23 +0100 Subject: [PATCH 040/399] FIX: missing property decorator --- pyotb/apps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyotb/apps.py b/pyotb/apps.py index b63da8a..5092fee 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -68,6 +68,7 @@ class App(OTBObject): super().__init__(*args, **kwargs) self.description = self.app.GetDocLongDescription() + @property def elapsed_time(self): """Get elapsed time between app init and end of exec or file writing.""" return self.time_end - self.time_start -- GitLab From 0b6d767a4e3ccb02de3294acc990df1cd9f25d8f Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 13 Jan 2023 15:02:40 +0100 Subject: [PATCH 041/399] STYLE: linting and docstrings --- pyotb/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index dc02141..d4f32c0 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -90,17 +90,18 @@ class OTBObject: """Expose app's output data values in a dictionary.""" skip_keys = tuple(self.out_param_types) + tuple(self.parameters) + ("ram", "elev.default") keys = (k for k in self.parameters_keys if k not in skip_keys) + def _check(v): return not isinstance(v, otb.ApplicationProxy) and v not in ("", None, [], ()) return {str(k): self[k] for k in keys if _check(self[k])} @property def metadata(self): + """Return first output image metadata dictionary""" if not self.key_output_image: raise TypeError(f"{self.name}: this application has no raster output") return dict(self.app.GetMetadataDictionary(self.key_output_image)) - @property def dtype(self) -> np.dtype: """Expose the pixel type of output image using numpy convention. @@ -323,11 +324,13 @@ class OTBObject: self.app.SetParameterOutputImagePixelType(key, dtype) def get_infos(self): + """Return a dict output of ReadImageInfo for the first image output""" if not self.key_output_image: raise TypeError(f"{self.name}: this application has no raster output") return OTBObject("ReadImageInfo", self, quiet=True).data def get_statistics(self): + """Return a dict output of ComputeImagesStatistics for the first image output""" if not self.key_output_image: raise TypeError(f"{self.name}: this application has no raster output") return OTBObject("ComputeImagesStatistics", self, quiet=True).data -- GitLab From 5cfa66f08e6949b412fdd1981d95d1790017f4bd Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 13 Jan 2023 15:11:23 +0100 Subject: [PATCH 042/399] STYLE: docstrings --- pyotb/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index d4f32c0..6186ef2 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -97,7 +97,7 @@ class OTBObject: @property def metadata(self): - """Return first output image metadata dictionary""" + """Return first output image metadata dictionary.""" if not self.key_output_image: raise TypeError(f"{self.name}: this application has no raster output") return dict(self.app.GetMetadataDictionary(self.key_output_image)) @@ -324,13 +324,13 @@ class OTBObject: self.app.SetParameterOutputImagePixelType(key, dtype) def get_infos(self): - """Return a dict output of ReadImageInfo for the first image output""" + """Return a dict output of ReadImageInfo for the first image output.""" if not self.key_output_image: raise TypeError(f"{self.name}: this application has no raster output") return OTBObject("ReadImageInfo", self, quiet=True).data def get_statistics(self): - """Return a dict output of ComputeImagesStatistics for the first image output""" + """Return a dict output of ComputeImagesStatistics for the first image output.""" if not self.key_output_image: raise TypeError(f"{self.name}: this application has no raster output") return OTBObject("ComputeImagesStatistics", self, quiet=True).data -- GitLab From 359a813260b87ae8b2998f4bb43384f74c89634f Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 16 Jan 2023 13:36:48 +0100 Subject: [PATCH 043/399] FIX: check value is str before literal_eval --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 6186ef2..0a8a91c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -203,7 +203,7 @@ class OTBObject: # Convert output param path to Output object if key in self.out_param_types: value = Output(self, key, value) - elif isinstance(key, str): + elif isinstance(value, str): try: value = literal_eval(value) except (ValueError, SyntaxError): -- GitLab From 844465de41d79fcba15e87ce4884038f9155628e Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 16 Jan 2023 13:56:31 +0100 Subject: [PATCH 044/399] ENH: avoid some keys in data property --- pyotb/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 0a8a91c..ddaa62b 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -88,7 +88,8 @@ class OTBObject: @property def data(self): """Expose app's output data values in a dictionary.""" - skip_keys = tuple(self.out_param_types) + tuple(self.parameters) + ("ram", "elev.default") + skip_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem") + skip_keys = skip_keys + tuple(self.out_param_types) + tuple(self.parameters) keys = (k for k in self.parameters_keys if k not in skip_keys) def _check(v): -- GitLab From 41d372d4ebfc8f50369725a2f2b64024c931b0db Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 24 Jan 2023 11:15:04 +0100 Subject: [PATCH 045/399] CI: add missing pylint exception --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 631be88..2ef1c71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ disable = [ "too-many-instance-attributes", "too-many-arguments", "too-many-return-statements", + "too-many-public-methods", "too-many-lines", "too-many-branches", ] -- GitLab From bdde3adc1d26f6eea00a8c37649df52acb7c42e8 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 24 Jan 2023 14:37:14 +0100 Subject: [PATCH 046/399] FIX: Output object must inherit from OTBObject --- pyotb/core.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index ddaa62b..df585a8 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -556,11 +556,9 @@ class OTBObject: AttributeError: when `name` is not an attribute of self.app """ - try: - res = getattr(self.app, name) - return res - except AttributeError as e: - raise AttributeError(f"{self.name}: could not find attribute `{name}`") from e + if name in dir(self.app): + return getattr(self.app, name) + raise AttributeError(f"{self.name}: could not find attribute `{name}`") def __getitem__(self, key): """Override the default __getitem__ behaviour. @@ -1260,20 +1258,21 @@ class Input(OTBObject): return f"<pyotb.Input object from {self.path}>" -class Output: +class Output(OTBObject): """Object that behave like a pointer to a specific application output file.""" - def __init__(self, source_app: OTBObject, param_key: str, filepath: str = None, mkdir: bool = True): + def __init__(self, pyotb_app: OTBObject, param_key: str, filepath: str = None, mkdir: bool = True): # pylint: disable=super-init-not-called """Constructor for an Output object. Args: - source_app: The pyotb App to store reference from + pyotb_app: The pyotb App to store reference from param_key: Output parameter key of the target app filepath: path of the output file (if not in memory) mkdir: create missing parent directories """ - self.source_app = source_app + self.pyotb_app, self.app = pyotb_app, pyotb_app.app + self.parameters = pyotb_app.parameters self.param_key = param_key self.filepath = None if filepath: @@ -1282,7 +1281,12 @@ class Output: self.filepath = Path(filepath) if mkdir: self.make_parent_dirs() - self.name = f"Output {param_key} from {self.source_app.name}" + self.name = f"Output {param_key} from {self.pyotb_app.name}" + + @property + def key_output_image(self): + """Overwrite OTBObject prop, in order to use Operation special methods with the right Output param_key.""" + return self.param_key def exists(self) -> bool: """Check file exist.""" -- GitLab From d0922315221b8449d7b764f5572aeaa884ae2679 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 24 Jan 2023 14:45:55 +0100 Subject: [PATCH 047/399] STYLE: max line length --- pyotb/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index df585a8..acb09eb 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1261,7 +1261,8 @@ class Input(OTBObject): class Output(OTBObject): """Object that behave like a pointer to a specific application output file.""" - def __init__(self, pyotb_app: OTBObject, param_key: str, filepath: str = None, mkdir: bool = True): # pylint: disable=super-init-not-called + def __init__(self, pyotb_app: OTBObject, param_key: str, + filepath: str = None, mkdir: bool = True): # pylint: disable=super-init-not-called """Constructor for an Output object. Args: -- GitLab From 6c67f2fa091c97c11ae3dd577ac4f59f2ea3c154 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 24 Jan 2023 14:47:00 +0100 Subject: [PATCH 048/399] STYLE: pylint ignore on wrong line --- pyotb/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index acb09eb..c7ceb04 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1261,13 +1261,13 @@ class Input(OTBObject): class Output(OTBObject): """Object that behave like a pointer to a specific application output file.""" - def __init__(self, pyotb_app: OTBObject, param_key: str, - filepath: str = None, mkdir: bool = True): # pylint: disable=super-init-not-called + def __init__(self, pyotb_app: OTBObject, # pylint: disable=super-init-not-called + param_key: str, filepath: str = None, mkdir: bool = True): """Constructor for an Output object. Args: pyotb_app: The pyotb App to store reference from - param_key: Output parameter key of the target app + param_key: Output parameter key of the target app filepath: path of the output file (if not in memory) mkdir: create missing parent directories -- GitLab From 8ab686b2d2045d013d828f588b107f39d4da6550 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 24 Jan 2023 14:50:53 +0100 Subject: [PATCH 049/399] STYLE: whitespace --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index c7ceb04..69a0fd2 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1267,7 +1267,7 @@ class Output(OTBObject): Args: pyotb_app: The pyotb App to store reference from - param_key: Output parameter key of the target app + param_key: Output parameter key of the target app filepath: path of the output file (if not in memory) mkdir: create missing parent directories -- GitLab From c8d19a1a91ff7ca677952ceb6fd5d9912c43df45 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 24 Jan 2023 15:49:38 +0100 Subject: [PATCH 050/399] STYLE: remove useless pylint exceptions --- pyotb/apps.py | 4 ++-- pyotb/core.py | 2 +- pyotb/functions.py | 6 +++--- pyproject.toml | 16 +++++----------- 4 files changed, 11 insertions(+), 17 deletions(-) diff --git a/pyotb/apps.py b/pyotb/apps.py index 5092fee..ad71f54 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -3,9 +3,10 @@ from __future__ import annotations import os import sys +import subprocess from pathlib import Path -import otbApplication as otb +import otbApplication as otb # pylint: disable=import-error from .core import OTBObject from .helpers import logger @@ -34,7 +35,6 @@ def get_available_applications(as_subprocess: bool = False) -> list[str]: pycmd = "import otbApplication; print(otbApplication.Registry.GetAvailableApplications())" cmd_args = [sys.executable, "-c", pycmd] try: - import subprocess # pylint: disable=import-outside-toplevel params = {"env": env, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE} with subprocess.Popen(cmd_args, **params) as p: logger.debug('Exec "%s \'%s\'"', ' '.join(cmd_args[:-1]), pycmd) diff --git a/pyotb/core.py b/pyotb/core.py index 69a0fd2..6f0a4dc 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -7,7 +7,7 @@ from ast import literal_eval from time import perf_counter import numpy as np -import otbApplication as otb +import otbApplication as otb # pylint: disable=import-error from .helpers import logger diff --git a/pyotb/functions.py b/pyotb/functions.py index aa161e6..71c298b 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -4,8 +4,9 @@ from __future__ import annotations import inspect import os import sys -import textwrap import uuid +import textwrap +import subprocess from collections import Counter from .core import OTBObject, Input, Operation, LogicalOperation, get_nbchannels, Output @@ -217,7 +218,7 @@ def run_tf_function(func): """ try: - from .apps import TensorflowModelServe + from .apps import TensorflowModelServe # pylint: disable=import-outside-toplevel except ImportError: logger.error('Could not run Tensorflow function: failed to import TensorflowModelServe.' 'Check that you have OTBTF configured (https://github.com/remicres/otbtf#how-to-install)') @@ -303,7 +304,6 @@ def run_tf_function(func): pycmd = get_tf_pycmd(out_savedmodel, channels, scalar_inputs) cmd_args = [sys.executable, "-c", pycmd] try: - import subprocess subprocess.run(cmd_args, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) except subprocess.SubprocessError: logger.debug("Failed to call subprocess") diff --git a/pyproject.toml b/pyproject.toml index 2ef1c71..9538fc1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers=[ ] [project.optional-dependencies] -dev = ["pytest", "pylint", "codespell", "pydocstyle", "tomli"] +dev = ["pytest", "pylint", "codespell", "pydocstyle", "tomli"] [project.urls] documentation = "https://pyotb.readthedocs.io" @@ -43,20 +43,14 @@ version = {attr = "pyotb.__version__"} max-line-length = 120 disable = [ "fixme", - "import-error", - "import-outside-toplevel", - "wrong-import-position", - "wrong-import-order", "invalid-name", - "too-many-nested-blocks", + "too-many-lines", "too-many-locals", + "too-many-branches", "too-many-statements", - "too-many-instance-attributes", - "too-many-arguments", - "too-many-return-statements", "too-many-public-methods", - "too-many-lines", - "too-many-branches", + "too-many-instance-attributes", + "wrong-import-position", ] [tool.pydocstyle] -- GitLab From 8ad470c51fe96ef1a51ae9c3cb47b1f392690e0e Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 11:00:07 +0000 Subject: [PATCH 051/399] WIP: refac classes --- pyotb/core.py | 676 ++++++++++++++++++++++++++------------------------ 1 file changed, 349 insertions(+), 327 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 6f0a4dc..87bb08c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- """This module is the core of pyotb.""" from __future__ import annotations -from typing import Any -from pathlib import Path + from ast import literal_eval +from pathlib import Path from time import perf_counter +from typing import Any import numpy as np import otbApplication as otb # pylint: disable=import-error @@ -13,50 +14,30 @@ from .helpers import logger class OTBObject: - """Base class that gathers common operations for any OTB application.""" - - def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, **kwargs): - """Common constructor for OTB applications. Handles in-memory connection between apps. - - Args: - name: name of the app, e.g. 'BandMath' - *args: used for passing application parameters. Can be : - - dictionary containing key-arguments enumeration. Useful when a key is python-reserved - (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") - - string, App or Output, useful when the user wants to specify the input "in" - - list, useful when the user wants to specify the input list 'il' - frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___ - quiet: whether to print logs of the OTB app + def __init__(self, name: str, app: otb.Application, image_dic: dict = None): + """ + name: + app: image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as the result of app.ExportImage(). Use it when the app takes a numpy array as input. See this related issue for why it is necessary to keep reference of object: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824 - **kwargs: used for passing application parameters. - e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif' """ - self.parameters = {} self.name = name - self.frozen = frozen - self.quiet = quiet + self.app = app self.image_dic = image_dic - self.exports_dic = {} - create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication - self.app = create(name) + self.parameters_keys = tuple(self.app.GetParametersKeys()) - self.time_start, self.time_end = 0, 0 self.all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys} self.out_param_types = {k: v for k, v in self.all_param_types.items() if v in (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename)} - if args or kwargs: - self.set_parameters(*args, **kwargs) - if not self.frozen: - self.execute() - if any(key in self.parameters for key in self.out_param_types): - self.flush() # auto flush if any output param was provided during app init + + self.transform = None # TODO: value? + self.exports_dic = {} def get_first_key(self, param_types: list[str]) -> str: """Get the first output param key for specific file types.""" @@ -85,17 +66,6 @@ class OTBObject: """Get the name of first output image parameter.""" return self.get_first_key(param_types=[otb.ParameterType_OutputImage]) - @property - def data(self): - """Expose app's output data values in a dictionary.""" - skip_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem") - skip_keys = skip_keys + tuple(self.out_param_types) + tuple(self.parameters) - keys = (k for k in self.parameters_keys if k not in skip_keys) - - def _check(v): - return not isinstance(v, otb.ApplicationProxy) and v not in ("", None, [], ()) - return {str(k): self[k] for k in keys if _check(self[k])} - @property def metadata(self): """Return first output image metadata dictionary.""" @@ -145,185 +115,6 @@ class OTBObject: origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2 return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y - def set_parameters(self, *args, **kwargs): - """Set some parameters of the app. - - When useful, e.g. for images list, this function appends the parameters - instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths - - Args: - *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved - (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") - - string, App or Output, useful when the user implicitly wants to set the param "in" - - list, useful when the user implicitly wants to set the param "il" - **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif' - - Raises: - Exception: when the setting of a parameter failed - - """ - parameters = kwargs - parameters.update(self.__parse_args(args)) - # Going through all arguments - for key, obj in parameters.items(): - if key not in self.parameters_keys: - raise KeyError(f'{self.name}: unknown parameter name "{key}"') - # When the parameter expects a list, if needed, change the value to list - if is_key_list(self, key) and not isinstance(obj, (list, tuple)): - obj = [obj] - logger.info('%s: argument for parameter "%s" was converted to list', self.name, key) - try: - # This is when we actually call self.app.SetParameter* - self.__set_param(key, obj) - except (RuntimeError, TypeError, ValueError, KeyError) as e: - raise Exception( - f"{self.name}: something went wrong before execution " - f"(while setting parameter '{key}' to '{obj}')" - ) from e - # Update _parameters using values from OtbApplication object - otb_params = self.app.GetParameters().items() - otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params} - # Update param dict and save values as object attributes - self.parameters.update({**parameters, **otb_params}) - self.save_objects() - - def save_objects(self): - """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`. - - This is useful when the key contains reserved characters such as a point eg "io.out" - """ - for key in self.parameters_keys: - if key in dir(self.__class__): - continue # skip forbidden attribute since it is already used by the class - value = self.parameters.get(key) # basic parameters - if value is None: - try: - value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results) - except RuntimeError: - continue # this is when there is no value for key - # Convert output param path to Output object - if key in self.out_param_types: - value = Output(self, key, value) - elif isinstance(value, str): - try: - value = literal_eval(value) - except (ValueError, SyntaxError): - pass - # Save attribute - setattr(self, key, value) - - def execute(self): - """Execute and write to disk if any output parameter has been set during init.""" - logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters) - self.time_start = perf_counter() - try: - self.app.Execute() - except (RuntimeError, FileNotFoundError) as e: - raise Exception(f"{self.name}: error during during app execution") from e - self.frozen = False - self.time_end = perf_counter() - logger.debug("%s: execution ended", self.name) - self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics - - def flush(self): - """Flush data to disk, this is when WriteOutput is actually called.""" - try: - logger.debug("%s: flushing data to disk", self.name) - self.app.WriteOutput() - except RuntimeError: - logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name) - self.app.ExecuteAndWriteOutput() - self.time_end = perf_counter() - - def write(self, *args, filename_extension: str = "", pixel_type: dict[str, str] | str = None, - preserve_dtype: bool = False, **kwargs): - """Set output pixel type and write the output raster files. - - Args: - *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key contains - non-standard characters such as a point, e.g. {'io.out':'output.tif'} - - string, useful when there is only one output, e.g. 'output.tif' - - None if output file was passed during App init - filename_extension: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES") - Will be used for all outputs (Default value = "") - pixel_type: Can be : - dictionary {output_parameter_key: pixeltype} when specifying for several outputs - - str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several - outputs, all outputs are written with this unique type. - Valid pixel types are uint8, uint16, uint32, int16, int32, float, double, - cint16, cint32, cfloat, cdouble. (Default value = None) - preserve_dtype: propagate main input pixel type to outputs, in case pixel_type is None - **kwargs: keyword arguments e.g. out='output.tif' - - """ - # Gather all input arguments in kwargs dict - for arg in args: - if isinstance(arg, dict): - kwargs.update(arg) - elif isinstance(arg, str) and kwargs: - logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg) - elif isinstance(arg, str) and self.key_output_image: - kwargs.update({self.key_output_image: arg}) - - # Append filename extension to filenames - if filename_extension: - logger.debug("%s: using extended filename for outputs: %s", self.name, filename_extension) - if not filename_extension.startswith("?"): - filename_extension = "?" + filename_extension - for key, value in kwargs.items(): - if self.out_param_types[key] == otb.ParameterType_OutputImage and '?' not in value: - kwargs[key] = value + filename_extension - - # Manage output pixel types - dtypes = {} - if pixel_type: - if isinstance(pixel_type, str): - type_name = self.app.ConvertPixelTypeToNumpy(parse_pixel_type(pixel_type)) - logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name) - for key in kwargs: - if self.out_param_types[key] == otb.ParameterType_OutputImage: - dtypes[key] = parse_pixel_type(pixel_type) - elif isinstance(pixel_type, dict): - dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()} - elif preserve_dtype: - self.propagate_dtype() # all outputs will have the same type as the main input raster - - # Set parameters and flush to disk - for key, output_filename in kwargs.items(): - if key in dtypes: - self.propagate_dtype(key, dtypes[key]) - self.set_parameters({key: output_filename}) - self.flush() - - def propagate_dtype(self, target_key: str = None, dtype: int = None): - """Propagate a pixel type from main input to every outputs, or to a target output key only. - - With multiple inputs (if dtype is not provided), the type of the first input is considered. - With multiple outputs (if target_key is not provided), all outputs will be converted to the same pixel type. - - Args: - target_key: output param key to change pixel type - dtype: data type to use - - """ - if not dtype: - param = self.parameters.get(self.key_input_image) - if not param: - logger.warning("%s: could not propagate pixel type from inputs to output", self.name) - return - if isinstance(param, (list, tuple)): - param = param[0] # first image in "il" - try: - dtype = get_pixel_type(param) - except (TypeError, RuntimeError): - logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param) - return - if target_key: - keys = [target_key] - else: - keys = [k for k, v in self.out_param_types.items() if v == otb.ParameterType_OutputImage] - for key in keys: - self.app.SetParameterOutputImagePixelType(key, dtype) - def get_infos(self): """Return a dict output of ReadImageInfo for the first image output.""" if not self.key_output_image: @@ -473,87 +264,31 @@ class OTBObject: row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x return abs(int(row)), int(col) - # Private functions - def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]: - """Gather all input arguments in kwargs dict. + # Special functions + def __hash__(self): + """Override the default behaviour of the hash function. + + Returns: + self hash + + """ + return id(self) + + def __getattr__(self, name): + """This method is called when the default attribute access fails. + + We choose to access the attribute `name` of self.app. + Thus, any method of otbApplication can be used transparently on OTBObject objects, + e.g. SetParameterOutputImagePixelType() or ExportImage() work Args: - args: the list of arguments passed to set_parameters() + name: attribute name Returns: - a dictionary with the right keyword depending on the object + attribute - """ - kwargs = {} - for arg in args: - if isinstance(arg, dict): - kwargs.update(arg) - elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.key_input): - kwargs.update({self.key_input: arg}) - return kwargs - - def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]): - """Set one parameter, decide which otb.Application method to use depending on target object.""" - if obj is None or (isinstance(obj, (list, tuple)) and not obj): - self.app.ClearValue(key) - return - if key not in self.parameters_keys: - raise KeyError( - f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}" - ) - # Single-parameter cases - if isinstance(obj, OTBObject): - self.app.ConnectImage(key, obj.app, obj.key_output_image) - elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB - self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0]) - elif key == "ram": # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200 - self.app.SetParameterInt("ram", int(obj)) - elif not isinstance(obj, list): # any other parameters (str, int...) - self.app.SetParameterValue(key, obj) - # Images list - elif is_key_images_list(self, key): - # To enable possible in-memory connections, we go through the list and set the parameters one by one - for inp in obj: - if isinstance(inp, OTBObject): - self.app.ConnectImage(key, inp.app, inp.key_output_image) - elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB - self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0]) - else: # here `input` should be an image filepath - # Append `input` to the list, do not overwrite any previously set element of the image list - self.app.AddParameterStringList(key, inp) - # List of any other types (str, int...) - else: - self.app.SetParameterValue(key, obj) - - # Special functions - def __hash__(self): - """Override the default behaviour of the hash function. - - Returns: - self hash - - """ - return id(self) - - def __str__(self): - """Return a nice string representation with object id.""" - return f"<pyotb.App {self.name} object id {id(self)}>" - - def __getattr__(self, name): - """This method is called when the default attribute access fails. - - We choose to access the attribute `name` of self.app. - Thus, any method of otbApplication can be used transparently on OTBObject objects, - e.g. SetParameterOutputImagePixelType() or ExportImage() work - - Args: - name: attribute name - - Returns: - attribute - - Raises: - AttributeError: when `name` is not an attribute of self.app + Raises: + AttributeError: when `name` is not an attribute of self.app """ if name in dir(self.app): @@ -598,7 +333,7 @@ class OTBObject: key = key + (slice(None, None, None),) return Slicer(self, *key) - def __add__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __add__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default addition and flavours it with BandMathX. Args: @@ -612,7 +347,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("+", self, other) - def __sub__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __sub__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default subtraction and flavours it with BandMathX. Args: @@ -626,7 +361,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("-", self, other) - def __mul__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __mul__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default subtraction and flavours it with BandMathX. Args: @@ -640,7 +375,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("*", self, other) - def __truediv__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __truediv__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default subtraction and flavours it with BandMathX. Args: @@ -654,7 +389,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("/", self, other) - def __radd__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __radd__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default reverse addition and flavours it with BandMathX. Args: @@ -668,7 +403,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("+", other, self) - def __rsub__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __rsub__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default subtraction and flavours it with BandMathX. Args: @@ -682,7 +417,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("-", other, self) - def __rmul__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __rmul__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default multiplication and flavours it with BandMathX. Args: @@ -696,7 +431,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("*", other, self) - def __rtruediv__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __rtruediv__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default division and flavours it with BandMathX. Args: @@ -719,7 +454,7 @@ class OTBObject: """ return Operation("abs", self) - def __ge__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __ge__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default greater or equal and flavours it with BandMathX. Args: @@ -733,7 +468,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation(">=", self, other) - def __le__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __le__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default less or equal and flavours it with BandMathX. Args: @@ -747,7 +482,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation("<=", self, other) - def __gt__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __gt__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default greater operator and flavours it with BandMathX. Args: @@ -761,7 +496,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation(">", self, other) - def __lt__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __lt__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default less operator and flavours it with BandMathX. Args: @@ -775,7 +510,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation("<", self, other) - def __eq__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __eq__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default eq operator and flavours it with BandMathX. Args: @@ -789,7 +524,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation("==", self, other) - def __ne__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __ne__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default different operator and flavours it with BandMathX. Args: @@ -803,7 +538,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation("!=", self, other) - def __or__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __or__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default or operator and flavours it with BandMathX. Args: @@ -817,7 +552,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation("||", self, other) - def __and__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __and__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default and operator and flavours it with BandMathX. Args: @@ -890,10 +625,300 @@ class OTBObject: return NotImplemented -class Slicer(OTBObject): +class App(OTBObject): + """Base class that gathers common operations for any OTB application.""" + + def __init__(self, otb_app_name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, + **kwargs): + """Common constructor for OTB applications. Handles in-memory connection between apps. + + Args: + otb_app_name: name of the OTB application, e.g. 'BandMath' + *args: used for passing application parameters. Can be : + - dictionary containing key-arguments enumeration. Useful when a key is python-reserved + (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") + - string or OTBObject, useful when the user wants to specify the input "in" + - list, useful when the user wants to specify the input list 'il' + frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___ + quiet: whether to print logs of the OTB app + image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as + the result of app.ExportImage(). Use it when the app takes a numpy array as input. + See this related issue for why it is necessary to keep reference of object: + https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824 + **kwargs: used for passing application parameters. + e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif' + + """ + + self.frozen = frozen + self.quiet = quiet + create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication + super().__init__(name=f"OTB Application {otb_app_name}", app=create(otb_app_name), image_dic=image_dic) + + # Set parameters + self.parameters = {} + if args or kwargs: + self.set_parameters(*args, **kwargs) + if not self.frozen: + self.execute() + if any(key in self.parameters for key in self.out_param_types): + self.flush() # auto flush if any output param was provided during app init + + # Elapsed time + self.time_start, self.time_end = 0, 0 + + @property + def data(self): + """Expose app's output data values in a dictionary.""" + skip_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem") + skip_keys = skip_keys + tuple(self.out_param_types) + tuple(self.parameters) + keys = (k for k in self.parameters_keys if k not in skip_keys) + + def _check(v): + return not isinstance(v, otb.ApplicationProxy) and v not in ("", None, [], ()) + + return {str(k): self[k] for k in keys if _check(self[k])} + + def set_parameters(self, *args, **kwargs): + """Set some parameters of the app. + + When useful, e.g. for images list, this function appends the parameters + instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths + + Args: + *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved + (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") + - string or OTBObject, useful when the user implicitly wants to set the param "in" + - list, useful when the user implicitly wants to set the param "il" + **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif' + + Raises: + Exception: when the setting of a parameter failed + + """ + parameters = kwargs + parameters.update(self.__parse_args(args)) + # Going through all arguments + for key, obj in parameters.items(): + if key not in self.parameters_keys: + raise KeyError(f'{self.name}: unknown parameter name "{key}"') + # When the parameter expects a list, if needed, change the value to list + if is_key_list(self, key) and not isinstance(obj, (list, tuple)): + obj = [obj] + logger.info('%s: argument for parameter "%s" was converted to list', self.name, key) + try: + # This is when we actually call self.app.SetParameter* + self.__set_param(key, obj) + except (RuntimeError, TypeError, ValueError, KeyError) as e: + raise Exception( + f"{self.name}: something went wrong before execution " + f"(while setting parameter '{key}' to '{obj}')" + ) from e + # Update _parameters using values from OtbApplication object + otb_params = self.app.GetParameters().items() + otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params} + # Update param dict and save values as object attributes + self.parameters.update({**parameters, **otb_params}) + self.save_objects() + + def save_objects(self): + """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`. + + This is useful when the key contains reserved characters such as a point eg "io.out" + """ + for key in self.parameters_keys: + if key in dir(self.__class__): + continue # skip forbidden attribute since it is already used by the class + value = self.parameters.get(key) # basic parameters + if value is None: + try: + value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results) + except RuntimeError: + continue # this is when there is no value for key + # Convert output param path to Output object + if key in self.out_param_types: + value = Output(self, key, value) + elif isinstance(value, str): + try: + value = literal_eval(value) + except (ValueError, SyntaxError): + pass + # Save attribute + setattr(self, key, value) + + def execute(self): + """Execute and write to disk if any output parameter has been set during init.""" + logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters) + self.time_start = perf_counter() + try: + self.app.Execute() + except (RuntimeError, FileNotFoundError) as e: + raise Exception(f"{self.name}: error during during app execution") from e + self.frozen = False + self.time_end = perf_counter() + logger.debug("%s: execution ended", self.name) + self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics + + def flush(self): + """Flush data to disk, this is when WriteOutput is actually called.""" + try: + logger.debug("%s: flushing data to disk", self.name) + self.app.WriteOutput() + except RuntimeError: + logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name) + self.app.ExecuteAndWriteOutput() + self.time_end = perf_counter() + + def write(self, *args, filename_extension: str = "", pixel_type: dict[str, str] | str = None, + preserve_dtype: bool = False, **kwargs): + """Set output pixel type and write the output raster files. + + Args: + *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key contains + non-standard characters such as a point, e.g. {'io.out':'output.tif'} + - string, useful when there is only one output, e.g. 'output.tif' + - None if output file was passed during App init + filename_extension: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES") + Will be used for all outputs (Default value = "") + pixel_type: Can be : - dictionary {output_parameter_key: pixeltype} when specifying for several outputs + - str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several + outputs, all outputs are written with this unique type. + Valid pixel types are uint8, uint16, uint32, int16, int32, float, double, + cint16, cint32, cfloat, cdouble. (Default value = None) + preserve_dtype: propagate main input pixel type to outputs, in case pixel_type is None + **kwargs: keyword arguments e.g. out='output.tif' + + """ + # Gather all input arguments in kwargs dict + for arg in args: + if isinstance(arg, dict): + kwargs.update(arg) + elif isinstance(arg, str) and kwargs: + logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg) + elif isinstance(arg, str) and self.key_output_image: + kwargs.update({self.key_output_image: arg}) + + # Append filename extension to filenames + if filename_extension: + logger.debug("%s: using extended filename for outputs: %s", self.name, filename_extension) + if not filename_extension.startswith("?"): + filename_extension = "?" + filename_extension + for key, value in kwargs.items(): + if self.out_param_types[key] == otb.ParameterType_OutputImage and '?' not in value: + kwargs[key] = value + filename_extension + + # Manage output pixel types + dtypes = {} + if pixel_type: + if isinstance(pixel_type, str): + type_name = self.app.ConvertPixelTypeToNumpy(parse_pixel_type(pixel_type)) + logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name) + for key in kwargs: + if self.out_param_types[key] == otb.ParameterType_OutputImage: + dtypes[key] = parse_pixel_type(pixel_type) + elif isinstance(pixel_type, dict): + dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()} + elif preserve_dtype: + self.propagate_dtype() # all outputs will have the same type as the main input raster + + # Set parameters and flush to disk + for key, output_filename in kwargs.items(): + if key in dtypes: + self.propagate_dtype(key, dtypes[key]) + self.set_parameters({key: output_filename}) + self.flush() + + def propagate_dtype(self, target_key: str = None, dtype: int = None): + """Propagate a pixel type from main input to every outputs, or to a target output key only. + + With multiple inputs (if dtype is not provided), the type of the first input is considered. + With multiple outputs (if target_key is not provided), all outputs will be converted to the same pixel type. + + Args: + target_key: output param key to change pixel type + dtype: data type to use + + """ + if not dtype: + param = self.parameters.get(self.key_input_image) + if not param: + logger.warning("%s: could not propagate pixel type from inputs to output", self.name) + return + if isinstance(param, (list, tuple)): + param = param[0] # first image in "il" + try: + dtype = get_pixel_type(param) + except (TypeError, RuntimeError): + logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param) + return + if target_key: + keys = [target_key] + else: + keys = [k for k, v in self.out_param_types.items() if v == otb.ParameterType_OutputImage] + for key in keys: + self.app.SetParameterOutputImagePixelType(key, dtype) + + # Private functions + def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]: + """Gather all input arguments in kwargs dict. + + Args: + args: the list of arguments passed to set_parameters() + + Returns: + a dictionary with the right keyword depending on the object + + """ + kwargs = {} + for arg in args: + if isinstance(arg, dict): + kwargs.update(arg) + elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.key_input): + kwargs.update({self.key_input: arg}) + return kwargs + + def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]): + """Set one parameter, decide which otb.Application method to use depending on target object.""" + if obj is None or (isinstance(obj, (list, tuple)) and not obj): + self.app.ClearValue(key) + return + if key not in self.parameters_keys: + raise KeyError( + f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}" + ) + # Single-parameter cases + if isinstance(obj, OTBObject): + self.app.ConnectImage(key, obj.app, obj.key_output_image) + elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB + self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0]) + elif key == "ram": # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200 + self.app.SetParameterInt("ram", int(obj)) + elif not isinstance(obj, list): # any other parameters (str, int...) + self.app.SetParameterValue(key, obj) + # Images list + elif is_key_images_list(self, key): + # To enable possible in-memory connections, we go through the list and set the parameters one by one + for inp in obj: + if isinstance(inp, OTBObject): + self.app.ConnectImage(key, inp.app, inp.key_output_image) + elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB + self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0]) + else: # here `input` should be an image filepath + # Append `input` to the list, do not overwrite any previously set element of the image list + self.app.AddParameterStringList(key, inp) + # List of any other types (str, int...) + else: + self.app.SetParameterValue(key, obj) + + def __str__(self): + """Return a nice string representation with object id.""" + return f"<pyotb.App {self.name} object id {id(self)}>" + + +class Slicer(App): """Slicer objects i.e. when we call something like raster[:, :, 2] from Python.""" - def __init__(self, obj: OTBObject | Output | str, rows: int, cols: int, channels: int): + def __init__(self, obj: OTBObject | str, rows: int, cols: int, channels: int): """Create a slicer object, that can be used directly for writing or inside a BandMath. It contains : @@ -957,7 +982,7 @@ class Slicer(OTBObject): self.execute() -class Operation(OTBObject): +class Operation(App): """Class for arithmetic/math operations done in Python. Example: @@ -986,7 +1011,7 @@ class Operation(OTBObject): Args: operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? - *inputs: inputs. Can be App, Output, Input, Operation, Slicer, filepath, int or float + *inputs: inputs. Can be OTBObject, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ @@ -1018,7 +1043,7 @@ class Operation(OTBObject): super().__init__(name, il=self.unique_inputs, exp=self.exp, quiet=True) self.name = f'Operation exp="{self.exp}"' - def create_fake_exp(self, operator: str, inputs: list[OTBObject | Output | str | int | float], + def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None): """Create a 'fake' expression. @@ -1026,7 +1051,7 @@ class Operation(OTBObject): Args: operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? - inputs: inputs. Can be App, Output, Input, Operation, Slicer, filepath, int or float + inputs: inputs. Can be OTBObject, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ @@ -1110,7 +1135,7 @@ class Operation(OTBObject): return exp_bands, exp @staticmethod - def create_one_input_fake_exp(x: OTBObject | Output | str, + def create_one_input_fake_exp(x: OTBObject | str, band: int, keep_logical: bool = False) -> tuple(str, list[OTBObject], int): """This an internal function, only to be used by `create_fake_exp`. @@ -1193,7 +1218,7 @@ class LogicalOperation(Operation): super().__init__(operator, *inputs, nb_bands=nb_bands) self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands) - def create_fake_exp(self, operator: str, inputs: list[OTBObject | Output | str | int | float], + def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None): """Create a 'fake' expression. @@ -1202,7 +1227,7 @@ class LogicalOperation(Operation): Args: operator: str (one of >, <, >=, <=, ==, !=, &, |) - inputs: Can be App, Output, Input, Operation, Slicer, filepath, int or float + inputs: Can be OTBObject, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ @@ -1237,7 +1262,7 @@ class LogicalOperation(Operation): self.fake_exp_bands.append(fake_exp) -class Input(OTBObject): +class Input(App): """Class for transforming a filepath to pyOTB object.""" def __init__(self, path: str): @@ -1272,9 +1297,7 @@ class Output(OTBObject): mkdir: create missing parent directories """ - self.pyotb_app, self.app = pyotb_app, pyotb_app.app - self.parameters = pyotb_app.parameters - self.param_key = param_key + super().__init__(name=f"Output {param_key} from {self.pyotb_app.name}", app=pyotb_app) self.filepath = None if filepath: if '?' in filepath: @@ -1282,7 +1305,6 @@ class Output(OTBObject): self.filepath = Path(filepath) if mkdir: self.make_parent_dirs() - self.name = f"Output {param_key} from {self.pyotb_app.name}" @property def key_output_image(self): -- GitLab From 2af2c562ab9c15e10cf86120aa3bd875c86f51ce Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 11:13:36 +0000 Subject: [PATCH 052/399] WIP: code and tests --- pyotb/apps.py | 37 +------------------------------------ pyotb/core.py | 26 ++++++++++++++++++++++++++ tests/bug_todel.py | 7 +++++++ tests/bug_todel2.py | 10 ++++++++++ tests/test_core.py | 11 ++++------- tests/test_numpy.py | 10 +++++----- tests/test_pipeline.py | 2 -- tests/test_serialization.py | 14 ++++++-------- tests/test_todel.py | 6 ++++++ tests/tests_data.py | 3 +++ 10 files changed, 68 insertions(+), 58 deletions(-) create mode 100644 tests/bug_todel.py create mode 100644 tests/bug_todel2.py create mode 100644 tests/test_todel.py create mode 100644 tests/tests_data.py diff --git a/pyotb/apps.py b/pyotb/apps.py index ad71f54..9421b28 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -7,7 +7,7 @@ import subprocess from pathlib import Path import otbApplication as otb # pylint: disable=import-error -from .core import OTBObject +from .core import App from .helpers import logger @@ -59,41 +59,6 @@ def get_available_applications(as_subprocess: bool = False) -> list[str]: return app_list -class App(OTBObject): - """Base class for UI related functions, will be subclassed using app name as class name, see CODE_TEMPLATE.""" - _name = "" - - def __init__(self, *args, **kwargs): - """Default App constructor, adds UI specific attributes and functions.""" - super().__init__(*args, **kwargs) - self.description = self.app.GetDocLongDescription() - - @property - def elapsed_time(self): - """Get elapsed time between app init and end of exec or file writing.""" - return self.time_end - self.time_start - - @property - def used_outputs(self) -> list[str]: - """List of used application outputs.""" - return [getattr(self, key) for key in self.out_param_types if key in self.parameters] - - def find_outputs(self) -> tuple[str]: - """Find output files on disk using path found in parameters. - - Returns: - list of files found on disk - - """ - files, missing = [], [] - for out in self.used_outputs: - dest = files if out.exists() else missing - dest.append(str(out.filepath.absolute())) - for filename in missing: - logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) - return tuple(files) - - class OTBTFApp(App): """Helper for OTBTF.""" @staticmethod diff --git a/pyotb/core.py b/pyotb/core.py index 87bb08c..b97c195 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -654,6 +654,7 @@ class App(OTBObject): self.quiet = quiet create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication super().__init__(name=f"OTB Application {otb_app_name}", app=create(otb_app_name), image_dic=image_dic) + self.description = self.app.GetDocLongDescription() # Set parameters self.parameters = {} @@ -667,6 +668,31 @@ class App(OTBObject): # Elapsed time self.time_start, self.time_end = 0, 0 + @property + def elapsed_time(self): + """Get elapsed time between app init and end of exec or file writing.""" + return self.time_end - self.time_start + + @property + def used_outputs(self) -> list[str]: + """List of used application outputs.""" + return [getattr(self, key) for key in self.out_param_types if key in self.parameters] + + def find_outputs(self) -> tuple[str]: + """Find output files on disk using path found in parameters. + + Returns: + list of files found on disk + + """ + files, missing = [], [] + for out in self.used_outputs: + dest = files if out.exists() else missing + dest.append(str(out.filepath.absolute())) + for filename in missing: + logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) + return tuple(files) + @property def data(self): """Expose app's output data values in a dictionary.""" diff --git a/tests/bug_todel.py b/tests/bug_todel.py new file mode 100644 index 0000000..ae3e64d --- /dev/null +++ b/tests/bug_todel.py @@ -0,0 +1,7 @@ +import pyotb + +img_pth = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/" \ + "otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false" +bm = pyotb.BandMath({"il": [img_pth], "exp": "im1b1"}) +bm.write("/tmp/toto.tif") # Comment this line --> Works +print(bm.shape) \ No newline at end of file diff --git a/tests/bug_todel2.py b/tests/bug_todel2.py new file mode 100644 index 0000000..cafc42d --- /dev/null +++ b/tests/bug_todel2.py @@ -0,0 +1,10 @@ +import otbApplication +img_pth = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false" +app = otbApplication.Registry.CreateApplication("BandMath") +app.SetParameterStringList("il", [img_pth]) +app.SetParameterString("exp", "im1b1") +app.Execute() +print(app.GetImageSize("out")) # WORKS +app.SetParameterString("out", "/tmp/toto.tif") +app.WriteOutput() +print(app.GetImageSize("out")) # ERROR diff --git a/tests/test_core.py b/tests/test_core.py index 436f99e..086c26e 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,13 +1,8 @@ -import os -import pyotb -from ast import literal_eval -from pathlib import Path - import pytest +import pyotb +from tests_data import INPUT -FILEPATH = os.environ["TEST_INPUT_IMAGE"] -INPUT = pyotb.Input(FILEPATH) TEST_IMAGE_STATS = { 'out.mean': [79.5505, 109.225, 115.456, 249.349], 'out.min': [33, 64, 91, 47], @@ -59,9 +54,11 @@ def test_nonraster_property(): with pytest.raises(TypeError): pyotb.ReadImageInfo(INPUT).dtype + def test_elapsed_time(): assert pyotb.ReadImageInfo(INPUT).elapsed_time < 1 + # Other functions def test_get_infos(): infos = INPUT.get_infos() diff --git a/tests/test_numpy.py b/tests/test_numpy.py index dd33425..0f42435 100644 --- a/tests/test_numpy.py +++ b/tests/test_numpy.py @@ -1,10 +1,10 @@ -import os import numpy as np import pyotb +from tests_data import INPUT +import numpy as np - -FILEPATH = os.environ["TEST_INPUT_IMAGE"] -INPUT = pyotb.Input(FILEPATH) +import pyotb +from tests_data import INPUT def test_export(): @@ -37,7 +37,7 @@ def test_convert_to_array(): def test_pixel_coords_otb_equals_numpy(): - assert INPUT[19,7] == list(INPUT.to_numpy()[19,7]) + assert INPUT[19, 7] == list(INPUT.to_numpy()[19, 7]) def test_add_noise_array(): diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 3c88153..f3fe4cf 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,9 +1,7 @@ -import sys import os import itertools import pytest import pyotb -from pyotb.helpers import logger # List of buildings blocks, we can add other pyotb objects here diff --git a/tests/test_serialization.py b/tests/test_serialization.py index e64f2f1..b56e447 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -1,18 +1,16 @@ -import os import pyotb - -filepath = os.environ["TEST_INPUT_IMAGE"] +from tests_data import FILEPATH def test_pipeline_simple(): # BandMath -> OrthoRectification -> ManageNoData - app1 = pyotb.BandMath({'il': [filepath], 'exp': 'im1b1'}) + app1 = pyotb.BandMath({'il': [FILEPATH], 'exp': 'im1b1'}) app2 = pyotb.OrthoRectification({'io.in': app1}) app3 = pyotb.ManageNoData({'in': app2}) summary = app3.summarize() reference = {'name': 'ManageNoData', 'parameters': {'in': { 'name': 'OrthoRectification', 'parameters': {'io.in': { - 'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}}, + 'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}}, 'map': 'utm', 'outputs.isotropic': True}}, 'mode': 'buildmask'}} @@ -21,19 +19,19 @@ def test_pipeline_simple(): def test_pipeline_diamond(): # Diamond graph - app1 = pyotb.BandMath({'il': [filepath], 'exp': 'im1b1'}) + app1 = pyotb.BandMath({'il': [FILEPATH], 'exp': 'im1b1'}) app2 = pyotb.OrthoRectification({'io.in': app1}) app3 = pyotb.ManageNoData({'in': app2}) app4 = pyotb.BandMathX({'il': [app2, app3], 'exp': 'im1+im2'}) summary = app4.summarize() reference = {'name': 'BandMathX', 'parameters': {'il': [ {'name': 'OrthoRectification', 'parameters': {'io.in': { - 'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}}, + 'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}}, 'map': 'utm', 'outputs.isotropic': True}}, {'name': 'ManageNoData', 'parameters': {'in': { 'name': 'OrthoRectification', 'parameters': { - 'io.in': {'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}}, + 'io.in': {'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}}, 'map': 'utm', 'outputs.isotropic': True}}, 'mode': 'buildmask'}} diff --git a/tests/test_todel.py b/tests/test_todel.py new file mode 100644 index 0000000..fa70ec7 --- /dev/null +++ b/tests/test_todel.py @@ -0,0 +1,6 @@ +import pyotb + +img_pth = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/" \ + "otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false" +bm = pyotb.MeanShiftSmoothing({"in": img_pth, "fout": "/tmp/toto.tif", "foutpos": "/tmp/titi.tif"}) +print(bm.find_outputs()) \ No newline at end of file diff --git a/tests/tests_data.py b/tests/tests_data.py new file mode 100644 index 0000000..b190f40 --- /dev/null +++ b/tests/tests_data.py @@ -0,0 +1,3 @@ +import pyotb +FILEPATH = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false" +INPUT = pyotb.Input(FILEPATH) -- GitLab From b773a4bf06ea2794438ab27f161da899654bc481 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 12:26:08 +0100 Subject: [PATCH 053/399] WIP: code and tests --- pyotb/core.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index b97c195..8cf8d71 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -629,7 +629,7 @@ class App(OTBObject): """Base class that gathers common operations for any OTB application.""" def __init__(self, otb_app_name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, - **kwargs): + name: str = None, **kwargs): """Common constructor for OTB applications. Handles in-memory connection between apps. Args: @@ -645,6 +645,7 @@ class App(OTBObject): the result of app.ExportImage(). Use it when the app takes a numpy array as input. See this related issue for why it is necessary to keep reference of object: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824 + name: override the application name **kwargs: used for passing application parameters. e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif' @@ -653,7 +654,7 @@ class App(OTBObject): self.frozen = frozen self.quiet = quiet create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication - super().__init__(name=f"OTB Application {otb_app_name}", app=create(otb_app_name), image_dic=image_dic) + super().__init__(name=name or f"OTB Application {otb_app_name}", app=create(otb_app_name), image_dic=image_dic) self.description = self.app.GetDocLongDescription() # Set parameters @@ -958,8 +959,8 @@ class Slicer(App): channels: channels, can be slicing, list or int """ - super().__init__("ExtractROI", {"in": obj, "mode": "extent"}, quiet=True, frozen=True) - self.name = "Slicer" + super().__init__(otb_app_name="ExtractROI", args={"in": obj, "mode": "extent"}, quiet=True, frozen=True, + name="Slicer") self.rows, self.cols = rows, cols parameters = {} @@ -1029,7 +1030,7 @@ class Operation(App): """ - def __init__(self, operator: str, *inputs, nb_bands: int = None): + def __init__(self, operator: str, *inputs, nb_bands: int = None, name: str = None): """Given some inputs and an operator, this function enables to transform this into an OTB application. Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator. @@ -1039,6 +1040,7 @@ class Operation(App): operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? *inputs: inputs. Can be OTBObject, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where + name: override the Operation name """ self.operator = operator @@ -1065,9 +1067,8 @@ class Operation(App): self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)] self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands) # Execute app - name = "BandMath" if len(self.exp_bands) == 1 else "BandMathX" - super().__init__(name, il=self.unique_inputs, exp=self.exp, quiet=True) - self.name = f'Operation exp="{self.exp}"' + super().__init__(otb_app_name="BandMath" if len(self.exp_bands) == 1 else "BandMathX", il=self.unique_inputs, + exp=self.exp, quiet=True, name=name or f'Operation exp="{self.exp}"') def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None): @@ -1241,14 +1242,13 @@ class LogicalOperation(Operation): nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ - super().__init__(operator, *inputs, nb_bands=nb_bands) + super().__init__(operator=operator, inputs=inputs, nb_bands=nb_bands, name="LogicalOperation") self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands) - def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float], - nb_bands: int = None): + def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None): """Create a 'fake' expression. - E.g for the operation input1 > input2, we create a fake expression that is like + e.g for the operation input1 > input2, we create a fake expression that is like "str(input1) > str(input2) ? 1 : 0" and a logical fake expression that is like "str(input1) > str(input2)" Args: @@ -1299,8 +1299,7 @@ class Input(App): """ self.path = path - super().__init__("ExtractROI", {"in": path}, frozen=True) - self.name = f"Input from {path}" + super().__init__(otb_app_name="ExtractROI", args={"in": path}, frozen=True, name=f"Input from {path}") self.propagate_dtype() self.execute() @@ -1323,7 +1322,8 @@ class Output(OTBObject): mkdir: create missing parent directories """ - super().__init__(name=f"Output {param_key} from {self.pyotb_app.name}", app=pyotb_app) + super().__init__(name=f"Output {param_key} from {self.pyotb_app.name}", app=pyotb_app.app) + self.parent_pyotb_app = pyotb_app # keep trace of parent app self.filepath = None if filepath: if '?' in filepath: -- GitLab From 6a6edc07b1277535c29c7046404f51a201b154a1 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 11:28:14 +0000 Subject: [PATCH 054/399] WIP: code and tests --- pyotb/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 8cf8d71..96a1f97 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -36,7 +36,6 @@ class OTBObject: otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename)} - self.transform = None # TODO: value? self.exports_dic = {} def get_first_key(self, param_types: list[str]) -> str: -- GitLab From e6738110f00f4f85f1fbc2297f6f2ce0a4befcbc Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 11:33:39 +0000 Subject: [PATCH 055/399] WIP: code and tests --- pyotb/core.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 96a1f97..6c16a4f 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -958,8 +958,7 @@ class Slicer(App): channels: channels, can be slicing, list or int """ - super().__init__(otb_app_name="ExtractROI", args={"in": obj, "mode": "extent"}, quiet=True, frozen=True, - name="Slicer") + super().__init__("ExtractROI", {"in": obj, "mode": "extent"}, quiet=True, frozen=True, name="Slicer") self.rows, self.cols = rows, cols parameters = {} @@ -1066,7 +1065,7 @@ class Operation(App): self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)] self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands) # Execute app - super().__init__(otb_app_name="BandMath" if len(self.exp_bands) == 1 else "BandMathX", il=self.unique_inputs, + super().__init__("BandMath" if len(self.exp_bands) == 1 else "BandMathX", il=self.unique_inputs, exp=self.exp, quiet=True, name=name or f'Operation exp="{self.exp}"') def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float], @@ -1298,7 +1297,7 @@ class Input(App): """ self.path = path - super().__init__(otb_app_name="ExtractROI", args={"in": path}, frozen=True, name=f"Input from {path}") + super().__init__("ExtractROI", {"in": path}, frozen=True, name=f"Input from {path}") self.propagate_dtype() self.execute() -- GitLab From 3745d6f55861815e5824326b374ac87ea2ce19aa Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 12:49:50 +0100 Subject: [PATCH 056/399] WIP: code and tests --- pyotb/core.py | 49 +++++++++++++++++++++++------------------- tests/test_pipeline.py | 3 +-- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 6c16a4f..ce30564 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -273,26 +273,31 @@ class OTBObject: """ return id(self) - def __getattr__(self, name): - """This method is called when the default attribute access fails. - - We choose to access the attribute `name` of self.app. - Thus, any method of otbApplication can be used transparently on OTBObject objects, - e.g. SetParameterOutputImagePixelType() or ExportImage() work - - Args: - name: attribute name - - Returns: - attribute - - Raises: - AttributeError: when `name` is not an attribute of self.app - - """ - if name in dir(self.app): - return getattr(self.app, name) - raise AttributeError(f"{self.name}: could not find attribute `{name}`") + # TODO: Remove completely this code? I think this is (1) dangerous, because it + # can break things in otb.Application, just by setting some mutable attribute, + # (2) harder to debug. And (3) I feel that this is the wrong design. The good + # design would be to inherit from otb.Application. + # + # def __getattr__(self, name): + # """This method is called when the default attribute access fails. + # + # We choose to access the attribute `name` of self.app. + # Thus, any method of otbApplication can be used transparently on OTBObject objects, + # e.g. SetParameterOutputImagePixelType() or ExportImage() work + # + # Args: + # name: attribute name + # + # Returns: + # attribute + # + # Raises: + # AttributeError: when `name` is not an attribute of self.app + # + # """ + # if name in dir(self.app): + # return getattr(self.app, name) + # raise AttributeError(f"{self.name}: could not find attribute `{name}`") def __getitem__(self, key): """Override the default __getitem__ behaviour. @@ -1320,7 +1325,7 @@ class Output(OTBObject): mkdir: create missing parent directories """ - super().__init__(name=f"Output {param_key} from {self.pyotb_app.name}", app=pyotb_app.app) + super().__init__(name=f"Output {param_key} from {pyotb_app.name}", app=pyotb_app.app) self.parent_pyotb_app = pyotb_app # keep trace of parent app self.filepath = None if filepath: @@ -1405,7 +1410,7 @@ def get_pixel_type(inp: str | OTBObject) -> str: raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}") pixel_type = getattr(otb, f'ImagePixelType_{datatype_to_pixeltype[datatype]}') elif isinstance(inp, OTBObject): - pixel_type = inp.GetParameterOutputImagePixelType(inp.key_output_image) + pixel_type = inp.app.GetParameterOutputImagePixelType(inp.key_output_image) else: raise TypeError(f'Could not get the pixel type of {type(inp)} object {inp}') return pixel_type diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index f3fe4cf..3acc354 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -2,6 +2,7 @@ import os import itertools import pytest import pyotb +from tests_data import INPUT, FILEPATH # List of buildings blocks, we can add other pyotb objects here @@ -21,8 +22,6 @@ PYOTB_BLOCKS = [ PIPELINES_LENGTH = [1, 2, 3] ALL_BLOCKS = PYOTB_BLOCKS + OTBAPPS_BLOCKS -FILEPATH = os.environ["TEST_INPUT_IMAGE"] -INPUT = pyotb.Input(FILEPATH) def generate_pipeline(inp, building_blocks): -- GitLab From ef5f6be2c54a8855b799d032a22e9ec8d498748b Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 12:59:13 +0100 Subject: [PATCH 057/399] WIP: code and tests --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index ce30564..9dd40c1 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1389,7 +1389,7 @@ def get_pixel_type(inp: str | OTBObject) -> str: """ if isinstance(inp, str): try: - info = OTBObject("ReadImageInfo", inp, quiet=True) + info = App("ReadImageInfo", inp, quiet=True) except Exception as info_err: # this happens when we pass a str that is not a filepath raise TypeError(f"Could not get the pixel type of `{inp}`. Not a filepath or wrong filepath") from info_err datatype = info.GetParameterString("datatype") # which is such as short, float... -- GitLab From 17f0b6e519cf8ae24f766774263061a9751e3026 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 13:08:10 +0100 Subject: [PATCH 058/399] WIP: code and tests --- pyotb/core.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 9dd40c1..3801dfe 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -118,13 +118,13 @@ class OTBObject: """Return a dict output of ReadImageInfo for the first image output.""" if not self.key_output_image: raise TypeError(f"{self.name}: this application has no raster output") - return OTBObject("ReadImageInfo", self, quiet=True).data + return App("ReadImageInfo", self, quiet=True).data def get_statistics(self): """Return a dict output of ComputeImagesStatistics for the first image output.""" if not self.key_output_image: raise TypeError(f"{self.name}: this application has no raster output") - return OTBObject("ComputeImagesStatistics", self, quiet=True).data + return App("ComputeImagesStatistics", self, quiet=True).data def read_values_at_coords(self, row: int, col: int, bands: int = None) -> list[int | float] | int | float: """Get pixel value(s) at a given YX coordinates. @@ -139,7 +139,7 @@ class OTBObject: """ channels = [] - app = OTBObject("PixelValue", self, coordx=col, coordy=row, frozen=True, quiet=True) + app = App("PixelValue", self, coordx=col, coordy=row, frozen=True, quiet=True) if bands is not None: if isinstance(bands, int): if bands < 0: @@ -619,7 +619,7 @@ class OTBObject: result_dic = image_dic result_dic["array"] = result_array # Importing back to OTB, pass the result_dic just to keep reference - app = OTBObject("ExtractROI", image_dic=result_dic, frozen=True, quiet=True) + app = App("ExtractROI", image_dic=result_dic, frozen=True, quiet=True) if result_array.shape[2] == 1: app.ImportImage("in", result_dic) else: @@ -1245,7 +1245,7 @@ class LogicalOperation(Operation): nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ - super().__init__(operator=operator, inputs=inputs, nb_bands=nb_bands, name="LogicalOperation") + super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation") self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands) def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None): @@ -1315,7 +1315,7 @@ class Output(OTBObject): """Object that behave like a pointer to a specific application output file.""" def __init__(self, pyotb_app: OTBObject, # pylint: disable=super-init-not-called - param_key: str, filepath: str = None, mkdir: bool = True): + param_key: str = None, filepath: str = None, mkdir: bool = True): """Constructor for an Output object. Args: @@ -1327,6 +1327,7 @@ class Output(OTBObject): """ super().__init__(name=f"Output {param_key} from {pyotb_app.name}", app=pyotb_app.app) self.parent_pyotb_app = pyotb_app # keep trace of parent app + self.param_key = param_key or super().key_output_image self.filepath = None if filepath: if '?' in filepath: @@ -1342,10 +1343,12 @@ class Output(OTBObject): def exists(self) -> bool: """Check file exist.""" + assert self.filepath, "Filepath not set" return self.filepath.exists() def make_parent_dirs(self): """Create missing parent directories.""" + assert self.filepath, "Filepath not set" if not self.filepath.parent.exists(): self.filepath.parent.mkdir(parents=True) @@ -1369,7 +1372,7 @@ def get_nbchannels(inp: str | OTBObject) -> int: else: # Executing the app, without printing its log try: - info = OTBObject("ReadImageInfo", inp, quiet=True) + info = App("ReadImageInfo", inp, quiet=True) nb_channels = info.GetParameterInt("numberbands") except Exception as e: # this happens when we pass a str that is not a filepath raise TypeError(f'Could not get the number of channels of `{inp}`. Not a filepath or wrong filepath') from e @@ -1392,7 +1395,7 @@ def get_pixel_type(inp: str | OTBObject) -> str: info = App("ReadImageInfo", inp, quiet=True) except Exception as info_err: # this happens when we pass a str that is not a filepath raise TypeError(f"Could not get the pixel type of `{inp}`. Not a filepath or wrong filepath") from info_err - datatype = info.GetParameterString("datatype") # which is such as short, float... + datatype = info.app.GetParameterString("datatype") # which is such as short, float... if not datatype: raise TypeError(f"Unable to read pixel type of image {inp}") datatype_to_pixeltype = { -- GitLab From 6d7e798f4e0e99786f90bf544990d0ac94536a53 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 12:47:05 +0000 Subject: [PATCH 059/399] WIP: code and tests --- pyotb/core.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 3801dfe..077b373 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -94,7 +94,7 @@ class OTBObject: """ if not self.key_output_image: - raise TypeError(f"{self.name}: this application has no raster output") + raise TypeError(f"\"{self.name}\" has no raster output") width, height = self.app.GetImageSize(self.key_output_image) bands = self.app.GetImageNbBands(self.key_output_image) return height, width, bands @@ -189,7 +189,7 @@ class OTBObject: params[k] = p.summarize() elif isinstance(p, list): # parameter list params[k] = [pi.summarize() if isinstance(pi, OTBObject) else pi for pi in p] - return {"name": self.name, "parameters": params} + return {"name": self.app.GetName(), "parameters": params} def export(self, key: str = None, preserve_dtype: bool = True) -> dict[str, dict[str, np.ndarray]]: """Export a specific output image as numpy array and store it in object exports_dic. @@ -619,13 +619,13 @@ class OTBObject: result_dic = image_dic result_dic["array"] = result_array # Importing back to OTB, pass the result_dic just to keep reference - app = App("ExtractROI", image_dic=result_dic, frozen=True, quiet=True) + pyotb_app = App("ExtractROI", image_dic=result_dic, frozen=True, quiet=True) if result_array.shape[2] == 1: - app.ImportImage("in", result_dic) + pyotb_app.app.ImportImage("in", result_dic) else: - app.ImportVectorImage("in", result_dic) - app.execute() - return app + pyotb_app.app.ImportVectorImage("in", result_dic) + pyotb_app.execute() + return pyotb_app return NotImplemented -- GitLab From 9b3afb6e061bd4ad9504a504f62c69e5b4ef14a5 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 12:58:56 +0000 Subject: [PATCH 060/399] DOC: code and tests --- pyotb/core.py | 40 ++++++++---------------------------- tests/bug_todel.py | 7 ------- tests/bug_todel2.py | 10 --------- tests/test-core.xml | 1 + tests/test-numpy.xml | 1 + tests/test-pipeline.xml | 1 + tests/test-serialization.xml | 1 + 7 files changed, 13 insertions(+), 48 deletions(-) delete mode 100644 tests/bug_todel.py delete mode 100644 tests/bug_todel2.py create mode 100644 tests/test-core.xml create mode 100644 tests/test-numpy.xml create mode 100644 tests/test-pipeline.xml create mode 100644 tests/test-serialization.xml diff --git a/pyotb/core.py b/pyotb/core.py index 077b373..0f04689 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -14,12 +14,17 @@ from .helpers import logger class OTBObject: + """ + Base class for all pyotb objects + """ + def __init__(self, name: str, app: otb.Application, image_dic: dict = None): - """ - name: - app: + """Constructor for an OTBObject. + + name: name of the object (e.g. "Slicer") + app: OTB application instance image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as - the result of app.ExportImage(). Use it when the app takes a numpy array as input. + the result of self.app.ExportImage(). Use it when the app takes a numpy array as input. See this related issue for why it is necessary to keep reference of object: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824 @@ -273,32 +278,6 @@ class OTBObject: """ return id(self) - # TODO: Remove completely this code? I think this is (1) dangerous, because it - # can break things in otb.Application, just by setting some mutable attribute, - # (2) harder to debug. And (3) I feel that this is the wrong design. The good - # design would be to inherit from otb.Application. - # - # def __getattr__(self, name): - # """This method is called when the default attribute access fails. - # - # We choose to access the attribute `name` of self.app. - # Thus, any method of otbApplication can be used transparently on OTBObject objects, - # e.g. SetParameterOutputImagePixelType() or ExportImage() work - # - # Args: - # name: attribute name - # - # Returns: - # attribute - # - # Raises: - # AttributeError: when `name` is not an attribute of self.app - # - # """ - # if name in dir(self.app): - # return getattr(self.app, name) - # raise AttributeError(f"{self.name}: could not find attribute `{name}`") - def __getitem__(self, key): """Override the default __getitem__ behaviour. @@ -654,7 +633,6 @@ class App(OTBObject): e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif' """ - self.frozen = frozen self.quiet = quiet create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication diff --git a/tests/bug_todel.py b/tests/bug_todel.py deleted file mode 100644 index ae3e64d..0000000 --- a/tests/bug_todel.py +++ /dev/null @@ -1,7 +0,0 @@ -import pyotb - -img_pth = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/" \ - "otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false" -bm = pyotb.BandMath({"il": [img_pth], "exp": "im1b1"}) -bm.write("/tmp/toto.tif") # Comment this line --> Works -print(bm.shape) \ No newline at end of file diff --git a/tests/bug_todel2.py b/tests/bug_todel2.py deleted file mode 100644 index cafc42d..0000000 --- a/tests/bug_todel2.py +++ /dev/null @@ -1,10 +0,0 @@ -import otbApplication -img_pth = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false" -app = otbApplication.Registry.CreateApplication("BandMath") -app.SetParameterStringList("il", [img_pth]) -app.SetParameterString("exp", "im1b1") -app.Execute() -print(app.GetImageSize("out")) # WORKS -app.SetParameterString("out", "/tmp/toto.tif") -app.WriteOutput() -print(app.GetImageSize("out")) # ERROR diff --git a/tests/test-core.xml b/tests/test-core.xml new file mode 100644 index 0000000..718a6f5 --- /dev/null +++ b/tests/test-core.xml @@ -0,0 +1 @@ +<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="0" time="0.007" timestamp="2022-09-20T19:17:53.478514" hostname="787f55c467b8" /></testsuites> \ No newline at end of file diff --git a/tests/test-numpy.xml b/tests/test-numpy.xml new file mode 100644 index 0000000..f3cd59b --- /dev/null +++ b/tests/test-numpy.xml @@ -0,0 +1 @@ +<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="0" time="0.007" timestamp="2022-09-20T19:17:53.684237" hostname="787f55c467b8" /></testsuites> \ No newline at end of file diff --git a/tests/test-pipeline.xml b/tests/test-pipeline.xml new file mode 100644 index 0000000..d1af1d1 --- /dev/null +++ b/tests/test-pipeline.xml @@ -0,0 +1 @@ +<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="0" time="0.007" timestamp="2022-09-20T19:17:53.890479" hostname="787f55c467b8" /></testsuites> \ No newline at end of file diff --git a/tests/test-serialization.xml b/tests/test-serialization.xml new file mode 100644 index 0000000..0e957e3 --- /dev/null +++ b/tests/test-serialization.xml @@ -0,0 +1 @@ +<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="0" time="0.007" timestamp="2022-09-20T19:17:54.099124" hostname="787f55c467b8" /></testsuites> \ No newline at end of file -- GitLab From 91c614be509390d2e6c7eb5f0344cc6aea921007 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 12:59:19 +0000 Subject: [PATCH 061/399] DOC: code and tests --- tests/test-core.xml | 1 - tests/test-numpy.xml | 1 - tests/test-pipeline.xml | 1 - tests/test-serialization.xml | 1 - 4 files changed, 4 deletions(-) delete mode 100644 tests/test-core.xml delete mode 100644 tests/test-numpy.xml delete mode 100644 tests/test-pipeline.xml delete mode 100644 tests/test-serialization.xml diff --git a/tests/test-core.xml b/tests/test-core.xml deleted file mode 100644 index 718a6f5..0000000 --- a/tests/test-core.xml +++ /dev/null @@ -1 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="0" time="0.007" timestamp="2022-09-20T19:17:53.478514" hostname="787f55c467b8" /></testsuites> \ No newline at end of file diff --git a/tests/test-numpy.xml b/tests/test-numpy.xml deleted file mode 100644 index f3cd59b..0000000 --- a/tests/test-numpy.xml +++ /dev/null @@ -1 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="0" time="0.007" timestamp="2022-09-20T19:17:53.684237" hostname="787f55c467b8" /></testsuites> \ No newline at end of file diff --git a/tests/test-pipeline.xml b/tests/test-pipeline.xml deleted file mode 100644 index d1af1d1..0000000 --- a/tests/test-pipeline.xml +++ /dev/null @@ -1 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="0" time="0.007" timestamp="2022-09-20T19:17:53.890479" hostname="787f55c467b8" /></testsuites> \ No newline at end of file diff --git a/tests/test-serialization.xml b/tests/test-serialization.xml deleted file mode 100644 index 0e957e3..0000000 --- a/tests/test-serialization.xml +++ /dev/null @@ -1 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="0" time="0.007" timestamp="2022-09-20T19:17:54.099124" hostname="787f55c467b8" /></testsuites> \ No newline at end of file -- GitLab From f38b86ad4df854d25483f4bec53b3bf0612fc55b Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 14:00:48 +0000 Subject: [PATCH 062/399] DOC: code and tests --- pyotb/core.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 0f04689..35885e9 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -14,13 +14,12 @@ from .helpers import logger class OTBObject: - """ - Base class for all pyotb objects - """ + """Base class for all pyotb objects.""" def __init__(self, name: str, app: otb.Application, image_dic: dict = None): """Constructor for an OTBObject. + Args: name: name of the object (e.g. "Slicer") app: OTB application instance image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as @@ -28,7 +27,6 @@ class OTBObject: See this related issue for why it is necessary to keep reference of object: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824 - """ self.name = name self.app = app -- GitLab From ec8d1b38e09c6d8d44b239451847edbff2113eac Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 15:59:25 +0100 Subject: [PATCH 063/399] DOC: code and tests --- pyotb/core.py | 18 +----------------- pyotb/functions.py | 8 ++++---- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 35885e9..f97c1a8 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -178,22 +178,6 @@ class OTBObject: return list(range(0, nb_channels, step)) raise ValueError(f"{self.name}: '{bands}' cannot be interpreted as valid slicing.") - def summarize(self) -> dict: - """Serialize an object and its pipeline into a dictionary. - - Returns: - nested dictionary summarizing the pipeline - - """ - params = self.parameters - for k, p in params.items(): - # In the following, we replace each parameter which is an OTBObject, with its summary. - if isinstance(p, OTBObject): # single parameter - params[k] = p.summarize() - elif isinstance(p, list): # parameter list - params[k] = [pi.summarize() if isinstance(pi, OTBObject) else pi for pi in p] - return {"name": self.app.GetName(), "parameters": params} - def export(self, key: str = None, preserve_dtype: bool = True) -> dict[str, dict[str, np.ndarray]]: """Export a specific output image as numpy array and store it in object exports_dic. @@ -1330,7 +1314,7 @@ class Output(OTBObject): def __str__(self) -> str: """Return a nice string representation with source app name and object id.""" - return f"<pyotb.Output {self.source_app.name} object, id {id(self)}>" + return f"<pyotb.Output {self.name} object, id {id(self)}>" def get_nbchannels(inp: str | OTBObject) -> int: diff --git a/pyotb/functions.py b/pyotb/functions.py index 71c298b..763dcb7 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -355,9 +355,9 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ metadatas = {} for inp in inputs: if isinstance(inp, str): # this is for filepaths - metadata = Input(inp).GetImageMetaData('out') + metadata = Input(inp).app.GetImageMetaData('out') elif isinstance(inp, OTBObject): - metadata = inp.GetImageMetaData(inp.output_param) + metadata = inp.app.GetImageMetaData(inp.output_param) else: raise TypeError(f"Wrong input : {inp}") metadatas[inp] = metadata @@ -429,7 +429,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ inputs = new_inputs # Update metadatas - metadatas = {input: input.GetImageMetaData('out') for input in inputs} + metadatas = {input: input.app.GetImageMetaData('out') for input in inputs} # Get a metadata of an arbitrary image. This is just to compare later with other images any_metadata = next(iter(metadatas.values())) @@ -464,7 +464,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ inputs = new_inputs # Update metadatas - metadatas = {inp: inp.GetImageMetaData('out') for inp in inputs} + metadatas = {inp: inp.app.GetImageMetaData('out') for inp in inputs} # Final superimposition to be sure to have the exact same image sizes # Getting the sizes of images -- GitLab From 919d0761151ac7a26148e5df98198c617e962e89 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 16:04:37 +0100 Subject: [PATCH 064/399] DOC: code and tests --- pyotb/core.py | 2 +- pyotb/functions.py | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index f97c1a8..7abac39 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1333,7 +1333,7 @@ def get_nbchannels(inp: str | OTBObject) -> int: # Executing the app, without printing its log try: info = App("ReadImageInfo", inp, quiet=True) - nb_channels = info.GetParameterInt("numberbands") + nb_channels = info.app.GetParameterInt("numberbands") except Exception as e: # this happens when we pass a str that is not a filepath raise TypeError(f'Could not get the number of channels of `{inp}`. Not a filepath or wrong filepath') from e return nb_channels diff --git a/pyotb/functions.py b/pyotb/functions.py index 763dcb7..d25610a 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -9,12 +9,11 @@ import textwrap import subprocess from collections import Counter -from .core import OTBObject, Input, Operation, LogicalOperation, get_nbchannels, Output +from .core import OTBObject, App, Operation, LogicalOperation, Input, get_nbchannels from .helpers import logger -def where(cond: OTBObject | Output | str, x: OTBObject | Output | str | int | float, - y: OTBObject | Output | str | int | float) -> Operation: +def where(cond: OTBObject | str, x: OTBObject | str | int | float, y: OTBObject | str | int | float) -> Operation: """Functionally similar to numpy.where. Where cond is True (!=0), returns x. Else returns y. Args: @@ -64,8 +63,7 @@ def where(cond: OTBObject | Output | str, x: OTBObject | Output | str | int | fl return operation -def clip(a: OTBObject | Output | str, a_min: OTBObject | Output | str | int | float, - a_max: OTBObject | Output | str | int | float): +def clip(a: OTBObject | str, a_min: OTBObject | str | int | float, a_max: OTBObject | str | int | float): """Clip values of image in a range of values. Args: @@ -416,7 +414,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ 'mode.extent.ulx': ulx, 'mode.extent.uly': lry, # bug in OTB <= 7.3 : 'mode.extent.lrx': lrx, 'mode.extent.lry': uly, # ULY/LRY are inverted } - new_input = OTBObject('ExtractROI', params) + new_input = App('ExtractROI', params) # TODO: OTB 7.4 fixes this bug, how to handle different versions of OTB? new_inputs.append(new_input) # Potentially update the reference inputs for later resampling @@ -457,7 +455,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ new_inputs = [] for inp in inputs: if metadatas[inp]['GeoTransform'][1] != pixel_size: - superimposed = OTBObject('Superimpose', inr=reference_input, inm=inp, interpolator=interpolator) + superimposed = App('Superimpose', inr=reference_input, inm=inp, interpolator=interpolator) new_inputs.append(superimposed) else: new_inputs.append(inp) @@ -482,7 +480,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ new_inputs = [] for inp in inputs: if image_sizes[inp] != most_common_image_size: - superimposed = OTBObject('Superimpose', inr=same_size_images[0], inm=inp, interpolator=interpolator) + superimposed = App('Superimpose', inr=same_size_images[0], inm=inp, interpolator=interpolator) new_inputs.append(superimposed) else: new_inputs.append(inp) -- GitLab From 39193a72c77c445220f450603c7f581e34bc8005 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 16:07:59 +0100 Subject: [PATCH 065/399] DOC: code and tests --- pyotb/core.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pyotb/core.py b/pyotb/core.py index 7abac39..dc0296b 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1183,6 +1183,23 @@ class Operation(App): fake_exp = f"{x}b{band}" return fake_exp, inputs, nb_channels + def summarize(self) -> dict: + """Serialize an object and its pipeline into a dictionary. + + Returns: + nested dictionary summarizing the pipeline + + """ + params = self.parameters + for k, p in params.items(): + # In the following, we replace each parameter which is an OTBObject, with its summary. + if isinstance(p, App): # single parameter + params[k] = p.summarize() + elif isinstance(p, list): # parameter list + params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p] + return {"name": self.name, "parameters": params} + + def __str__(self) -> str: """Return a nice string representation with operator and object id.""" return f"<pyotb.Operation `{self.operator}` object, id {id(self)}>" -- GitLab From 3078e0e0eaff1c40808595e5af9ab59fc551a802 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 16:10:25 +0100 Subject: [PATCH 066/399] DOC: code and tests --- pyotb/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index dc0296b..3e96a41 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1199,7 +1199,6 @@ class Operation(App): params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p] return {"name": self.name, "parameters": params} - def __str__(self) -> str: """Return a nice string representation with operator and object id.""" return f"<pyotb.Operation `{self.operator}` object, id {id(self)}>" -- GitLab From 6eb1c98276357bbd25f3cc60bb8d2e649298605e Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 16:12:13 +0100 Subject: [PATCH 067/399] DOC: code and tests --- tests/test_todel.py | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 tests/test_todel.py diff --git a/tests/test_todel.py b/tests/test_todel.py deleted file mode 100644 index fa70ec7..0000000 --- a/tests/test_todel.py +++ /dev/null @@ -1,6 +0,0 @@ -import pyotb - -img_pth = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/" \ - "otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false" -bm = pyotb.MeanShiftSmoothing({"in": img_pth, "fout": "/tmp/toto.tif", "foutpos": "/tmp/titi.tif"}) -print(bm.find_outputs()) \ No newline at end of file -- GitLab From 495104255dcd521f03297abf59b60f8fbb8b833e Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 16:25:40 +0100 Subject: [PATCH 068/399] REFAC: move summarize() to App --- pyotb/core.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 3e96a41..9850264 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -901,6 +901,22 @@ class App(OTBObject): else: self.app.SetParameterValue(key, obj) + def summarize(self) -> dict: + """Serialize an object and its pipeline into a dictionary. + + Returns: + nested dictionary summarizing the pipeline + + """ + params = self.parameters + for k, p in params.items(): + # In the following, we replace each parameter which is an OTBObject, with its summary. + if isinstance(p, App): # single parameter + params[k] = p.summarize() + elif isinstance(p, list): # parameter list + params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p] + return {"name": self.name, "parameters": params} + def __str__(self): """Return a nice string representation with object id.""" return f"<pyotb.App {self.name} object id {id(self)}>" @@ -1183,22 +1199,6 @@ class Operation(App): fake_exp = f"{x}b{band}" return fake_exp, inputs, nb_channels - def summarize(self) -> dict: - """Serialize an object and its pipeline into a dictionary. - - Returns: - nested dictionary summarizing the pipeline - - """ - params = self.parameters - for k, p in params.items(): - # In the following, we replace each parameter which is an OTBObject, with its summary. - if isinstance(p, App): # single parameter - params[k] = p.summarize() - elif isinstance(p, list): # parameter list - params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p] - return {"name": self.name, "parameters": params} - def __str__(self) -> str: """Return a nice string representation with operator and object id.""" return f"<pyotb.Operation `{self.operator}` object, id {id(self)}>" -- GitLab From c01d2d942331ce3733161584fbcb1d83960fe290 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 25 Jan 2023 16:33:50 +0100 Subject: [PATCH 069/399] FIX: summarize() with self.app.GetName() --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 9850264..fb07a7b 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -915,7 +915,7 @@ class App(OTBObject): params[k] = p.summarize() elif isinstance(p, list): # parameter list params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p] - return {"name": self.name, "parameters": params} + return {"name": self.app.GetName(), "parameters": params} def __str__(self): """Return a nice string representation with object id.""" -- GitLab From e778895bcc8c1333bbf81eccf318906b996b944b Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 25 Jan 2023 23:34:41 +0100 Subject: [PATCH 070/399] ENH: remove OTBObject --- pyotb/__init__.py | 1 + pyotb/core.py | 432 ++++++++++++++++++++--------------------- pyotb/functions.py | 10 +- tests/test_core.py | 10 +- tests/test_numpy.py | 6 +- tests/test_pipeline.py | 5 +- 6 files changed, 225 insertions(+), 239 deletions(-) diff --git a/pyotb/__init__.py b/pyotb/__init__.py index 6164870..e3bc23a 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -9,6 +9,7 @@ otb = find_otb() from .apps import * from .core import ( + App, Input, Output, get_nbchannels, diff --git a/pyotb/core.py b/pyotb/core.py index fb07a7b..7928356 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -6,6 +6,7 @@ from ast import literal_eval from pathlib import Path from time import perf_counter from typing import Any +from abc import ABC, abstractmethod import numpy as np import otbApplication as otb # pylint: disable=import-error @@ -13,66 +14,23 @@ import otbApplication as otb # pylint: disable=import-error from .helpers import logger -class OTBObject: - """Base class for all pyotb objects.""" - - def __init__(self, name: str, app: otb.Application, image_dic: dict = None): - """Constructor for an OTBObject. - - Args: - name: name of the object (e.g. "Slicer") - app: OTB application instance - image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as - the result of self.app.ExportImage(). Use it when the app takes a numpy array as input. - See this related issue for why it is necessary to keep reference of object: - https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824 - - """ - self.name = name - self.app = app - self.image_dic = image_dic - - self.parameters_keys = tuple(self.app.GetParametersKeys()) - self.all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys} - self.out_param_types = {k: v for k, v in self.all_param_types.items() - if v in (otb.ParameterType_OutputImage, - otb.ParameterType_OutputVectorData, - otb.ParameterType_OutputFilename)} - - self.exports_dic = {} - - def get_first_key(self, param_types: list[str]) -> str: - """Get the first output param key for specific file types.""" - for key, param_type in sorted(self.all_param_types.items()): - if param_type in param_types: - return key - return None - - @property - def key_input(self) -> str: - """Get the name of first input parameter, raster > vector > file.""" - return self.get_first_key(param_types=[otb.ParameterType_InputImage, - otb.ParameterType_InputImageList]) \ - or self.get_first_key(param_types=[otb.ParameterType_InputVectorData, - otb.ParameterType_InputVectorDataList]) \ - or self.get_first_key(param_types=[otb.ParameterType_InputFilename, - otb.ParameterType_InputFilenameList]) +class RasterInterface(ABC): + """Abstraction of an image object.""" + app: otb.Application + exports_dic: dict @property - def key_input_image(self) -> str: - """Get the name of first input image parameter.""" - return self.get_first_key(param_types=[otb.ParameterType_InputImage, otb.ParameterType_InputImageList]) + @abstractmethod + def key_output_image(self): + """Returns the name of a parameter associated to an image. Property defined in App and Output.""" - @property - def key_output_image(self) -> str: - """Get the name of first output image parameter.""" - return self.get_first_key(param_types=[otb.ParameterType_OutputImage]) + @abstractmethod + def write(self): + """Write image, this is defined in App. Output will use App.write for a specific key.""" @property def metadata(self): """Return first output image metadata dictionary.""" - if not self.key_output_image: - raise TypeError(f"{self.name}: this application has no raster output") return dict(self.app.GetMetadataDictionary(self.key_output_image)) @property @@ -83,8 +41,6 @@ class OTBObject: dtype: pixel type of the output image """ - if not self.key_output_image: - raise TypeError(f"{self.name}: this application has no raster output") enum = self.app.GetParameterOutputImagePixelType(self.key_output_image) return self.app.ConvertPixelTypeToNumpy(enum) @@ -96,8 +52,6 @@ class OTBObject: shape: (height, width, bands) """ - if not self.key_output_image: - raise TypeError(f"\"{self.name}\" has no raster output") width, height = self.app.GetImageSize(self.key_output_image) bands = self.app.GetImageNbBands(self.key_output_image) return height, width, bands @@ -109,8 +63,6 @@ class OTBObject: Returns: transform: (X spacing, X offset, X origin, Y offset, Y spacing, Y origin) """ - if not self.key_output_image: - raise TypeError(f"{self.name}: this application has no raster output") spacing_x, spacing_y = self.app.GetImageSpacing(self.key_output_image) origin_x, origin_y = self.app.GetImageOrigin(self.key_output_image) # Shift image origin since OTB is giving coordinates of pixel center instead of corners @@ -119,14 +71,10 @@ class OTBObject: def get_infos(self): """Return a dict output of ReadImageInfo for the first image output.""" - if not self.key_output_image: - raise TypeError(f"{self.name}: this application has no raster output") return App("ReadImageInfo", self, quiet=True).data def get_statistics(self): """Return a dict output of ComputeImagesStatistics for the first image output.""" - if not self.key_output_image: - raise TypeError(f"{self.name}: this application has no raster output") return App("ComputeImagesStatistics", self, quiet=True).data def read_values_at_coords(self, row: int, col: int, bands: int = None) -> list[int | float] | int | float: @@ -151,7 +99,7 @@ class OTBObject: elif isinstance(bands, slice): channels = self.channels_list_from_slice(bands) elif not isinstance(bands, list): - raise TypeError(f"{self.name}: type '{type(bands)}' cannot be interpreted as a valid slicing") + raise TypeError(f"{self.app.GetName()}: type '{type(bands)}' cannot be interpreted as a valid slicing") if channels: app.app.Execute() app.set_parameters({"cl": [f"Channel{n + 1}" for n in channels]}) @@ -176,7 +124,7 @@ class OTBObject: return list(range(0, stop, step)) if start is None and stop is None: return list(range(0, nb_channels, step)) - raise ValueError(f"{self.name}: '{bands}' cannot be interpreted as valid slicing.") + raise ValueError(f"{self.app.GetName()}: '{bands}' cannot be interpreted as valid slicing.") def export(self, key: str = None, preserve_dtype: bool = True) -> dict[str, dict[str, np.ndarray]]: """Export a specific output image as numpy array and store it in object exports_dic. @@ -184,7 +132,7 @@ class OTBObject: Args: key: parameter key to export, if None then the default one will be used preserve_dtype: when set to True, the numpy array is converted to the same pixel type as - the OTBObject first output. Default is True + the App first output. Default is True Returns: the exported numpy array @@ -204,7 +152,7 @@ class OTBObject: Args: key: the output parameter name to export as numpy array preserve_dtype: when set to True, the numpy array is converted to the same pixel type as - the OTBObject first output. Default is True + the App first output. Default is True copy: whether to copy the output array, default is False required to True if preserve_dtype is False and the source app reference is lost @@ -250,55 +198,7 @@ class OTBObject: row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x return abs(int(row)), int(col) - # Special functions - def __hash__(self): - """Override the default behaviour of the hash function. - - Returns: - self hash - - """ - return id(self) - - def __getitem__(self, key): - """Override the default __getitem__ behaviour. - - This function enables 2 things : - - access attributes like that : object['any_attribute'] - - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3] - selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]] - selecting 1000x1000 subset : object[:1000, :1000] - - access pixel value(s) at a specified row, col index - - Args: - key: attribute key - - Returns: - attribute, pixel values or Slicer - - """ - # Accessing string attributes - if isinstance(key, str): - return self.__dict__.get(key) - # Accessing pixel value(s) using Y/X coordinates - if isinstance(key, tuple) and len(key) >= 2: - row, col = key[0], key[1] - if isinstance(row, int) and isinstance(col, int): - if row < 0 or col < 0: - raise ValueError(f"{self.name}: can't read pixel value at negative coordinates ({row}, {col})") - channels = None - if len(key) == 3: - channels = key[2] - return self.read_values_at_coords(row, col, channels) - # Slicing - if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)): - raise ValueError(f'"{key}"cannot be interpreted as valid slicing. Slicing should be 2D or 3D.') - if isinstance(key, tuple) and len(key) == 2: - # Adding a 3rd dimension - key = key + (slice(None, None, None),) - return Slicer(self, *key) - - def __add__(self, other: OTBObject | str | int | float) -> Operation: + def __add__(self, other: App | str | int | float) -> Operation: """Overrides the default addition and flavours it with BandMathX. Args: @@ -312,7 +212,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("+", self, other) - def __sub__(self, other: OTBObject | str | int | float) -> Operation: + def __sub__(self, other: App | str | int | float) -> Operation: """Overrides the default subtraction and flavours it with BandMathX. Args: @@ -326,7 +226,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("-", self, other) - def __mul__(self, other: OTBObject | str | int | float) -> Operation: + def __mul__(self, other: App | str | int | float) -> Operation: """Overrides the default subtraction and flavours it with BandMathX. Args: @@ -340,7 +240,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("*", self, other) - def __truediv__(self, other: OTBObject | str | int | float) -> Operation: + def __truediv__(self, other: App | str | int | float) -> Operation: """Overrides the default subtraction and flavours it with BandMathX. Args: @@ -354,7 +254,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("/", self, other) - def __radd__(self, other: OTBObject | str | int | float) -> Operation: + def __radd__(self, other: App | str | int | float) -> Operation: """Overrides the default reverse addition and flavours it with BandMathX. Args: @@ -368,7 +268,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("+", other, self) - def __rsub__(self, other: OTBObject | str | int | float) -> Operation: + def __rsub__(self, other: App | str | int | float) -> Operation: """Overrides the default subtraction and flavours it with BandMathX. Args: @@ -382,7 +282,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("-", other, self) - def __rmul__(self, other: OTBObject | str | int | float) -> Operation: + def __rmul__(self, other: App | str | int | float) -> Operation: """Overrides the default multiplication and flavours it with BandMathX. Args: @@ -396,7 +296,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("*", other, self) - def __rtruediv__(self, other: OTBObject | str | int | float) -> Operation: + def __rtruediv__(self, other: App | str | int | float) -> Operation: """Overrides the default division and flavours it with BandMathX. Args: @@ -419,7 +319,7 @@ class OTBObject: """ return Operation("abs", self) - def __ge__(self, other: OTBObject | str | int | float) -> Operation: + def __ge__(self, other: App | str | int | float) -> Operation: """Overrides the default greater or equal and flavours it with BandMathX. Args: @@ -433,7 +333,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation(">=", self, other) - def __le__(self, other: OTBObject | str | int | float) -> Operation: + def __le__(self, other: App | str | int | float) -> Operation: """Overrides the default less or equal and flavours it with BandMathX. Args: @@ -447,7 +347,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation("<=", self, other) - def __gt__(self, other: OTBObject | str | int | float) -> Operation: + def __gt__(self, other: App | str | int | float) -> Operation: """Overrides the default greater operator and flavours it with BandMathX. Args: @@ -461,7 +361,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation(">", self, other) - def __lt__(self, other: OTBObject | str | int | float) -> Operation: + def __lt__(self, other: App | str | int | float) -> Operation: """Overrides the default less operator and flavours it with BandMathX. Args: @@ -475,7 +375,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation("<", self, other) - def __eq__(self, other: OTBObject | str | int | float) -> Operation: + def __eq__(self, other: App | str | int | float) -> Operation: """Overrides the default eq operator and flavours it with BandMathX. Args: @@ -489,7 +389,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation("==", self, other) - def __ne__(self, other: OTBObject | str | int | float) -> Operation: + def __ne__(self, other: App | str | int | float) -> Operation: """Overrides the default different operator and flavours it with BandMathX. Args: @@ -503,7 +403,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation("!=", self, other) - def __or__(self, other: OTBObject | str | int | float) -> Operation: + def __or__(self, other: App | str | int | float) -> Operation: """Overrides the default or operator and flavours it with BandMathX. Args: @@ -517,7 +417,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation("||", self, other) - def __and__(self, other: OTBObject | str | int | float) -> Operation: + def __and__(self, other: App | str | int | float) -> Operation: """Overrides the default and operator and flavours it with BandMathX. Args: @@ -543,7 +443,7 @@ class OTBObject: """ return self.to_numpy() - def __array_ufunc__(self, ufunc, method, *inputs, **kwargs) -> OTBObject: + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs) -> App: """This is called whenever a numpy function is called on a pyotb object. Operation is performed in numpy, then imported back to pyotb with the same georeference as input. @@ -566,7 +466,7 @@ class OTBObject: for inp in inputs: if isinstance(inp, (float, int, np.ndarray, np.generic)): arrays.append(inp) - elif isinstance(inp, OTBObject): + elif isinstance(inp, App): if not inp.exports_dic: inp.export() image_dic = inp.exports_dic[inp.key_output_image] @@ -590,19 +490,18 @@ class OTBObject: return NotImplemented -class App(OTBObject): +class App(RasterInterface): """Base class that gathers common operations for any OTB application.""" - def __init__(self, otb_app_name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, - name: str = None, **kwargs): + def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, **kwargs): """Common constructor for OTB applications. Handles in-memory connection between apps. Args: - otb_app_name: name of the OTB application, e.g. 'BandMath' + name: name of the app, e.g. 'BandMath' *args: used for passing application parameters. Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") - - string or OTBObject, useful when the user wants to specify the input "in" + - string, App or Output, useful when the user wants to specify the input "in" - list, useful when the user wants to specify the input list 'il' frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___ quiet: whether to print logs of the OTB app @@ -610,19 +509,26 @@ class App(OTBObject): the result of app.ExportImage(). Use it when the app takes a numpy array as input. See this related issue for why it is necessary to keep reference of object: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824 - name: override the application name + **kwargs: used for passing application parameters. e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif' """ + self.parameters = {} + self.name = name self.frozen = frozen self.quiet = quiet + self.image_dic = image_dic + self.exports_dic = {} create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication - super().__init__(name=name or f"OTB Application {otb_app_name}", app=create(otb_app_name), image_dic=image_dic) - self.description = self.app.GetDocLongDescription() - - # Set parameters - self.parameters = {} + self.app = create(name) + self.parameters_keys = tuple(self.app.GetParametersKeys()) + self.all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys} + self.out_param_types = {k: v for k, v in self.all_param_types.items() + if v in (otb.ParameterType_OutputImage, + otb.ParameterType_OutputVectorData, + otb.ParameterType_OutputFilename)} + self.time_start, self.time_end = 0, 0 if args or kwargs: self.set_parameters(*args, **kwargs) if not self.frozen: @@ -630,8 +536,32 @@ class App(OTBObject): if any(key in self.parameters for key in self.out_param_types): self.flush() # auto flush if any output param was provided during app init - # Elapsed time - self.time_start, self.time_end = 0, 0 + def get_first_key(self, param_types: list[str]) -> str: + """Get the first output param key for specific file types.""" + for key, param_type in sorted(self.all_param_types.items()): + if param_type in param_types: + return key + return None + + @property + def key_input(self) -> str: + """Get the name of first input parameter, raster > vector > file.""" + return self.get_first_key(param_types=[otb.ParameterType_InputImage, + otb.ParameterType_InputImageList]) \ + or self.get_first_key(param_types=[otb.ParameterType_InputVectorData, + otb.ParameterType_InputVectorDataList]) \ + or self.get_first_key(param_types=[otb.ParameterType_InputFilename, + otb.ParameterType_InputFilenameList]) + + @property + def key_input_image(self) -> str: + """Get the name of first input image parameter.""" + return self.get_first_key(param_types=[otb.ParameterType_InputImage, otb.ParameterType_InputImageList]) + + @property + def key_output_image(self) -> str: + """Get the name of first output image parameter.""" + return self.get_first_key(param_types=[otb.ParameterType_OutputImage]) @property def elapsed_time(self): @@ -679,7 +609,7 @@ class App(OTBObject): Args: *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") - - string or OTBObject, useful when the user implicitly wants to set the param "in" + - string or App, useful when the user implicitly wants to set the param "in" - list, useful when the user implicitly wants to set the param "il" **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif' @@ -712,6 +642,36 @@ class App(OTBObject): self.parameters.update({**parameters, **otb_params}) self.save_objects() + def propagate_dtype(self, target_key: str = None, dtype: int = None): + """Propagate a pixel type from main input to every outputs, or to a target output key only. + + With multiple inputs (if dtype is not provided), the type of the first input is considered. + With multiple outputs (if target_key is not provided), all outputs will be converted to the same pixel type. + + Args: + target_key: output param key to change pixel type + dtype: data type to use + + """ + if not dtype: + param = self.parameters.get(self.key_input_image) + if not param: + logger.warning("%s: could not propagate pixel type from inputs to output", self.name) + return + if isinstance(param, (list, tuple)): + param = param[0] # first image in "il" + try: + dtype = get_pixel_type(param) + except (TypeError, RuntimeError): + logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param) + return + if target_key: + keys = [target_key] + else: + keys = [k for k, v in self.out_param_types.items() if v == otb.ParameterType_OutputImage] + for key in keys: + self.app.SetParameterOutputImagePixelType(key, dtype) + def save_objects(self): """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`. @@ -819,38 +779,24 @@ class App(OTBObject): self.set_parameters({key: output_filename}) self.flush() - def propagate_dtype(self, target_key: str = None, dtype: int = None): - """Propagate a pixel type from main input to every outputs, or to a target output key only. - - With multiple inputs (if dtype is not provided), the type of the first input is considered. - With multiple outputs (if target_key is not provided), all outputs will be converted to the same pixel type. + def summarize(self) -> dict: + """Serialize an object and its pipeline into a dictionary. - Args: - target_key: output param key to change pixel type - dtype: data type to use + Returns: + nested dictionary summarizing the pipeline """ - if not dtype: - param = self.parameters.get(self.key_input_image) - if not param: - logger.warning("%s: could not propagate pixel type from inputs to output", self.name) - return - if isinstance(param, (list, tuple)): - param = param[0] # first image in "il" - try: - dtype = get_pixel_type(param) - except (TypeError, RuntimeError): - logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param) - return - if target_key: - keys = [target_key] - else: - keys = [k for k, v in self.out_param_types.items() if v == otb.ParameterType_OutputImage] - for key in keys: - self.app.SetParameterOutputImagePixelType(key, dtype) + params = self.parameters + for k, p in params.items(): + # In the following, we replace each parameter which is an App, with its summary. + if isinstance(p, App): # single parameter + params[k] = p.summarize() + elif isinstance(p, list): # parameter list + params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p] + return {"name": self.app.GetName(), "parameters": params} # Private functions - def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]: + def __parse_args(self, args: list[str | App | dict | list]) -> dict[str, Any]: """Gather all input arguments in kwargs dict. Args: @@ -864,11 +810,11 @@ class App(OTBObject): for arg in args: if isinstance(arg, dict): kwargs.update(arg) - elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.key_input): + elif isinstance(arg, (str, App)) or isinstance(arg, list) and is_key_list(self, self.key_input): kwargs.update({self.key_input: arg}) return kwargs - def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]): + def __set_param(self, key: str, obj: list | tuple | App | otb.Application | list[Any]): """Set one parameter, decide which otb.Application method to use depending on target object.""" if obj is None or (isinstance(obj, (list, tuple)) and not obj): self.app.ClearValue(key) @@ -878,7 +824,7 @@ class App(OTBObject): f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}" ) # Single-parameter cases - if isinstance(obj, OTBObject): + if isinstance(obj, App): self.app.ConnectImage(key, obj.app, obj.key_output_image) elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0]) @@ -890,7 +836,7 @@ class App(OTBObject): elif is_key_images_list(self, key): # To enable possible in-memory connections, we go through the list and set the parameters one by one for inp in obj: - if isinstance(inp, OTBObject): + if isinstance(inp, App): self.app.ConnectImage(key, inp.app, inp.key_output_image) elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0]) @@ -901,21 +847,53 @@ class App(OTBObject): else: self.app.SetParameterValue(key, obj) - def summarize(self) -> dict: - """Serialize an object and its pipeline into a dictionary. + # Special functions + def __hash__(self): + """Override the default behaviour of the hash function. Returns: - nested dictionary summarizing the pipeline + self hash """ - params = self.parameters - for k, p in params.items(): - # In the following, we replace each parameter which is an OTBObject, with its summary. - if isinstance(p, App): # single parameter - params[k] = p.summarize() - elif isinstance(p, list): # parameter list - params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p] - return {"name": self.app.GetName(), "parameters": params} + return id(self) + + def __getitem__(self, key): + """Override the default __getitem__ behaviour. + + This function enables 2 things : + - access attributes like that : object['any_attribute'] + - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3] + selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]] + selecting 1000x1000 subset : object[:1000, :1000] + - access pixel value(s) at a specified row, col index + + Args: + key: attribute key + + Returns: + attribute, pixel values or Slicer + + """ + # Accessing string attributes + if isinstance(key, str): + return self.__dict__.get(key) + # Accessing pixel value(s) using Y/X coordinates + if isinstance(key, tuple) and len(key) >= 2: + row, col = key[0], key[1] + if isinstance(row, int) and isinstance(col, int): + if row < 0 or col < 0: + raise ValueError(f"{self.name}: can't read pixel value at negative coordinates ({row}, {col})") + channels = None + if len(key) == 3: + channels = key[2] + return self.read_values_at_coords(row, col, channels) + # Slicing + if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)): + raise ValueError(f'"{key}"cannot be interpreted as valid slicing. Slicing should be 2D or 3D.') + if isinstance(key, tuple) and len(key) == 2: + # Adding a 3rd dimension + key = key + (slice(None, None, None),) + return Slicer(self, *key) def __str__(self): """Return a nice string representation with object id.""" @@ -925,7 +903,7 @@ class App(OTBObject): class Slicer(App): """Slicer objects i.e. when we call something like raster[:, :, 2] from Python.""" - def __init__(self, obj: OTBObject | str, rows: int, cols: int, channels: int): + def __init__(self, obj: App | str, rows: int, cols: int, channels: int): """Create a slicer object, that can be used directly for writing or inside a BandMath. It contains : @@ -939,7 +917,8 @@ class Slicer(App): channels: channels, can be slicing, list or int """ - super().__init__("ExtractROI", {"in": obj, "mode": "extent"}, quiet=True, frozen=True, name="Slicer") + self.name = "Slicer" + super().__init__("ExtractROI", obj, mode="extent", quiet=True, frozen=True) self.rows, self.cols = rows, cols parameters = {} @@ -1009,7 +988,7 @@ class Operation(App): """ - def __init__(self, operator: str, *inputs, nb_bands: int = None, name: str = None): + def __init__(self, operator: str, *inputs, nb_bands: int = None): """Given some inputs and an operator, this function enables to transform this into an OTB application. Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator. @@ -1017,7 +996,7 @@ class Operation(App): Args: operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? - *inputs: inputs. Can be OTBObject, filepath, int or float + *inputs: inputs. Can be App, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where name: override the Operation name @@ -1045,11 +1024,12 @@ class Operation(App): # Getting unique image inputs, in the order im1, im2, im3 ... self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)] self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands) + self.name = f'Operation exp="{self.exp}"' + appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX" # Execute app - super().__init__("BandMath" if len(self.exp_bands) == 1 else "BandMathX", il=self.unique_inputs, - exp=self.exp, quiet=True, name=name or f'Operation exp="{self.exp}"') + super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True) - def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float], + def create_fake_exp(self, operator: str, inputs: list[App | str | int | float], nb_bands: int = None): """Create a 'fake' expression. @@ -1057,7 +1037,7 @@ class Operation(App): Args: operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? - inputs: inputs. Can be OTBObject, filepath, int or float + inputs: inputs. Can be App, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ @@ -1092,14 +1072,10 @@ class Operation(App): cond_band = 1 else: cond_band = band - fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp( - inp, cond_band, keep_logical=True - ) + fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp(inp, cond_band, keep_logical=True) else: # Any other input - fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp( - inp, band, keep_logical=False - ) + fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp(inp, band, keep_logical=False) fake_exps.append(fake_exp) # Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates) if i == 0 and corresponding_inputs and nb_channels: @@ -1141,8 +1117,7 @@ class Operation(App): return exp_bands, exp @staticmethod - def create_one_input_fake_exp(x: OTBObject | str, - band: int, keep_logical: bool = False) -> tuple(str, list[OTBObject], int): + def make_fake_exp(x: App | str, band: int, keep_logical: bool = False) -> tuple(str, list[App], int): """This an internal function, only to be used by `create_fake_exp`. Enable to create a fake expression just for one input and one band. @@ -1211,6 +1186,7 @@ class LogicalOperation(Operation): logical expression (e.g. "im1b1 > 0") """ + name = "LogicalOperation" def __init__(self, operator: str, *inputs, nb_bands: int = None): """Constructor for a LogicalOperation object. @@ -1221,10 +1197,10 @@ class LogicalOperation(Operation): nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ - super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation") + super().__init__(operator, *inputs, nb_bands=nb_bands) self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands) - def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None): + def create_fake_exp(self, operator: str, inputs: list[App | str | int | float], nb_bands: int = None): """Create a 'fake' expression. e.g for the operation input1 > input2, we create a fake expression that is like @@ -1232,7 +1208,7 @@ class LogicalOperation(Operation): Args: operator: str (one of >, <, >=, <=, ==, !=, &, |) - inputs: Can be OTBObject, filepath, int or float + inputs: Can be App, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ @@ -1251,7 +1227,7 @@ class LogicalOperation(Operation): for i, band in enumerate(range(1, nb_bands + 1)): fake_exps = [] for inp in inputs: - fake_exp, corresp_inputs, nb_channels = super().create_one_input_fake_exp(inp, band, keep_logical=True) + fake_exp, corresp_inputs, nb_channels = super().make_fake_exp(inp, band, keep_logical=True) fake_exps.append(fake_exp) # Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates) if i == 0 and corresp_inputs and nb_channels: @@ -1278,7 +1254,8 @@ class Input(App): """ self.path = path - super().__init__("ExtractROI", {"in": path}, frozen=True, name=f"Input from {path}") + self.name = f"Input from {path}" + super().__init__("ExtractROI", {"in": path}, frozen=True) self.propagate_dtype() self.execute() @@ -1287,11 +1264,10 @@ class Input(App): return f"<pyotb.Input object from {self.path}>" -class Output(OTBObject): +class Output(RasterInterface): """Object that behave like a pointer to a specific application output file.""" - def __init__(self, pyotb_app: OTBObject, # pylint: disable=super-init-not-called - param_key: str = None, filepath: str = None, mkdir: bool = True): + def __init__(self, pyotb_app: App, param_key: str = None, filepath: str = None, mkdir: bool = True): """Constructor for an Output object. Args: @@ -1301,9 +1277,10 @@ class Output(OTBObject): mkdir: create missing parent directories """ - super().__init__(name=f"Output {param_key} from {pyotb_app.name}", app=pyotb_app.app) - self.parent_pyotb_app = pyotb_app # keep trace of parent app - self.param_key = param_key or super().key_output_image + self.name = f"Output {param_key} from {pyotb_app.name}" + self.parent_app = pyotb_app # keep trace of parent app + self.app = pyotb_app.app + self.param_key = param_key self.filepath = None if filepath: if '?' in filepath: @@ -1314,7 +1291,6 @@ class Output(OTBObject): @property def key_output_image(self): - """Overwrite OTBObject prop, in order to use Operation special methods with the right Output param_key.""" return self.param_key def exists(self) -> bool: @@ -1328,12 +1304,18 @@ class Output(OTBObject): if not self.filepath.parent.exists(): self.filepath.parent.mkdir(parents=True) + def write(self, filepath: str = "", **kwargs): + """Write output to disk, filepath is not required if it was provided to parent App during init""" + if not filepath and self.filepath: + return self.parent_app.write({self.key_output_image: self.filepath}, **kwargs) + return self.parent_app.write({self.key_output_image: filepath}, **kwargs) + def __str__(self) -> str: """Return a nice string representation with source app name and object id.""" return f"<pyotb.Output {self.name} object, id {id(self)}>" -def get_nbchannels(inp: str | OTBObject) -> int: +def get_nbchannels(inp: str | App) -> int: """Get the nb of bands of input image. Args: @@ -1343,7 +1325,7 @@ def get_nbchannels(inp: str | OTBObject) -> int: number of bands in image """ - if isinstance(inp, OTBObject): + if isinstance(inp, App): nb_channels = inp.shape[-1] else: # Executing the app, without printing its log @@ -1355,7 +1337,7 @@ def get_nbchannels(inp: str | OTBObject) -> int: return nb_channels -def get_pixel_type(inp: str | OTBObject) -> str: +def get_pixel_type(inp: str | App) -> str: """Get the encoding of input image pixels. Args: @@ -1363,7 +1345,7 @@ def get_pixel_type(inp: str | OTBObject) -> str: Returns: pixel_type: OTB enum e.g. `otbApplication.ImagePixelType_uint8', which actually is an int. - For an App with several outputs, only the pixel type of the first output is returned + make_fake_exp For an App with several outputs, only the pixel type of the first output is returned """ if isinstance(inp, str): @@ -1388,7 +1370,7 @@ def get_pixel_type(inp: str | OTBObject) -> str: if datatype not in datatype_to_pixeltype: raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}") pixel_type = getattr(otb, f'ImagePixelType_{datatype_to_pixeltype[datatype]}') - elif isinstance(inp, OTBObject): + elif isinstance(inp, App): pixel_type = inp.app.GetParameterOutputImagePixelType(inp.key_output_image) else: raise TypeError(f'Could not get the pixel type of {type(inp)} object {inp}') @@ -1412,7 +1394,7 @@ def parse_pixel_type(pixel_type: str | int) -> int: raise ValueError(f'Bad pixel type specification ({pixel_type})') -def is_key_list(pyotb_app: OTBObject, key: str) -> bool: +def is_key_list(pyotb_app: App, key: str) -> bool: """Check if a key of the App is an input parameter list.""" return pyotb_app.app.GetParameterType(key) in ( otb.ParameterType_InputImageList, @@ -1423,7 +1405,7 @@ def is_key_list(pyotb_app: OTBObject, key: str) -> bool: ) -def is_key_images_list(pyotb_app: OTBObject, key: str) -> bool: +def is_key_images_list(pyotb_app: App, key: str) -> bool: """Check if a key of the App is an input parameter image list.""" return pyotb_app.app.GetParameterType(key) in ( otb.ParameterType_InputImageList, @@ -1431,6 +1413,6 @@ def is_key_images_list(pyotb_app: OTBObject, key: str) -> bool: ) -def get_out_images_param_keys(app: OTBObject) -> list[str]: +def get_out_images_param_keys(app: App) -> list[str]: """Return every output parameter keys of an OTB app.""" return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage] diff --git a/pyotb/functions.py b/pyotb/functions.py index d25610a..250f79e 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -9,11 +9,11 @@ import textwrap import subprocess from collections import Counter -from .core import OTBObject, App, Operation, LogicalOperation, Input, get_nbchannels +from .core import App, Operation, LogicalOperation, Input, get_nbchannels from .helpers import logger -def where(cond: OTBObject | str, x: OTBObject | str | int | float, y: OTBObject | str | int | float) -> Operation: +def where(cond: App | str, x: App | str | int | float, y: App | str | int | float) -> Operation: """Functionally similar to numpy.where. Where cond is True (!=0), returns x. Else returns y. Args: @@ -63,7 +63,7 @@ def where(cond: OTBObject | str, x: OTBObject | str | int | float, y: OTBObject return operation -def clip(a: OTBObject | str, a_min: OTBObject | str | int | float, a_max: OTBObject | str | int | float): +def clip(a: App | str, a_min: App | str | int | float, a_max: App | str | int | float): """Clip values of image in a range of values. Args: @@ -324,7 +324,7 @@ def run_tf_function(func): def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_rule: str = 'minimal', interpolator: str = 'nn', reference_window_input: dict = None, - reference_pixel_size_input: str = None) -> list[OTBObject]: + reference_pixel_size_input: str = None) -> list[App]: """Given several inputs, this function handles the potential resampling and cropping to same extent. WARNING: Not fully implemented / tested @@ -354,7 +354,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ for inp in inputs: if isinstance(inp, str): # this is for filepaths metadata = Input(inp).app.GetImageMetaData('out') - elif isinstance(inp, OTBObject): + elif isinstance(inp, App): metadata = inp.app.GetImageMetaData(inp.output_param) else: raise TypeError(f"Wrong input : {inp}") diff --git a/tests/test_core.py b/tests/test_core.py index 086c26e..0d1fc6f 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,3 +1,4 @@ +import os import pytest import pyotb @@ -56,7 +57,7 @@ def test_nonraster_property(): def test_elapsed_time(): - assert pyotb.ReadImageInfo(INPUT).elapsed_time < 1 + assert 0 < pyotb.ReadImageInfo(INPUT).elapsed_time < 1 # Other functions @@ -76,6 +77,13 @@ def test_xy_to_rowcol(): def test_write(): INPUT.write("/tmp/missing_dir/test_write.tif") assert INPUT.out.exists() + os.remove("/tmp/missing_dir/test_write.tif") + + +def test_output_write(): + INPUT.out.write("/tmp/missing_dir/test_write.tif") + assert INPUT.out.exists() + os.remove("/tmp/missing_dir/test_write.tif") # Slicer diff --git a/tests/test_numpy.py b/tests/test_numpy.py index 0f42435..682fe2a 100644 --- a/tests/test_numpy.py +++ b/tests/test_numpy.py @@ -1,10 +1,6 @@ import numpy as np import pyotb from tests_data import INPUT -import numpy as np - -import pyotb -from tests_data import INPUT def test_export(): @@ -43,7 +39,7 @@ def test_pixel_coords_otb_equals_numpy(): def test_add_noise_array(): white_noise = np.random.normal(0, 50, size=INPUT.shape) noisy_image = INPUT + white_noise - assert isinstance(noisy_image, pyotb.core.OTBObject) + assert isinstance(noisy_image, pyotb.core.App) assert noisy_image.shape == INPUT.shape diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 3acc354..76f9872 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -103,8 +103,7 @@ PIPELINES, NAMES = make_pipelines_list() @pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES) def test_pipeline_shape(pipe): - for i, app in enumerate(pipe): - print(app.shape) + for app in pipe: assert bool(app.shape) @@ -116,7 +115,7 @@ def test_pipeline_shape_nointermediate(pipe): @pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES) def test_pipeline_shape_backward(pipe): - for i, app in enumerate(reversed(pipe)): + for app in reversed(pipe): assert bool(app.shape) -- GitLab From ab38817c10fd43d067f7aa09557abf5947268902 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 00:02:50 +0100 Subject: [PATCH 071/399] CI: linting + test use filepath --- pyotb/core.py | 6 +++--- tests/test_core.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 7928356..4d3d848 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -902,6 +902,7 @@ class App(RasterInterface): class Slicer(App): """Slicer objects i.e. when we call something like raster[:, :, 2] from Python.""" + name = "Slicer" def __init__(self, obj: App | str, rows: int, cols: int, channels: int): """Create a slicer object, that can be used directly for writing or inside a BandMath. @@ -917,7 +918,6 @@ class Slicer(App): channels: channels, can be slicing, list or int """ - self.name = "Slicer" super().__init__("ExtractROI", obj, mode="extent", quiet=True, frozen=True) self.rows, self.cols = rows, cols parameters = {} @@ -1186,7 +1186,6 @@ class LogicalOperation(Operation): logical expression (e.g. "im1b1 > 0") """ - name = "LogicalOperation" def __init__(self, operator: str, *inputs, nb_bands: int = None): """Constructor for a LogicalOperation object. @@ -1291,6 +1290,7 @@ class Output(RasterInterface): @property def key_output_image(self): + """Force the right key to be used when accessing the RasterInterface.""" return self.param_key def exists(self) -> bool: @@ -1305,7 +1305,7 @@ class Output(RasterInterface): self.filepath.parent.mkdir(parents=True) def write(self, filepath: str = "", **kwargs): - """Write output to disk, filepath is not required if it was provided to parent App during init""" + """Write output to disk, filepath is not required if it was provided to parent App during init.""" if not filepath and self.filepath: return self.parent_app.write({self.key_output_image: self.filepath}, **kwargs) return self.parent_app.write({self.key_output_image: filepath}, **kwargs) diff --git a/tests/test_core.py b/tests/test_core.py index 0d1fc6f..13aa1b0 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -75,15 +75,16 @@ def test_xy_to_rowcol(): def test_write(): - INPUT.write("/tmp/missing_dir/test_write.tif") + INPUT.write("/tmp/test_write.tif") assert INPUT.out.exists() - os.remove("/tmp/missing_dir/test_write.tif") + INPUT.out.filepath.unlink() def test_output_write(): INPUT.out.write("/tmp/missing_dir/test_write.tif") assert INPUT.out.exists() os.remove("/tmp/missing_dir/test_write.tif") + INPUT.out.filepath.unlink() # Slicer -- GitLab From 14f236940ee641bdc6f2e846761258b3c3d1078c Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 00:08:44 +0100 Subject: [PATCH 072/399] CI: remove bad test --- tests/test_core.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 13aa1b0..ccd386f 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -80,13 +80,6 @@ def test_write(): INPUT.out.filepath.unlink() -def test_output_write(): - INPUT.out.write("/tmp/missing_dir/test_write.tif") - assert INPUT.out.exists() - os.remove("/tmp/missing_dir/test_write.tif") - INPUT.out.filepath.unlink() - - # Slicer def test_slicer_shape(): extract = INPUT[:50, :60, :3] -- GitLab From 5fd852ef19b776b6d5fa85491f6aa47473de8a4b Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 00:21:54 +0100 Subject: [PATCH 073/399] FIX: do not overwrite Slicer and Operation name with App name --- pyotb/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 4d3d848..e90286f 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -902,7 +902,6 @@ class App(RasterInterface): class Slicer(App): """Slicer objects i.e. when we call something like raster[:, :, 2] from Python.""" - name = "Slicer" def __init__(self, obj: App | str, rows: int, cols: int, channels: int): """Create a slicer object, that can be used directly for writing or inside a BandMath. @@ -919,6 +918,7 @@ class Slicer(App): """ super().__init__("ExtractROI", obj, mode="extent", quiet=True, frozen=True) + self.name = "Slicer" self.rows, self.cols = rows, cols parameters = {} @@ -1024,10 +1024,10 @@ class Operation(App): # Getting unique image inputs, in the order im1, im2, im3 ... self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)] self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands) - self.name = f'Operation exp="{self.exp}"' appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX" # Execute app super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True) + self.name = f'Operation exp="{self.exp}"' def create_fake_exp(self, operator: str, inputs: list[App | str | int | float], nb_bands: int = None): -- GitLab From 4ca9f91f385d64cd8043b6b0855b6915f8cbd016 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 10:37:17 +0100 Subject: [PATCH 074/399] ENH: trying to make Slicer fake_exp functions clearer --- pyotb/core.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index e90286f..4df617f 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1008,7 +1008,7 @@ class Operation(App): self.nb_channels = {} self.fake_exp_bands = [] self.logical_fake_exp_bands = [] - self.create_fake_exp(operator, inputs, nb_bands=nb_bands) + self.build_fake_expressions(operator, inputs, nb_bands=nb_bands) # Transforming images to the adequate im#, e.g. `input1` to "im1" # creating a dictionary that is like {str(input1): 'im1', 'image2.tif': 'im2', ...}. # NB: the keys of the dictionary are strings-only, instead of 'complex' objects, to enable easy serialization @@ -1029,9 +1029,8 @@ class Operation(App): super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True) self.name = f'Operation exp="{self.exp}"' - def create_fake_exp(self, operator: str, inputs: list[App | str | int | float], - nb_bands: int = None): - """Create a 'fake' expression. + def build_fake_expressions(self, operator: str, inputs: list[App | str | int | float], nb_bands: int = None): + """Create a list of 'fake' expressions, one for each band. E.g for the operation input1 + input2, we create a fake expression that is like "str(input1) + str(input2)" @@ -1062,7 +1061,7 @@ class Operation(App): # Create a list of fake expressions, each item of the list corresponding to one band self.fake_exp_bands.clear() for i, band in enumerate(range(1, nb_bands + 1)): - fake_exps = [] + expressions = [] for k, inp in enumerate(inputs): # Generating the fake expression of the current input, # this is a special case for the condition of the ternary operator `cond ? x : y` @@ -1076,7 +1075,7 @@ class Operation(App): else: # Any other input fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp(inp, band, keep_logical=False) - fake_exps.append(fake_exp) + expressions.append(fake_exp) # Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates) if i == 0 and corresponding_inputs and nb_channels: self.inputs.extend(corresponding_inputs) @@ -1084,13 +1083,13 @@ class Operation(App): # Generating the fake expression of the whole operation if len(inputs) == 1: # this is only for 'abs' - fake_exp = f"({operator}({fake_exps[0]}))" + fake_exp = f"({operator}({expressions[0]}))" elif len(inputs) == 2: # We create here the "fake" expression. For example, for a BandMathX expression such as '2 * im1 + im2', # the false expression stores the expression 2 * str(input1) + str(input2) - fake_exp = f"({fake_exps[0]} {operator} {fake_exps[1]})" + fake_exp = f"({expressions[0]} {operator} {expressions[1]})" elif len(inputs) == 3 and operator == "?": # this is only for ternary expression - fake_exp = f"({fake_exps[0]} ? {fake_exps[1]} : {fake_exps[2]})" + fake_exp = f"({expressions[0]} ? {expressions[1]} : {expressions[2]})" self.fake_exp_bands.append(fake_exp) def get_real_exp(self, fake_exp_bands: str) -> tuple(list[str], str): @@ -1118,7 +1117,7 @@ class Operation(App): @staticmethod def make_fake_exp(x: App | str, band: int, keep_logical: bool = False) -> tuple(str, list[App], int): - """This an internal function, only to be used by `create_fake_exp`. + """This an internal function, only to be used by `build_fake_expressions`. Enable to create a fake expression just for one input and one band. @@ -1199,8 +1198,8 @@ class LogicalOperation(Operation): super().__init__(operator, *inputs, nb_bands=nb_bands) self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands) - def create_fake_exp(self, operator: str, inputs: list[App | str | int | float], nb_bands: int = None): - """Create a 'fake' expression. + def build_fake_expressions(self, operator: str, inputs: list[App | str | int | float], nb_bands: int = None): + """Create a list of 'fake' expressions, one for each band. e.g for the operation input1 > input2, we create a fake expression that is like "str(input1) > str(input2) ? 1 : 0" and a logical fake expression that is like "str(input1) > str(input2)" @@ -1224,17 +1223,17 @@ class LogicalOperation(Operation): # Create a list of fake exp, each item of the list corresponding to one band for i, band in enumerate(range(1, nb_bands + 1)): - fake_exps = [] + expressions = [] for inp in inputs: fake_exp, corresp_inputs, nb_channels = super().make_fake_exp(inp, band, keep_logical=True) - fake_exps.append(fake_exp) + expressions.append(fake_exp) # Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates) if i == 0 and corresp_inputs and nb_channels: self.inputs.extend(corresp_inputs) self.nb_channels.update(nb_channels) # We create here the "fake" expression. For example, for a BandMathX expression such as 'im1 > im2', # the logical fake expression stores the expression "str(input1) > str(input2)" - logical_fake_exp = f"({fake_exps[0]} {operator} {fake_exps[1]})" + logical_fake_exp = f"({expressions[0]} {operator} {expressions[1]})" # We keep the logical expression, useful if later combined with other logical operations self.logical_fake_exp_bands.append(logical_fake_exp) # We create a valid BandMath expression, e.g. "str(input1) > str(input2) ? 1 : 0" @@ -1345,7 +1344,7 @@ def get_pixel_type(inp: str | App) -> str: Returns: pixel_type: OTB enum e.g. `otbApplication.ImagePixelType_uint8', which actually is an int. - make_fake_exp For an App with several outputs, only the pixel type of the first output is returned + For an App with several outputs, only the pixel type of the first output is returned """ if isinstance(inp, str): -- GitLab From 923ce740dc61af2c443e86d510b551b65d93a403 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 11:03:03 +0100 Subject: [PATCH 075/399] STYLE: type hints + write allows Path type --- pyotb/core.py | 81 ++++++++++++++++++++++++++------------------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 4df617f..9278f8f 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -29,7 +29,7 @@ class RasterInterface(ABC): """Write image, this is defined in App. Output will use App.write for a specific key.""" @property - def metadata(self): + def metadata(self) -> dict[str, (str, float, list[float])]: """Return first output image metadata dictionary.""" return dict(self.app.GetMetadataDictionary(self.key_output_image)) @@ -45,7 +45,7 @@ class RasterInterface(ABC): return self.app.ConvertPixelTypeToNumpy(enum) @property - def shape(self) -> tuple(int): + def shape(self) -> tuple[int]: """Enables to retrieve the shape of a pyotb object using numpy convention. Returns: @@ -57,7 +57,7 @@ class RasterInterface(ABC): return height, width, bands @property - def transform(self) -> tuple(int): + def transform(self) -> tuple[int]: """Get image affine transform, rasterio style (see https://www.perrygeo.com/python-affine-transforms.html). Returns: @@ -69,11 +69,11 @@ class RasterInterface(ABC): origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2 return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y - def get_infos(self): + def get_infos(self) -> dict[str, (str, float, list[float])]: """Return a dict output of ReadImageInfo for the first image output.""" return App("ReadImageInfo", self, quiet=True).data - def get_statistics(self): + def get_statistics(self) -> dict[str, (str, float, list[float])]: """Return a dict output of ComputeImagesStatistics for the first image output.""" return App("ComputeImagesStatistics", self, quiet=True).data @@ -536,7 +536,7 @@ class App(RasterInterface): if any(key in self.parameters for key in self.out_param_types): self.flush() # auto flush if any output param was provided during app init - def get_first_key(self, param_types: list[str]) -> str: + def get_first_key(self, param_types: list[int]) -> str: """Get the first output param key for specific file types.""" for key, param_type in sorted(self.all_param_types.items()): if param_type in param_types: @@ -564,7 +564,7 @@ class App(RasterInterface): return self.get_first_key(param_types=[otb.ParameterType_OutputImage]) @property - def elapsed_time(self): + def elapsed_time(self) -> float: """Get elapsed time between app init and end of exec or file writing.""" return self.time_end - self.time_start @@ -573,23 +573,8 @@ class App(RasterInterface): """List of used application outputs.""" return [getattr(self, key) for key in self.out_param_types if key in self.parameters] - def find_outputs(self) -> tuple[str]: - """Find output files on disk using path found in parameters. - - Returns: - list of files found on disk - - """ - files, missing = [], [] - for out in self.used_outputs: - dest = files if out.exists() else missing - dest.append(str(out.filepath.absolute())) - for filename in missing: - logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) - return tuple(files) - @property - def data(self): + def data(self) -> dict[str, float, list[float]]: """Expose app's output data values in a dictionary.""" skip_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem") skip_keys = skip_keys + tuple(self.out_param_types) + tuple(self.parameters) @@ -746,8 +731,8 @@ class App(RasterInterface): kwargs.update(arg) elif isinstance(arg, str) and kwargs: logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg) - elif isinstance(arg, str) and self.key_output_image: - kwargs.update({self.key_output_image: arg}) + elif isinstance(arg, (str, Path)) and self.key_output_image: + kwargs.update({self.key_output_image: str(arg)}) # Append filename extension to filenames if filename_extension: @@ -779,7 +764,22 @@ class App(RasterInterface): self.set_parameters({key: output_filename}) self.flush() - def summarize(self) -> dict: + def find_outputs(self) -> tuple[str]: + """Find output files on disk using path found in parameters. + + Returns: + list of files found on disk + + """ + files, missing = [], [] + for out in self.used_outputs: + dest = files if out.exists() else missing + dest.append(str(out.filepath.absolute())) + for filename in missing: + logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) + return tuple(files) + + def summarize(self) -> dict[str, str | dict[str, Any]]: """Serialize an object and its pipeline into a dictionary. Returns: @@ -848,7 +848,7 @@ class App(RasterInterface): self.app.SetParameterValue(key, obj) # Special functions - def __hash__(self): + def __hash__(self) -> int: """Override the default behaviour of the hash function. Returns: @@ -857,7 +857,7 @@ class App(RasterInterface): """ return id(self) - def __getitem__(self, key): + def __getitem__(self, key) -> Any | list[int | float] | int | float | Slicer : """Override the default __getitem__ behaviour. This function enables 2 things : @@ -895,7 +895,7 @@ class App(RasterInterface): key = key + (slice(None, None, None),) return Slicer(self, *key) - def __str__(self): + def __str__(self) -> str: """Return a nice string representation with object id.""" return f"<pyotb.App {self.name} object id {id(self)}>" @@ -1092,7 +1092,7 @@ class Operation(App): fake_exp = f"({expressions[0]} ? {expressions[1]} : {expressions[2]})" self.fake_exp_bands.append(fake_exp) - def get_real_exp(self, fake_exp_bands: str) -> tuple(list[str], str): + def get_real_exp(self, fake_exp_bands: str) -> tuple[list[str], str]: """Generates the BandMathX expression. Args: @@ -1116,7 +1116,7 @@ class Operation(App): return exp_bands, exp @staticmethod - def make_fake_exp(x: App | str, band: int, keep_logical: bool = False) -> tuple(str, list[App], int): + def make_fake_exp(x: App | str, band: int, keep_logical: bool = False) -> tuple[str, list[App], int]: """This an internal function, only to be used by `build_fake_expressions`. Enable to create a fake expression just for one input and one band. @@ -1126,7 +1126,7 @@ class Operation(App): band: which band to consider (bands start at 1) keep_logical: whether to keep the logical expressions "as is" in case the input is a logical operation. ex: if True, for `input1 > input2`, returned fake expression is "str(input1) > str(input2)" - if False, for `input1 > input2`, returned fake exp is "str(input1) > str(input2) ? 1 : 0". + if False, for `input1 > input2`, returned fake exp is "str(input1) > str(input2) ? 1 : 0"] Default False Returns: @@ -1288,24 +1288,27 @@ class Output(RasterInterface): self.make_parent_dirs() @property - def key_output_image(self): + def key_output_image(self) -> str: """Force the right key to be used when accessing the RasterInterface.""" return self.param_key def exists(self) -> bool: """Check file exist.""" - assert self.filepath, "Filepath not set" + if self.filepath is None: + raise ValueError("Filepath is not set") return self.filepath.exists() def make_parent_dirs(self): """Create missing parent directories.""" - assert self.filepath, "Filepath not set" - if not self.filepath.parent.exists(): - self.filepath.parent.mkdir(parents=True) + if self.filepath is None: + raise ValueError("Filepath is not set") + self.filepath.parent.mkdir(parents=True, exist_ok=True) - def write(self, filepath: str = "", **kwargs): + def write(self, filepath: None | str | Path = None, **kwargs): """Write output to disk, filepath is not required if it was provided to parent App during init.""" - if not filepath and self.filepath: + if filepath is None and self.filepath: + if self.filepath.exists(): + logger.warning("Overwriting file %s", self.filepath) return self.parent_app.write({self.key_output_image: self.filepath}, **kwargs) return self.parent_app.write({self.key_output_image: filepath}, **kwargs) -- GitLab From 43d18833d897fb80eb358af5c813d183cbc0e0cd Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 11:12:08 +0100 Subject: [PATCH 076/399] STYLE: linter --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 9278f8f..19a23f9 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -857,7 +857,7 @@ class App(RasterInterface): """ return id(self) - def __getitem__(self, key) -> Any | list[int | float] | int | float | Slicer : + def __getitem__(self, key) -> Any | list[int | float] | int | float | Slicer: """Override the default __getitem__ behaviour. This function enables 2 things : -- GitLab From a133ed8bf2828b01b5b42303af34a2b173bffc3d Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 11:27:46 +0100 Subject: [PATCH 077/399] ENH: raise warning when overwriting file with App.write() --- pyotb/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 19a23f9..acf0541 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -759,6 +759,8 @@ class App(RasterInterface): # Set parameters and flush to disk for key, output_filename in kwargs.items(): + if Path(output_filename).exists(): + logger.warning("%s: overwriting file %s", self.name, output_filename) if key in dtypes: self.propagate_dtype(key, dtypes[key]) self.set_parameters({key: output_filename}) @@ -1307,8 +1309,6 @@ class Output(RasterInterface): def write(self, filepath: None | str | Path = None, **kwargs): """Write output to disk, filepath is not required if it was provided to parent App during init.""" if filepath is None and self.filepath: - if self.filepath.exists(): - logger.warning("Overwriting file %s", self.filepath) return self.parent_app.write({self.key_output_image: self.filepath}, **kwargs) return self.parent_app.write({self.key_output_image: filepath}, **kwargs) -- GitLab From eb735137d4739a4bc7cf2a5fcdab85a47d2d551f Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 11:31:45 +0100 Subject: [PATCH 078/399] ENH: fix raise error and print available values if a bad key is passed to set_parameters --- pyotb/core.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index acf0541..b9cc5a1 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -607,7 +607,9 @@ class App(RasterInterface): # Going through all arguments for key, obj in parameters.items(): if key not in self.parameters_keys: - raise KeyError(f'{self.name}: unknown parameter name "{key}"') + raise KeyError( + f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}" + ) # When the parameter expects a list, if needed, change the value to list if is_key_list(self, key) and not isinstance(obj, (list, tuple)): obj = [obj] @@ -821,10 +823,6 @@ class App(RasterInterface): if obj is None or (isinstance(obj, (list, tuple)) and not obj): self.app.ClearValue(key) return - if key not in self.parameters_keys: - raise KeyError( - f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}" - ) # Single-parameter cases if isinstance(obj, App): self.app.ConnectImage(key, obj.app, obj.key_output_image) -- GitLab From beb168dd000e505ed9a8eebbcc49388f9621def9 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 11:58:59 +0100 Subject: [PATCH 079/399] ENH: back to Output attr parent_pyotb_app to avoid confusion with bare OTB app --- pyotb/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index b9cc5a1..1f727ff 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1276,7 +1276,7 @@ class Output(RasterInterface): """ self.name = f"Output {param_key} from {pyotb_app.name}" - self.parent_app = pyotb_app # keep trace of parent app + self.parent_pyotb_app = pyotb_app # keep trace of parent app self.app = pyotb_app.app self.param_key = param_key self.filepath = None @@ -1307,8 +1307,8 @@ class Output(RasterInterface): def write(self, filepath: None | str | Path = None, **kwargs): """Write output to disk, filepath is not required if it was provided to parent App during init.""" if filepath is None and self.filepath: - return self.parent_app.write({self.key_output_image: self.filepath}, **kwargs) - return self.parent_app.write({self.key_output_image: filepath}, **kwargs) + return self.parent_pyotb_app.write({self.key_output_image: self.filepath}, **kwargs) + return self.parent_pyotb_app.write({self.key_output_image: filepath}, **kwargs) def __str__(self) -> str: """Return a nice string representation with source app name and object id.""" -- GitLab From c3aa56231b014f6bfcfeed32a7f9c31865426bee Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 14:25:15 +0100 Subject: [PATCH 080/399] ENH: search OTB on MacOS default app dir --- pyotb/helpers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index ebccc48..5d9267e 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -259,15 +259,15 @@ def __find_otb_root(scan_userdir: bool = False): logger.info("Found %s", path.parent) prefix = path.parent.absolute() elif sys.platform == "darwin": - # TODO: find OTB in macOS - pass - + for path in (Path.home() / "Applications").glob("**/OTB-*/lib"): + logger.info("Found %s", path.parent) + prefix = path.parent.absolute() # If possible, use OTB found in user's HOME tree (this may take some time) if scan_userdir: - for path in Path().home().glob("**/OTB-*/lib"): + for path in Path.home().glob("**/OTB-*/lib"): logger.info("Found %s", path.parent) prefix = path.parent.absolute() - + # Return latest found prefix (and version), see precedence in function def find_otb() return prefix -- GitLab From fbc0360dd8a47d62163f22f44bc21e779e0e9477 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 14:34:28 +0100 Subject: [PATCH 081/399] STYLE: auto format and save some space --- pyotb/core.py | 105 +++++++++++++++++++++-------------------------- pyotb/helpers.py | 13 +++--- 2 files changed, 54 insertions(+), 64 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 1f727ff..b50b7ff 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -16,6 +16,7 @@ from .helpers import logger class RasterInterface(ABC): """Abstraction of an image object.""" + app: otb.Application exports_dic: dict @@ -175,14 +176,13 @@ class RasterInterface(ABC): """ array = self.to_numpy(preserve_dtype=True, copy=False) - array = np.moveaxis(array, 2, 0) + height, width, count = array.shape proj = self.app.GetImageProjection(self.key_output_image) profile = { - 'crs': proj, 'dtype': array.dtype, - 'count': array.shape[0], 'height': array.shape[1], 'width': array.shape[2], - 'transform': self.transform + 'crs': proj, 'dtype': array.dtype, 'transform': self.transform, + 'count': count, 'height': height, 'width': width, } - return array, profile + return np.moveaxis(array, 2, 0), profile def xy_to_rowcol(self, x: float, y: float) -> tuple[int, int]: """Find (row, col) index using (x, y) projected coordinates - image CRS is expected. @@ -514,12 +514,14 @@ class App(RasterInterface): e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif' """ - self.parameters = {} self.name = name self.frozen = frozen self.quiet = quiet self.image_dic = image_dic + self.time_start, self.time_end = 0, 0 self.exports_dic = {} + self.parameters = {} + # Initialize app, set parameters and execute if not frozen create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication self.app = create(name) self.parameters_keys = tuple(self.app.GetParametersKeys()) @@ -546,12 +548,9 @@ class App(RasterInterface): @property def key_input(self) -> str: """Get the name of first input parameter, raster > vector > file.""" - return self.get_first_key(param_types=[otb.ParameterType_InputImage, - otb.ParameterType_InputImageList]) \ - or self.get_first_key(param_types=[otb.ParameterType_InputVectorData, - otb.ParameterType_InputVectorDataList]) \ - or self.get_first_key(param_types=[otb.ParameterType_InputFilename, - otb.ParameterType_InputFilenameList]) + return self.get_first_key([otb.ParameterType_InputImage, otb.ParameterType_InputImageList]) \ + or self.get_first_key([otb.ParameterType_InputVectorData, otb.ParameterType_InputVectorDataList]) \ + or self.get_first_key([otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList]) @property def key_input_image(self) -> str: @@ -576,14 +575,14 @@ class App(RasterInterface): @property def data(self) -> dict[str, float, list[float]]: """Expose app's output data values in a dictionary.""" - skip_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem") - skip_keys = skip_keys + tuple(self.out_param_types) + tuple(self.parameters) - keys = (k for k in self.parameters_keys if k not in skip_keys) - - def _check(v): - return not isinstance(v, otb.ApplicationProxy) and v not in ("", None, [], ()) - - return {str(k): self[k] for k in keys if _check(self[k])} + known_bad_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem") + skip_keys = known_bad_keys + tuple(self._out_param_types) + tuple(self.parameters) + data_dict = {} + for key in filter(lambda k: k not in skip_keys, self.parameters_keys): + value = self.__dict__.get(key) + if not isinstance(value, otb.ApplicationProxy) and value not in (None, "", [], ()): + data_dict[str(key)] = value + return data_dict def set_parameters(self, *args, **kwargs): """Set some parameters of the app. @@ -1053,8 +1052,7 @@ class Operation(App): else: nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))] # check that all inputs have the same nb of bands - if len(nb_bands_list) > 1: - if not all(x == nb_bands_list[0] for x in nb_bands_list): + if len(nb_bands_list) > 1 and not all(x == nb_bands_list[0] for x in nb_bands_list): raise ValueError("All images do not have the same number of bands") nb_bands = nb_bands_list[0] @@ -1067,10 +1065,7 @@ class Operation(App): # this is a special case for the condition of the ternary operator `cond ? x : y` if len(inputs) == 3 and k == 0: # When cond is monoband whereas the result is multiband, we expand the cond to multiband - if nb_bands != inp.shape[2]: - cond_band = 1 - else: - cond_band = band + cond_band = 1 if nb_bands != inp.shape[2] else band fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp(inp, cond_band, keep_logical=True) else: # Any other input @@ -1139,38 +1134,31 @@ class Operation(App): if isinstance(x, Slicer) and hasattr(x, "one_band_sliced"): if keep_logical and isinstance(x.input, LogicalOperation): fake_exp = x.input.logical_fake_exp_bands[x.one_band_sliced - 1] - inputs = x.input.inputs - nb_channels = x.input.nb_channels + inputs, nb_channels = x.input.inputs, x.input.nb_channels elif isinstance(x.input, Operation): # Keep only one band of the expression fake_exp = x.input.fake_exp_bands[x.one_band_sliced - 1] - inputs = x.input.inputs - nb_channels = x.input.nb_channels + inputs, nb_channels = x.input.inputs, x.input.nb_channels else: # Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1') fake_exp = f"{x.input}b{x.one_band_sliced}" - inputs = [x.input] - nb_channels = {x.input: 1} + inputs, nb_channels = [x.input], {x.input: 1} # For LogicalOperation, we save almost the same attributes as an Operation elif keep_logical and isinstance(x, LogicalOperation): fake_exp = x.logical_fake_exp_bands[band - 1] - inputs = x.inputs - nb_channels = x.nb_channels + inputs, nb_channels = x.inputs, x.nb_channels elif isinstance(x, Operation): fake_exp = x.fake_exp_bands[band - 1] - inputs = x.inputs - nb_channels = x.nb_channels + inputs, nb_channels = x.inputs, x.nb_channels # For int or float input, we just need to save their value elif isinstance(x, (int, float)): fake_exp = str(x) - inputs = None - nb_channels = None + inputs, nb_channels = None, None # We go on with other inputs, i.e. pyotb objects, filepaths... else: - nb_channels = {x: get_nbchannels(x)} - inputs = [x] # Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1') fake_exp = f"{x}b{band}" + inputs, nb_channels = [x], {x: get_nbchannels(x)} return fake_exp, inputs, nb_channels def __str__(self) -> str: @@ -1216,8 +1204,7 @@ class LogicalOperation(Operation): else: nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))] # check that all inputs have the same nb of bands - if len(nb_bands_list) > 1: - if not all(x == nb_bands_list[0] for x in nb_bands_list): + if len(nb_bands_list) > 1 and not all(x == nb_bands_list[0] for x in nb_bands_list): raise ValueError("All images do not have the same number of bands") nb_bands = nb_bands_list[0] @@ -1281,8 +1268,8 @@ class Output(RasterInterface): self.param_key = param_key self.filepath = None if filepath: - if '?' in filepath: - filepath = filepath.split('?')[0] + if "?" in filepath: + filepath = filepath.split("?")[0] self.filepath = Path(filepath) if mkdir: self.make_parent_dirs() @@ -1333,7 +1320,7 @@ def get_nbchannels(inp: str | App) -> int: info = App("ReadImageInfo", inp, quiet=True) nb_channels = info.app.GetParameterInt("numberbands") except Exception as e: # this happens when we pass a str that is not a filepath - raise TypeError(f'Could not get the number of channels of `{inp}`. Not a filepath or wrong filepath') from e + raise TypeError(f"Could not get the number of channels of '{inp}'. Not a filepath or wrong filepath") from e return nb_channels @@ -1357,23 +1344,23 @@ def get_pixel_type(inp: str | App) -> str: if not datatype: raise TypeError(f"Unable to read pixel type of image {inp}") datatype_to_pixeltype = { - 'unsigned_char': 'uint8', - 'short': 'int16', - 'unsigned_short': 'uint16', - 'int': 'int32', - 'unsigned_int': 'uint32', - 'long': 'int32', - 'ulong': 'uint32', - 'float': 'float', - 'double': 'double' + "unsigned_char": "uint8", + "short": "int16", + "unsigned_short": "uint16", + "int": "int32", + "unsigned_int": "uint32", + "long": "int32", + "ulong": "uint32", + "float": "float", + "double": "double", } if datatype not in datatype_to_pixeltype: raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}") - pixel_type = getattr(otb, f'ImagePixelType_{datatype_to_pixeltype[datatype]}') + pixel_type = getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[datatype]}") elif isinstance(inp, App): pixel_type = inp.app.GetParameterOutputImagePixelType(inp.key_output_image) else: - raise TypeError(f'Could not get the pixel type of {type(inp)} object {inp}') + raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}") return pixel_type @@ -1388,10 +1375,10 @@ def parse_pixel_type(pixel_type: str | int) -> int: """ if isinstance(pixel_type, str): # this correspond to 'uint8' etc... - return getattr(otb, f'ImagePixelType_{pixel_type}') + return getattr(otb, f"ImagePixelType_{pixel_type}") if isinstance(pixel_type, int): return pixel_type - raise ValueError(f'Bad pixel type specification ({pixel_type})') + raise ValueError(f"Bad pixel type specification ({pixel_type})") def is_key_list(pyotb_app: App, key: str) -> bool: @@ -1409,7 +1396,7 @@ def is_key_images_list(pyotb_app: App, key: str) -> bool: """Check if a key of the App is an input parameter image list.""" return pyotb_app.app.GetParameterType(key) in ( otb.ParameterType_InputImageList, - otb.ParameterType_InputFilenameList + otb.ParameterType_InputFilenameList, ) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 5d9267e..fee8d00 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""This module provides some helpers to properly initialize pyotb.""" +"""This module helps to ensure we properly initialize pyotb: only in case OTB is found and apps are available.""" import os import sys import logging @@ -41,8 +41,9 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru Path precedence : OTB_ROOT > python bindings directory OR search for releases installations : HOME - OR (for linux) : /opt/otbtf > /opt/otb > /usr/local > /usr - OR (for windows) : C:/Program Files + OR (for Linux) : /opt/otbtf > /opt/otb > /usr/local > /usr + OR (for MacOS) : ~/Applications + OR (for Windows) : C:/Program Files Args: prefix: prefix to search OTB in (Default value = OTB_ROOT) @@ -135,9 +136,9 @@ def set_environment(prefix: str): os.environ["OTB_APPLICATION_PATH"] = apps_path else: raise EnvironmentError("Can't find OTB applications directory") - os.environ["LC_NUMERIC"] = "C" os.environ["GDAL_DRIVER_PATH"] = "disable" + if (prefix / "share/gdal").exists(): # Local GDAL (OTB Superbuild, .run, .exe) gdal_data = str(prefix / "share/gdal") @@ -151,7 +152,6 @@ def set_environment(prefix: str): proj_lib = str(prefix / "share/proj") else: raise EnvironmentError(f"Can't find GDAL location with current OTB prefix '{prefix}' or in /usr") - os.environ["GDAL_DATA"] = gdal_data os.environ["PROJ_LIB"] = proj_lib @@ -303,3 +303,6 @@ def __suggest_fix_import(error_message: str, prefix: str): " first use 'call otbenv.bat' then try to import pyotb once again") docs_link = "https://www.orfeo-toolbox.org/CookBook/Installation.html" logger.critical("You can verify installation requirements for your OS at %s", docs_link) + +# Since helpers is the first module to be inititialized, this will prevent pyotb to run if OTB is not found +find_otb() -- GitLab From 2f0fb7e5b27ee165c5d5d365624ef181e299bca8 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 14:34:35 +0100 Subject: [PATCH 082/399] ENH: move find_otb call to helpers module --- pyotb/__init__.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pyotb/__init__.py b/pyotb/__init__.py index e3bc23a..ac6264c 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -2,12 +2,8 @@ """This module provides convenient python wrapping of otbApplications.""" __version__ = "1.6.0" -from .helpers import find_otb, logger, set_logger_level - -otb = find_otb() - +from .helpers import logger, set_logger_level from .apps import * - from .core import ( App, Input, @@ -15,7 +11,6 @@ from .core import ( get_nbchannels, get_pixel_type ) - from .functions import ( # pylint: disable=redefined-builtin all, any, -- GitLab From ea9e744555f36c72d077a4b5cf33ecb66d8a8ec5 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 14:35:08 +0100 Subject: [PATCH 083/399] ENH: make out_param_types and all_param_types private --- pyotb/core.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index b50b7ff..40e809b 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -525,22 +525,19 @@ class App(RasterInterface): create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication self.app = create(name) self.parameters_keys = tuple(self.app.GetParametersKeys()) - self.all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys} - self.out_param_types = {k: v for k, v in self.all_param_types.items() - if v in (otb.ParameterType_OutputImage, - otb.ParameterType_OutputVectorData, - otb.ParameterType_OutputFilename)} - self.time_start, self.time_end = 0, 0 + self._all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys} + otypes = (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename) + self._out_param_types = {k: v for k, v in self._all_param_types.items() if v in otypes} if args or kwargs: self.set_parameters(*args, **kwargs) if not self.frozen: self.execute() - if any(key in self.parameters for key in self.out_param_types): + if any(key in self.parameters for key in self._out_param_types): self.flush() # auto flush if any output param was provided during app init def get_first_key(self, param_types: list[int]) -> str: """Get the first output param key for specific file types.""" - for key, param_type in sorted(self.all_param_types.items()): + for key, param_type in sorted(self._all_param_types.items()): if param_type in param_types: return key return None @@ -570,7 +567,7 @@ class App(RasterInterface): @property def used_outputs(self) -> list[str]: """List of used application outputs.""" - return [getattr(self, key) for key in self.out_param_types if key in self.parameters] + return [getattr(self, key) for key in self._out_param_types if key in self.parameters] @property def data(self) -> dict[str, float, list[float]]: @@ -654,7 +651,7 @@ class App(RasterInterface): if target_key: keys = [target_key] else: - keys = [k for k, v in self.out_param_types.items() if v == otb.ParameterType_OutputImage] + keys = [k for k, v in self._out_param_types.items() if v == otb.ParameterType_OutputImage] for key in keys: self.app.SetParameterOutputImagePixelType(key, dtype) @@ -673,7 +670,7 @@ class App(RasterInterface): except RuntimeError: continue # this is when there is no value for key # Convert output param path to Output object - if key in self.out_param_types: + if key in self._out_param_types: value = Output(self, key, value) elif isinstance(value, str): try: @@ -741,7 +738,7 @@ class App(RasterInterface): if not filename_extension.startswith("?"): filename_extension = "?" + filename_extension for key, value in kwargs.items(): - if self.out_param_types[key] == otb.ParameterType_OutputImage and '?' not in value: + if self._out_param_types[key] == otb.ParameterType_OutputImage and "?" not in value: kwargs[key] = value + filename_extension # Manage output pixel types @@ -751,7 +748,7 @@ class App(RasterInterface): type_name = self.app.ConvertPixelTypeToNumpy(parse_pixel_type(pixel_type)) logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name) for key in kwargs: - if self.out_param_types[key] == otb.ParameterType_OutputImage: + if self._out_param_types[key] == otb.ParameterType_OutputImage: dtypes[key] = parse_pixel_type(pixel_type) elif isinstance(pixel_type, dict): dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()} -- GitLab From 9dd47e300e2aec377dc2f3ed0b2b4f34a4676d4d Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 14:35:58 +0100 Subject: [PATCH 084/399] ENH: ensure to raise AttributeError when unknown attribute is accessed --- pyotb/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 40e809b..4d6d8a5 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -872,7 +872,7 @@ class App(RasterInterface): """ # Accessing string attributes if isinstance(key, str): - return self.__dict__.get(key) + return getattr(self, key) # Accessing pixel value(s) using Y/X coordinates if isinstance(key, tuple) and len(key) >= 2: row, col = key[0], key[1] @@ -1050,7 +1050,7 @@ class Operation(App): nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))] # check that all inputs have the same nb of bands if len(nb_bands_list) > 1 and not all(x == nb_bands_list[0] for x in nb_bands_list): - raise ValueError("All images do not have the same number of bands") + raise ValueError("All images do not have the same number of bands") nb_bands = nb_bands_list[0] # Create a list of fake expressions, each item of the list corresponding to one band @@ -1202,7 +1202,7 @@ class LogicalOperation(Operation): nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))] # check that all inputs have the same nb of bands if len(nb_bands_list) > 1 and not all(x == nb_bands_list[0] for x in nb_bands_list): - raise ValueError("All images do not have the same number of bands") + raise ValueError("All images do not have the same number of bands") nb_bands = nb_bands_list[0] # Create a list of fake exp, each item of the list corresponding to one band -- GitLab From e8f376f796524c175c70963a2a1274253ef66f64 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 14:37:19 +0100 Subject: [PATCH 085/399] STYLE: remove useless pylint exceptions + linting --- pyotb/helpers.py | 1 + pyproject.toml | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index fee8d00..69aac02 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -304,5 +304,6 @@ def __suggest_fix_import(error_message: str, prefix: str): docs_link = "https://www.orfeo-toolbox.org/CookBook/Installation.html" logger.critical("You can verify installation requirements for your OS at %s", docs_link) + # Since helpers is the first module to be inititialized, this will prevent pyotb to run if OTB is not found find_otb() diff --git a/pyproject.toml b/pyproject.toml index 9538fc1..10c15b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,9 +48,7 @@ disable = [ "too-many-locals", "too-many-branches", "too-many-statements", - "too-many-public-methods", - "too-many-instance-attributes", - "wrong-import-position", + "too-many-instance-attributes" ] [tool.pydocstyle] -- GitLab From cd750a192c8ed1f1fc58ad460a111b4072a2ce32 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 15:30:38 +0100 Subject: [PATCH 086/399] STYLE: better pylint config + save some space --- pyotb/apps.py | 4 ++-- pyotb/core.py | 32 +++++++++++++------------- pyotb/functions.py | 56 +++++++++++++--------------------------------- pyotb/helpers.py | 4 ++-- pyproject.toml | 4 ++-- 5 files changed, 37 insertions(+), 63 deletions(-) diff --git a/pyotb/apps.py b/pyotb/apps.py index 9421b28..bad5089 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -36,9 +36,9 @@ def get_available_applications(as_subprocess: bool = False) -> list[str]: cmd_args = [sys.executable, "-c", pycmd] try: params = {"env": env, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE} - with subprocess.Popen(cmd_args, **params) as p: + with subprocess.Popen(cmd_args, **params) as process: logger.debug('Exec "%s \'%s\'"', ' '.join(cmd_args[:-1]), pycmd) - stdout, stderr = p.communicate() + stdout, stderr = process.communicate() stdout, stderr = stdout.decode(), stderr.decode() # ast.literal_eval is secure and will raise more handy Exceptions than eval from ast import literal_eval # pylint: disable=import-outside-toplevel diff --git a/pyotb/core.py b/pyotb/core.py index 4d6d8a5..b488a38 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -526,8 +526,8 @@ class App(RasterInterface): self.app = create(name) self.parameters_keys = tuple(self.app.GetParametersKeys()) self._all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys} - otypes = (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename) - self._out_param_types = {k: v for k, v in self._all_param_types.items() if v in otypes} + types = (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename) + self._out_param_types = {k: v for k, v in self._all_param_types.items() if v in types} if args or kwargs: self.set_parameters(*args, **kwargs) if not self.frozen: @@ -786,14 +786,14 @@ class App(RasterInterface): nested dictionary summarizing the pipeline """ - params = self.parameters - for k, p in params.items(): + parameters = self.parameters.copy() + for key, param in parameters.items(): # In the following, we replace each parameter which is an App, with its summary. - if isinstance(p, App): # single parameter - params[k] = p.summarize() - elif isinstance(p, list): # parameter list - params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p] - return {"name": self.app.GetName(), "parameters": params} + if isinstance(param, App): # single parameter + parameters[key] = param.summarize() + elif isinstance(param, list): # parameter list + parameters[key] = [p.summarize() if isinstance(p, App) else p for p in param] + return {"name": self.app.GetName(), "parameters": parameters} # Private functions def __parse_args(self, args: list[str | App | dict | list]) -> dict[str, Any]: @@ -1104,8 +1104,7 @@ class Operation(App): one_band_exp = one_band_exp.replace(str(inp), self.im_dic[str(inp)]) exp_bands.append(one_band_exp) # Form the final expression (e.g. 'im1b1 + 1; im1b2 + 1') - exp = ";".join(exp_bands) - return exp_bands, exp + return exp_bands, ";".join(exp_bands) @staticmethod def make_fake_exp(x: App | str, band: int, keep_logical: bool = False) -> tuple[str, list[App], int]: @@ -1156,6 +1155,7 @@ class Operation(App): # Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1') fake_exp = f"{x}b{band}" inputs, nb_channels = [x], {x: get_nbchannels(x)} + return fake_exp, inputs, nb_channels def __str__(self) -> str: @@ -1204,7 +1204,6 @@ class LogicalOperation(Operation): if len(nb_bands_list) > 1 and not all(x == nb_bands_list[0] for x in nb_bands_list): raise ValueError("All images do not have the same number of bands") nb_bands = nb_bands_list[0] - # Create a list of fake exp, each item of the list corresponding to one band for i, band in enumerate(range(1, nb_bands + 1)): expressions = [] @@ -1380,21 +1379,20 @@ def parse_pixel_type(pixel_type: str | int) -> int: def is_key_list(pyotb_app: App, key: str) -> bool: """Check if a key of the App is an input parameter list.""" - return pyotb_app.app.GetParameterType(key) in ( + types = ( otb.ParameterType_InputImageList, otb.ParameterType_StringList, otb.ParameterType_InputFilenameList, otb.ParameterType_ListView, otb.ParameterType_InputVectorDataList, ) + return pyotb_app.app.GetParameterType(key) in types def is_key_images_list(pyotb_app: App, key: str) -> bool: """Check if a key of the App is an input parameter image list.""" - return pyotb_app.app.GetParameterType(key) in ( - otb.ParameterType_InputImageList, - otb.ParameterType_InputFilenameList, - ) + types = (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList) + return pyotb_app.app.GetParameterType(key) in types def get_out_images_param_keys(app: App) -> list[str]: diff --git a/pyotb/functions.py b/pyotb/functions.py index 250f79e..d782a0e 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -34,7 +34,6 @@ def where(cond: App | str, x: App | str | int | float, y: App | str | int | floa x_nb_channels = get_nbchannels(x) if not isinstance(y, (int, float)): y_nb_channels = get_nbchannels(y) - if x_nb_channels and y_nb_channels: if x_nb_channels != y_nb_channels: raise ValueError('X and Y images do not have the same number of bands. ' @@ -42,44 +41,38 @@ def where(cond: App | str, x: App | str | int | float, y: App | str | int | floa x_or_y_nb_channels = x_nb_channels if x_nb_channels else y_nb_channels cond_nb_channels = get_nbchannels(cond) - - # Get the number of bands of the result - if x_or_y_nb_channels: # if X or Y is a raster - out_nb_channels = x_or_y_nb_channels - else: # if only cond is a raster - out_nb_channels = cond_nb_channels - if cond_nb_channels != 1 and x_or_y_nb_channels and cond_nb_channels != x_or_y_nb_channels: raise ValueError('Condition and X&Y do not have the same number of bands. Condition has ' f'{cond_nb_channels} bands whereas X&Y have {x_or_y_nb_channels} bands') - # If needed, duplicate the single band binary mask to multiband to match the dimensions of x & y if cond_nb_channels == 1 and x_or_y_nb_channels and x_or_y_nb_channels != 1: logger.info('The condition has one channel whereas X/Y has/have %s channels. Expanding number' ' of channels of condition to match the number of channels of X/Y', x_or_y_nb_channels) - operation = Operation('?', cond, x, y, nb_bands=out_nb_channels) + # Get the number of bands of the result + if x_or_y_nb_channels: # if X or Y is a raster + out_nb_channels = x_or_y_nb_channels + else: # if only cond is a raster + out_nb_channels = cond_nb_channels - return operation + return Operation('?', cond, x, y, nb_bands=out_nb_channels) -def clip(a: App | str, a_min: App | str | int | float, a_max: App | str | int | float): +def clip(image: App | str, v_min: App | str | int | float, v_max: App | str | int | float): """Clip values of image in a range of values. Args: - a: input raster, can be filepath or any pyotb object - a_min: minimum value of the range - a_max: maximum value of the range + image: input raster, can be filepath or any pyotb object + v_min: minimum value of the range + v_max: maximum value of the range Returns: raster whose values are clipped in the range """ - if isinstance(a, str): - a = Input(a) - - res = where(a <= a_min, a_min, - where(a >= a_max, a_max, a)) + if isinstance(image, str): + image = Input(image) + res = where(image <= v_min, v_min, where(image >= v_max, v_max, image)) return res @@ -102,11 +95,9 @@ def all(*inputs): # pylint: disable=redefined-builtin # If necessary, flatten inputs if len(inputs) == 1 and isinstance(inputs[0], (list, tuple)): inputs = inputs[0] - # Add support for generator inputs (to have the same behavior as built-in `all` function) if isinstance(inputs, tuple) and len(inputs) == 1 and inspect.isgenerator(inputs[0]): inputs = list(inputs[0]) - # Transforming potential filepaths to pyotb objects inputs = [Input(inp) if isinstance(inp, str) else inp for inp in inputs] @@ -117,13 +108,11 @@ def all(*inputs): # pylint: disable=redefined-builtin res = inp[:, :, 0] else: res = (inp[:, :, 0] != 0) - for band in range(1, inp.shape[-1]): if isinstance(inp, LogicalOperation): res = res & inp[:, :, band] else: res = res & (inp[:, :, band] != 0) - # Checking that all images are True else: if isinstance(inputs[0], LogicalOperation): @@ -157,11 +146,9 @@ def any(*inputs): # pylint: disable=redefined-builtin # If necessary, flatten inputs if len(inputs) == 1 and isinstance(inputs[0], (list, tuple)): inputs = inputs[0] - # Add support for generator inputs (to have the same behavior as built-in `any` function) if isinstance(inputs, tuple) and len(inputs) == 1 and inspect.isgenerator(inputs[0]): inputs = list(inputs[0]) - # Transforming potential filepaths to pyotb objects inputs = [Input(inp) if isinstance(inp, str) else inp for inp in inputs] @@ -240,7 +227,6 @@ def run_tf_function(func): func_name = func.__name__ create_and_save_model_str = func_def_str - # Adding the instructions to create the model and save it to output dir create_and_save_model_str += textwrap.dedent(f""" import tensorflow as tf @@ -348,7 +334,6 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ inputs.extend(arg) else: inputs.append(arg) - # Getting metadatas of inputs metadatas = {} for inp in inputs: @@ -362,7 +347,6 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ # Get a metadata of an arbitrary image. This is just to compare later with other images any_metadata = next(iter(metadatas.values())) - # Checking if all images have the same projection if not all(metadata['ProjectionRef'] == any_metadata['ProjectionRef'] for metadata in metadatas.values()): @@ -403,9 +387,8 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ # TODO : it is when the user wants the final bounding box to be the union of all bounding box # It should replace any 'outside' pixel by some NoData -> add `fillvalue` argument in the function - logger.info('Cropping all images to extent Upper Left (%s, %s), Lower Right (%s, %s)', ulx, uly, lrx, lry) - # Applying this bounding box to all inputs + logger.info('Cropping all images to extent Upper Left (%s, %s), Lower Right (%s, %s)', ulx, uly, lrx, lry) new_inputs = [] for inp in inputs: try: @@ -425,13 +408,11 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ logger.error('Cannot define the processing area for input %s: %s', inp, e) raise inputs = new_inputs - # Update metadatas metadatas = {input: input.app.GetImageMetaData('out') for input in inputs} # Get a metadata of an arbitrary image. This is just to compare later with other images any_metadata = next(iter(metadatas.values())) - # Handling different pixel sizes if not all(metadata['GeoTransform'][1] == any_metadata['GeoTransform'][1] and metadata['GeoTransform'][5] == any_metadata['GeoTransform'][5] @@ -449,9 +430,9 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ pass # TODO : when the user explicitly specify the pixel size -> add argument inside the function pixel_size = metadatas[reference_input]['GeoTransform'][1] - logger.info('Resampling all inputs to resolution: %s', pixel_size) # Perform resampling on inputs that do not comply with the target pixel size + logger.info('Resampling all inputs to resolution: %s', pixel_size) new_inputs = [] for inp in inputs: if metadatas[inp]['GeoTransform'][1] != pixel_size: @@ -460,18 +441,14 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ else: new_inputs.append(inp) inputs = new_inputs - - # Update metadatas metadatas = {inp: inp.app.GetImageMetaData('out') for inp in inputs} # Final superimposition to be sure to have the exact same image sizes - # Getting the sizes of images image_sizes = {} for inp in inputs: if isinstance(inp, str): inp = Input(inp) image_sizes[inp] = inp.shape[:2] - # Selecting the most frequent image size. It will be used as reference. most_common_image_size, _ = Counter(image_sizes.values()).most_common(1)[0] same_size_images = [inp for inp, image_size in image_sizes.items() if image_size == most_common_image_size] @@ -484,6 +461,5 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ new_inputs.append(superimposed) else: new_inputs.append(inp) - inputs = new_inputs - return inputs + return new_inputs diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 69aac02..03200a2 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -77,9 +77,9 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru otb.Registry.SetApplicationPath(apps_path) return otb except ImportError as e: - PYTHONPATH = os.environ.get("PYTHONPATH") + pythonpath = os.environ.get("PYTHONPATH") if not scan: - raise SystemExit(f"Failed to import OTB with env PYTHONPATH={PYTHONPATH}") from e + raise SystemExit(f"Failed to import OTB with env PYTHONPATH={pythonpath}") from e # Else search system logger.info("Failed to import OTB. Searching for it...") prefix = __find_otb_root(scan_userdir) diff --git a/pyproject.toml b/pyproject.toml index 10c15b9..845e218 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,10 +41,10 @@ version = {attr = "pyotb.__version__"} [tool.pylint] max-line-length = 120 +max-module-lines = 2000 +good-names = ["x", "y", "i", "j", "k", "e"] disable = [ "fixme", - "invalid-name", - "too-many-lines", "too-many-locals", "too-many-branches", "too-many-statements", -- GitLab From a9217fc79e5fcb55bbe17706fb037744881a309a Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 15:34:26 +0100 Subject: [PATCH 087/399] ENH: make time_start and time_end private --- pyotb/core.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index b488a38..180562b 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -518,7 +518,7 @@ class App(RasterInterface): self.frozen = frozen self.quiet = quiet self.image_dic = image_dic - self.time_start, self.time_end = 0, 0 + self._time_start, self._time_end = 0, 0 self.exports_dic = {} self.parameters = {} # Initialize app, set parameters and execute if not frozen @@ -562,7 +562,7 @@ class App(RasterInterface): @property def elapsed_time(self) -> float: """Get elapsed time between app init and end of exec or file writing.""" - return self.time_end - self.time_start + return self._time_end - self._time_start @property def used_outputs(self) -> list[str]: @@ -683,13 +683,13 @@ class App(RasterInterface): def execute(self): """Execute and write to disk if any output parameter has been set during init.""" logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters) - self.time_start = perf_counter() + self._time_start = perf_counter() try: self.app.Execute() except (RuntimeError, FileNotFoundError) as e: raise Exception(f"{self.name}: error during during app execution") from e self.frozen = False - self.time_end = perf_counter() + self._time_end = perf_counter() logger.debug("%s: execution ended", self.name) self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics @@ -701,7 +701,7 @@ class App(RasterInterface): except RuntimeError: logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name) self.app.ExecuteAndWriteOutput() - self.time_end = perf_counter() + self._time_end = perf_counter() def write(self, *args, filename_extension: str = "", pixel_type: dict[str, str] | str = None, preserve_dtype: bool = False, **kwargs): -- GitLab From 86b8cbf9637f35fac6dd3dc4e5880bd244fcd8b0 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 21:16:53 +0100 Subject: [PATCH 088/399] ENH: Input use Path and filepath attr as in Output --- pyotb/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 180562b..e881c48 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1234,15 +1234,15 @@ class Input(App): path: Anything supported by GDAL (local file on the filesystem, remote resource e.g. /vsicurl/.., etc.) """ - self.path = path - self.name = f"Input from {path}" super().__init__("ExtractROI", {"in": path}, frozen=True) + self.name = f"Input from {path}" + self.filepath = Path(path) self.propagate_dtype() self.execute() def __str__(self) -> str: """Return a nice string representation with file path.""" - return f"<pyotb.Input object from {self.path}>" + return f"<pyotb.Input object from {self.filepath}>" class Output(RasterInterface): -- GitLab From edefdc1f14889c38b764a4fbf4b411f3f3987594 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 26 Jan 2023 21:24:48 +0100 Subject: [PATCH 089/399] FIX: missing exports_dic in Output, add tests --- pyotb/core.py | 1 + tests/test_core.py | 7 ++++++- tests/test_numpy.py | 9 +++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index e881c48..2f7a50c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1261,6 +1261,7 @@ class Output(RasterInterface): self.name = f"Output {param_key} from {pyotb_app.name}" self.parent_pyotb_app = pyotb_app # keep trace of parent app self.app = pyotb_app.app + self.exports_dic = pyotb_app.exports_dic self.param_key = param_key self.filepath = None if filepath: diff --git a/tests/test_core.py b/tests/test_core.py index ccd386f..848fe15 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,4 +1,3 @@ -import os import pytest import pyotb @@ -80,6 +79,12 @@ def test_write(): INPUT.out.filepath.unlink() +def test_output_write(): + INPUT.out.write("/tmp/test_output_write.tif") + assert INPUT.out.exists() + INPUT.out.filepath.unlink() + + # Slicer def test_slicer_shape(): extract = INPUT[:50, :60, :3] diff --git a/tests/test_numpy.py b/tests/test_numpy.py index 682fe2a..d36dfb7 100644 --- a/tests/test_numpy.py +++ b/tests/test_numpy.py @@ -5,10 +5,15 @@ from tests_data import INPUT def test_export(): INPUT.export() - assert "out" in INPUT.exports_dic - array = INPUT.exports_dic["out"]["array"] + array = INPUT.exports_dic[INPUT.key_output_image]["array"] assert isinstance(array, np.ndarray) assert array.dtype == "uint8" + del INPUT.exports_dic["out"] + + +def test_output_export(): + INPUT.out.export() + assert INPUT.out.key_output_image in INPUT.out.exports_dic def test_to_numpy(): -- GitLab From 35832c6d74acde9f088294c831db57af7131334e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr> Date: Thu, 2 Feb 2023 09:08:18 +0000 Subject: [PATCH 090/399] WIP: refac classes --- pyotb/apps.py | 37 +- pyotb/core.py | 920 +++++++++++++++++++----------------- pyotb/functions.py | 22 +- tests/test_core.py | 11 +- tests/test_numpy.py | 10 +- tests/test_pipeline.py | 5 +- tests/test_serialization.py | 14 +- tests/tests_data.py | 3 + 8 files changed, 505 insertions(+), 517 deletions(-) create mode 100644 tests/tests_data.py diff --git a/pyotb/apps.py b/pyotb/apps.py index ad71f54..9421b28 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -7,7 +7,7 @@ import subprocess from pathlib import Path import otbApplication as otb # pylint: disable=import-error -from .core import OTBObject +from .core import App from .helpers import logger @@ -59,41 +59,6 @@ def get_available_applications(as_subprocess: bool = False) -> list[str]: return app_list -class App(OTBObject): - """Base class for UI related functions, will be subclassed using app name as class name, see CODE_TEMPLATE.""" - _name = "" - - def __init__(self, *args, **kwargs): - """Default App constructor, adds UI specific attributes and functions.""" - super().__init__(*args, **kwargs) - self.description = self.app.GetDocLongDescription() - - @property - def elapsed_time(self): - """Get elapsed time between app init and end of exec or file writing.""" - return self.time_end - self.time_start - - @property - def used_outputs(self) -> list[str]: - """List of used application outputs.""" - return [getattr(self, key) for key in self.out_param_types if key in self.parameters] - - def find_outputs(self) -> tuple[str]: - """Find output files on disk using path found in parameters. - - Returns: - list of files found on disk - - """ - files, missing = [], [] - for out in self.used_outputs: - dest = files if out.exists() else missing - dest.append(str(out.filepath.absolute())) - for filename in missing: - logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) - return tuple(files) - - class OTBTFApp(App): """Helper for OTBTF.""" @staticmethod diff --git a/pyotb/core.py b/pyotb/core.py index 6f0a4dc..fb07a7b 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- """This module is the core of pyotb.""" from __future__ import annotations -from typing import Any -from pathlib import Path + from ast import literal_eval +from pathlib import Path from time import perf_counter +from typing import Any import numpy as np import otbApplication as otb # pylint: disable=import-error @@ -13,50 +14,32 @@ from .helpers import logger class OTBObject: - """Base class that gathers common operations for any OTB application.""" + """Base class for all pyotb objects.""" - def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, **kwargs): - """Common constructor for OTB applications. Handles in-memory connection between apps. + def __init__(self, name: str, app: otb.Application, image_dic: dict = None): + """Constructor for an OTBObject. Args: - name: name of the app, e.g. 'BandMath' - *args: used for passing application parameters. Can be : - - dictionary containing key-arguments enumeration. Useful when a key is python-reserved - (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") - - string, App or Output, useful when the user wants to specify the input "in" - - list, useful when the user wants to specify the input list 'il' - frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___ - quiet: whether to print logs of the OTB app + name: name of the object (e.g. "Slicer") + app: OTB application instance image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as - the result of app.ExportImage(). Use it when the app takes a numpy array as input. + the result of self.app.ExportImage(). Use it when the app takes a numpy array as input. See this related issue for why it is necessary to keep reference of object: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824 - **kwargs: used for passing application parameters. - e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif' - """ - self.parameters = {} self.name = name - self.frozen = frozen - self.quiet = quiet + self.app = app self.image_dic = image_dic - self.exports_dic = {} - create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication - self.app = create(name) + self.parameters_keys = tuple(self.app.GetParametersKeys()) - self.time_start, self.time_end = 0, 0 self.all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys} self.out_param_types = {k: v for k, v in self.all_param_types.items() if v in (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename)} - if args or kwargs: - self.set_parameters(*args, **kwargs) - if not self.frozen: - self.execute() - if any(key in self.parameters for key in self.out_param_types): - self.flush() # auto flush if any output param was provided during app init + + self.exports_dic = {} def get_first_key(self, param_types: list[str]) -> str: """Get the first output param key for specific file types.""" @@ -85,17 +68,6 @@ class OTBObject: """Get the name of first output image parameter.""" return self.get_first_key(param_types=[otb.ParameterType_OutputImage]) - @property - def data(self): - """Expose app's output data values in a dictionary.""" - skip_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem") - skip_keys = skip_keys + tuple(self.out_param_types) + tuple(self.parameters) - keys = (k for k in self.parameters_keys if k not in skip_keys) - - def _check(v): - return not isinstance(v, otb.ApplicationProxy) and v not in ("", None, [], ()) - return {str(k): self[k] for k in keys if _check(self[k])} - @property def metadata(self): """Return first output image metadata dictionary.""" @@ -125,7 +97,7 @@ class OTBObject: """ if not self.key_output_image: - raise TypeError(f"{self.name}: this application has no raster output") + raise TypeError(f"\"{self.name}\" has no raster output") width, height = self.app.GetImageSize(self.key_output_image) bands = self.app.GetImageNbBands(self.key_output_image) return height, width, bands @@ -145,196 +117,17 @@ class OTBObject: origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2 return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y - def set_parameters(self, *args, **kwargs): - """Set some parameters of the app. - - When useful, e.g. for images list, this function appends the parameters - instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths - - Args: - *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved - (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") - - string, App or Output, useful when the user implicitly wants to set the param "in" - - list, useful when the user implicitly wants to set the param "il" - **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif' - - Raises: - Exception: when the setting of a parameter failed - - """ - parameters = kwargs - parameters.update(self.__parse_args(args)) - # Going through all arguments - for key, obj in parameters.items(): - if key not in self.parameters_keys: - raise KeyError(f'{self.name}: unknown parameter name "{key}"') - # When the parameter expects a list, if needed, change the value to list - if is_key_list(self, key) and not isinstance(obj, (list, tuple)): - obj = [obj] - logger.info('%s: argument for parameter "%s" was converted to list', self.name, key) - try: - # This is when we actually call self.app.SetParameter* - self.__set_param(key, obj) - except (RuntimeError, TypeError, ValueError, KeyError) as e: - raise Exception( - f"{self.name}: something went wrong before execution " - f"(while setting parameter '{key}' to '{obj}')" - ) from e - # Update _parameters using values from OtbApplication object - otb_params = self.app.GetParameters().items() - otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params} - # Update param dict and save values as object attributes - self.parameters.update({**parameters, **otb_params}) - self.save_objects() - - def save_objects(self): - """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`. - - This is useful when the key contains reserved characters such as a point eg "io.out" - """ - for key in self.parameters_keys: - if key in dir(self.__class__): - continue # skip forbidden attribute since it is already used by the class - value = self.parameters.get(key) # basic parameters - if value is None: - try: - value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results) - except RuntimeError: - continue # this is when there is no value for key - # Convert output param path to Output object - if key in self.out_param_types: - value = Output(self, key, value) - elif isinstance(value, str): - try: - value = literal_eval(value) - except (ValueError, SyntaxError): - pass - # Save attribute - setattr(self, key, value) - - def execute(self): - """Execute and write to disk if any output parameter has been set during init.""" - logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters) - self.time_start = perf_counter() - try: - self.app.Execute() - except (RuntimeError, FileNotFoundError) as e: - raise Exception(f"{self.name}: error during during app execution") from e - self.frozen = False - self.time_end = perf_counter() - logger.debug("%s: execution ended", self.name) - self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics - - def flush(self): - """Flush data to disk, this is when WriteOutput is actually called.""" - try: - logger.debug("%s: flushing data to disk", self.name) - self.app.WriteOutput() - except RuntimeError: - logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name) - self.app.ExecuteAndWriteOutput() - self.time_end = perf_counter() - - def write(self, *args, filename_extension: str = "", pixel_type: dict[str, str] | str = None, - preserve_dtype: bool = False, **kwargs): - """Set output pixel type and write the output raster files. - - Args: - *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key contains - non-standard characters such as a point, e.g. {'io.out':'output.tif'} - - string, useful when there is only one output, e.g. 'output.tif' - - None if output file was passed during App init - filename_extension: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES") - Will be used for all outputs (Default value = "") - pixel_type: Can be : - dictionary {output_parameter_key: pixeltype} when specifying for several outputs - - str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several - outputs, all outputs are written with this unique type. - Valid pixel types are uint8, uint16, uint32, int16, int32, float, double, - cint16, cint32, cfloat, cdouble. (Default value = None) - preserve_dtype: propagate main input pixel type to outputs, in case pixel_type is None - **kwargs: keyword arguments e.g. out='output.tif' - - """ - # Gather all input arguments in kwargs dict - for arg in args: - if isinstance(arg, dict): - kwargs.update(arg) - elif isinstance(arg, str) and kwargs: - logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg) - elif isinstance(arg, str) and self.key_output_image: - kwargs.update({self.key_output_image: arg}) - - # Append filename extension to filenames - if filename_extension: - logger.debug("%s: using extended filename for outputs: %s", self.name, filename_extension) - if not filename_extension.startswith("?"): - filename_extension = "?" + filename_extension - for key, value in kwargs.items(): - if self.out_param_types[key] == otb.ParameterType_OutputImage and '?' not in value: - kwargs[key] = value + filename_extension - - # Manage output pixel types - dtypes = {} - if pixel_type: - if isinstance(pixel_type, str): - type_name = self.app.ConvertPixelTypeToNumpy(parse_pixel_type(pixel_type)) - logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name) - for key in kwargs: - if self.out_param_types[key] == otb.ParameterType_OutputImage: - dtypes[key] = parse_pixel_type(pixel_type) - elif isinstance(pixel_type, dict): - dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()} - elif preserve_dtype: - self.propagate_dtype() # all outputs will have the same type as the main input raster - - # Set parameters and flush to disk - for key, output_filename in kwargs.items(): - if key in dtypes: - self.propagate_dtype(key, dtypes[key]) - self.set_parameters({key: output_filename}) - self.flush() - - def propagate_dtype(self, target_key: str = None, dtype: int = None): - """Propagate a pixel type from main input to every outputs, or to a target output key only. - - With multiple inputs (if dtype is not provided), the type of the first input is considered. - With multiple outputs (if target_key is not provided), all outputs will be converted to the same pixel type. - - Args: - target_key: output param key to change pixel type - dtype: data type to use - - """ - if not dtype: - param = self.parameters.get(self.key_input_image) - if not param: - logger.warning("%s: could not propagate pixel type from inputs to output", self.name) - return - if isinstance(param, (list, tuple)): - param = param[0] # first image in "il" - try: - dtype = get_pixel_type(param) - except (TypeError, RuntimeError): - logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param) - return - if target_key: - keys = [target_key] - else: - keys = [k for k, v in self.out_param_types.items() if v == otb.ParameterType_OutputImage] - for key in keys: - self.app.SetParameterOutputImagePixelType(key, dtype) - def get_infos(self): """Return a dict output of ReadImageInfo for the first image output.""" if not self.key_output_image: raise TypeError(f"{self.name}: this application has no raster output") - return OTBObject("ReadImageInfo", self, quiet=True).data + return App("ReadImageInfo", self, quiet=True).data def get_statistics(self): """Return a dict output of ComputeImagesStatistics for the first image output.""" if not self.key_output_image: raise TypeError(f"{self.name}: this application has no raster output") - return OTBObject("ComputeImagesStatistics", self, quiet=True).data + return App("ComputeImagesStatistics", self, quiet=True).data def read_values_at_coords(self, row: int, col: int, bands: int = None) -> list[int | float] | int | float: """Get pixel value(s) at a given YX coordinates. @@ -349,7 +142,7 @@ class OTBObject: """ channels = [] - app = OTBObject("PixelValue", self, coordx=col, coordy=row, frozen=True, quiet=True) + app = App("PixelValue", self, coordx=col, coordy=row, frozen=True, quiet=True) if bands is not None: if isinstance(bands, int): if bands < 0: @@ -385,22 +178,6 @@ class OTBObject: return list(range(0, nb_channels, step)) raise ValueError(f"{self.name}: '{bands}' cannot be interpreted as valid slicing.") - def summarize(self) -> dict: - """Serialize an object and its pipeline into a dictionary. - - Returns: - nested dictionary summarizing the pipeline - - """ - params = self.parameters - for k, p in params.items(): - # In the following, we replace each parameter which is an OTBObject, with its summary. - if isinstance(p, OTBObject): # single parameter - params[k] = p.summarize() - elif isinstance(p, list): # parameter list - params[k] = [pi.summarize() if isinstance(pi, OTBObject) else pi for pi in p] - return {"name": self.name, "parameters": params} - def export(self, key: str = None, preserve_dtype: bool = True) -> dict[str, dict[str, np.ndarray]]: """Export a specific output image as numpy array and store it in object exports_dic. @@ -473,58 +250,6 @@ class OTBObject: row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x return abs(int(row)), int(col) - # Private functions - def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]: - """Gather all input arguments in kwargs dict. - - Args: - args: the list of arguments passed to set_parameters() - - Returns: - a dictionary with the right keyword depending on the object - - """ - kwargs = {} - for arg in args: - if isinstance(arg, dict): - kwargs.update(arg) - elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.key_input): - kwargs.update({self.key_input: arg}) - return kwargs - - def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]): - """Set one parameter, decide which otb.Application method to use depending on target object.""" - if obj is None or (isinstance(obj, (list, tuple)) and not obj): - self.app.ClearValue(key) - return - if key not in self.parameters_keys: - raise KeyError( - f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}" - ) - # Single-parameter cases - if isinstance(obj, OTBObject): - self.app.ConnectImage(key, obj.app, obj.key_output_image) - elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB - self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0]) - elif key == "ram": # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200 - self.app.SetParameterInt("ram", int(obj)) - elif not isinstance(obj, list): # any other parameters (str, int...) - self.app.SetParameterValue(key, obj) - # Images list - elif is_key_images_list(self, key): - # To enable possible in-memory connections, we go through the list and set the parameters one by one - for inp in obj: - if isinstance(inp, OTBObject): - self.app.ConnectImage(key, inp.app, inp.key_output_image) - elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB - self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0]) - else: # here `input` should be an image filepath - # Append `input` to the list, do not overwrite any previously set element of the image list - self.app.AddParameterStringList(key, inp) - # List of any other types (str, int...) - else: - self.app.SetParameterValue(key, obj) - # Special functions def __hash__(self): """Override the default behaviour of the hash function. @@ -535,31 +260,6 @@ class OTBObject: """ return id(self) - def __str__(self): - """Return a nice string representation with object id.""" - return f"<pyotb.App {self.name} object id {id(self)}>" - - def __getattr__(self, name): - """This method is called when the default attribute access fails. - - We choose to access the attribute `name` of self.app. - Thus, any method of otbApplication can be used transparently on OTBObject objects, - e.g. SetParameterOutputImagePixelType() or ExportImage() work - - Args: - name: attribute name - - Returns: - attribute - - Raises: - AttributeError: when `name` is not an attribute of self.app - - """ - if name in dir(self.app): - return getattr(self.app, name) - raise AttributeError(f"{self.name}: could not find attribute `{name}`") - def __getitem__(self, key): """Override the default __getitem__ behaviour. @@ -598,7 +298,7 @@ class OTBObject: key = key + (slice(None, None, None),) return Slicer(self, *key) - def __add__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __add__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default addition and flavours it with BandMathX. Args: @@ -612,7 +312,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("+", self, other) - def __sub__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __sub__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default subtraction and flavours it with BandMathX. Args: @@ -626,7 +326,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("-", self, other) - def __mul__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __mul__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default subtraction and flavours it with BandMathX. Args: @@ -640,7 +340,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("*", self, other) - def __truediv__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __truediv__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default subtraction and flavours it with BandMathX. Args: @@ -654,7 +354,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("/", self, other) - def __radd__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __radd__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default reverse addition and flavours it with BandMathX. Args: @@ -668,7 +368,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("+", other, self) - def __rsub__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __rsub__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default subtraction and flavours it with BandMathX. Args: @@ -682,7 +382,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("-", other, self) - def __rmul__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __rmul__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default multiplication and flavours it with BandMathX. Args: @@ -696,7 +396,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return Operation("*", other, self) - def __rtruediv__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __rtruediv__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default division and flavours it with BandMathX. Args: @@ -719,7 +419,7 @@ class OTBObject: """ return Operation("abs", self) - def __ge__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __ge__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default greater or equal and flavours it with BandMathX. Args: @@ -733,7 +433,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation(">=", self, other) - def __le__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __le__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default less or equal and flavours it with BandMathX. Args: @@ -747,7 +447,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation("<=", self, other) - def __gt__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __gt__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default greater operator and flavours it with BandMathX. Args: @@ -761,7 +461,7 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation(">", self, other) - def __lt__(self, other: OTBObject | Output | str | int | float) -> Operation: + def __lt__(self, other: OTBObject | str | int | float) -> Operation: """Overrides the default less operator and flavours it with BandMathX. Args: @@ -775,125 +475,457 @@ class OTBObject: return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return LogicalOperation("<", self, other) - def __eq__(self, other: OTBObject | Output | str | int | float) -> Operation: - """Overrides the default eq operator and flavours it with BandMathX. + def __eq__(self, other: OTBObject | str | int | float) -> Operation: + """Overrides the default eq operator and flavours it with BandMathX. + + Args: + other: the other member of the operation + + Returns: + self == other + + """ + if isinstance(other, (np.ndarray, np.generic)): + return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ + return LogicalOperation("==", self, other) + + def __ne__(self, other: OTBObject | str | int | float) -> Operation: + """Overrides the default different operator and flavours it with BandMathX. + + Args: + other: the other member of the operation + + Returns: + self != other + + """ + if isinstance(other, (np.ndarray, np.generic)): + return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ + return LogicalOperation("!=", self, other) + + def __or__(self, other: OTBObject | str | int | float) -> Operation: + """Overrides the default or operator and flavours it with BandMathX. + + Args: + other: the other member of the operation + + Returns: + self || other + + """ + if isinstance(other, (np.ndarray, np.generic)): + return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ + return LogicalOperation("||", self, other) + + def __and__(self, other: OTBObject | str | int | float) -> Operation: + """Overrides the default and operator and flavours it with BandMathX. + + Args: + other: the other member of the operation + + Returns: + self && other + + """ + if isinstance(other, (np.ndarray, np.generic)): + return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ + return LogicalOperation("&&", self, other) + + # TODO: other operations ? + # e.g. __pow__... cf https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types + + def __array__(self) -> np.ndarray: + """This is called when running np.asarray(pyotb_object). + + Returns: + a numpy array + + """ + return self.to_numpy() + + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs) -> OTBObject: + """This is called whenever a numpy function is called on a pyotb object. + + Operation is performed in numpy, then imported back to pyotb with the same georeference as input. + + Args: + ufunc: numpy function + method: an internal numpy argument + inputs: inputs, at least one being pyotb object. If there are several pyotb objects, they must all have + the same georeference and pixel size. + **kwargs: kwargs of the numpy function + + Returns: + a pyotb object + + """ + if method == "__call__": + # Converting potential pyotb inputs to arrays + arrays = [] + image_dic = None + for inp in inputs: + if isinstance(inp, (float, int, np.ndarray, np.generic)): + arrays.append(inp) + elif isinstance(inp, OTBObject): + if not inp.exports_dic: + inp.export() + image_dic = inp.exports_dic[inp.key_output_image] + array = image_dic["array"] + arrays.append(array) + else: + logger.debug(type(self)) + return NotImplemented + # Performing the numpy operation + result_array = ufunc(*arrays, **kwargs) + result_dic = image_dic + result_dic["array"] = result_array + # Importing back to OTB, pass the result_dic just to keep reference + pyotb_app = App("ExtractROI", image_dic=result_dic, frozen=True, quiet=True) + if result_array.shape[2] == 1: + pyotb_app.app.ImportImage("in", result_dic) + else: + pyotb_app.app.ImportVectorImage("in", result_dic) + pyotb_app.execute() + return pyotb_app + return NotImplemented + + +class App(OTBObject): + """Base class that gathers common operations for any OTB application.""" + + def __init__(self, otb_app_name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, + name: str = None, **kwargs): + """Common constructor for OTB applications. Handles in-memory connection between apps. + + Args: + otb_app_name: name of the OTB application, e.g. 'BandMath' + *args: used for passing application parameters. Can be : + - dictionary containing key-arguments enumeration. Useful when a key is python-reserved + (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") + - string or OTBObject, useful when the user wants to specify the input "in" + - list, useful when the user wants to specify the input list 'il' + frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___ + quiet: whether to print logs of the OTB app + image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as + the result of app.ExportImage(). Use it when the app takes a numpy array as input. + See this related issue for why it is necessary to keep reference of object: + https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824 + name: override the application name + **kwargs: used for passing application parameters. + e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif' + + """ + self.frozen = frozen + self.quiet = quiet + create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication + super().__init__(name=name or f"OTB Application {otb_app_name}", app=create(otb_app_name), image_dic=image_dic) + self.description = self.app.GetDocLongDescription() + + # Set parameters + self.parameters = {} + if args or kwargs: + self.set_parameters(*args, **kwargs) + if not self.frozen: + self.execute() + if any(key in self.parameters for key in self.out_param_types): + self.flush() # auto flush if any output param was provided during app init + + # Elapsed time + self.time_start, self.time_end = 0, 0 + + @property + def elapsed_time(self): + """Get elapsed time between app init and end of exec or file writing.""" + return self.time_end - self.time_start + + @property + def used_outputs(self) -> list[str]: + """List of used application outputs.""" + return [getattr(self, key) for key in self.out_param_types if key in self.parameters] + + def find_outputs(self) -> tuple[str]: + """Find output files on disk using path found in parameters. + + Returns: + list of files found on disk + + """ + files, missing = [], [] + for out in self.used_outputs: + dest = files if out.exists() else missing + dest.append(str(out.filepath.absolute())) + for filename in missing: + logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) + return tuple(files) + + @property + def data(self): + """Expose app's output data values in a dictionary.""" + skip_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem") + skip_keys = skip_keys + tuple(self.out_param_types) + tuple(self.parameters) + keys = (k for k in self.parameters_keys if k not in skip_keys) + + def _check(v): + return not isinstance(v, otb.ApplicationProxy) and v not in ("", None, [], ()) + + return {str(k): self[k] for k in keys if _check(self[k])} + + def set_parameters(self, *args, **kwargs): + """Set some parameters of the app. + + When useful, e.g. for images list, this function appends the parameters + instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths + + Args: + *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved + (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") + - string or OTBObject, useful when the user implicitly wants to set the param "in" + - list, useful when the user implicitly wants to set the param "il" + **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif' + + Raises: + Exception: when the setting of a parameter failed + + """ + parameters = kwargs + parameters.update(self.__parse_args(args)) + # Going through all arguments + for key, obj in parameters.items(): + if key not in self.parameters_keys: + raise KeyError(f'{self.name}: unknown parameter name "{key}"') + # When the parameter expects a list, if needed, change the value to list + if is_key_list(self, key) and not isinstance(obj, (list, tuple)): + obj = [obj] + logger.info('%s: argument for parameter "%s" was converted to list', self.name, key) + try: + # This is when we actually call self.app.SetParameter* + self.__set_param(key, obj) + except (RuntimeError, TypeError, ValueError, KeyError) as e: + raise Exception( + f"{self.name}: something went wrong before execution " + f"(while setting parameter '{key}' to '{obj}')" + ) from e + # Update _parameters using values from OtbApplication object + otb_params = self.app.GetParameters().items() + otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params} + # Update param dict and save values as object attributes + self.parameters.update({**parameters, **otb_params}) + self.save_objects() + + def save_objects(self): + """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`. - Args: - other: the other member of the operation + This is useful when the key contains reserved characters such as a point eg "io.out" + """ + for key in self.parameters_keys: + if key in dir(self.__class__): + continue # skip forbidden attribute since it is already used by the class + value = self.parameters.get(key) # basic parameters + if value is None: + try: + value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results) + except RuntimeError: + continue # this is when there is no value for key + # Convert output param path to Output object + if key in self.out_param_types: + value = Output(self, key, value) + elif isinstance(value, str): + try: + value = literal_eval(value) + except (ValueError, SyntaxError): + pass + # Save attribute + setattr(self, key, value) - Returns: - self == other + def execute(self): + """Execute and write to disk if any output parameter has been set during init.""" + logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters) + self.time_start = perf_counter() + try: + self.app.Execute() + except (RuntimeError, FileNotFoundError) as e: + raise Exception(f"{self.name}: error during during app execution") from e + self.frozen = False + self.time_end = perf_counter() + logger.debug("%s: execution ended", self.name) + self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return LogicalOperation("==", self, other) + def flush(self): + """Flush data to disk, this is when WriteOutput is actually called.""" + try: + logger.debug("%s: flushing data to disk", self.name) + self.app.WriteOutput() + except RuntimeError: + logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name) + self.app.ExecuteAndWriteOutput() + self.time_end = perf_counter() - def __ne__(self, other: OTBObject | Output | str | int | float) -> Operation: - """Overrides the default different operator and flavours it with BandMathX. + def write(self, *args, filename_extension: str = "", pixel_type: dict[str, str] | str = None, + preserve_dtype: bool = False, **kwargs): + """Set output pixel type and write the output raster files. Args: - other: the other member of the operation - - Returns: - self != other + *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key contains + non-standard characters such as a point, e.g. {'io.out':'output.tif'} + - string, useful when there is only one output, e.g. 'output.tif' + - None if output file was passed during App init + filename_extension: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES") + Will be used for all outputs (Default value = "") + pixel_type: Can be : - dictionary {output_parameter_key: pixeltype} when specifying for several outputs + - str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several + outputs, all outputs are written with this unique type. + Valid pixel types are uint8, uint16, uint32, int16, int32, float, double, + cint16, cint32, cfloat, cdouble. (Default value = None) + preserve_dtype: propagate main input pixel type to outputs, in case pixel_type is None + **kwargs: keyword arguments e.g. out='output.tif' """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return LogicalOperation("!=", self, other) + # Gather all input arguments in kwargs dict + for arg in args: + if isinstance(arg, dict): + kwargs.update(arg) + elif isinstance(arg, str) and kwargs: + logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg) + elif isinstance(arg, str) and self.key_output_image: + kwargs.update({self.key_output_image: arg}) - def __or__(self, other: OTBObject | Output | str | int | float) -> Operation: - """Overrides the default or operator and flavours it with BandMathX. + # Append filename extension to filenames + if filename_extension: + logger.debug("%s: using extended filename for outputs: %s", self.name, filename_extension) + if not filename_extension.startswith("?"): + filename_extension = "?" + filename_extension + for key, value in kwargs.items(): + if self.out_param_types[key] == otb.ParameterType_OutputImage and '?' not in value: + kwargs[key] = value + filename_extension - Args: - other: the other member of the operation + # Manage output pixel types + dtypes = {} + if pixel_type: + if isinstance(pixel_type, str): + type_name = self.app.ConvertPixelTypeToNumpy(parse_pixel_type(pixel_type)) + logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name) + for key in kwargs: + if self.out_param_types[key] == otb.ParameterType_OutputImage: + dtypes[key] = parse_pixel_type(pixel_type) + elif isinstance(pixel_type, dict): + dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()} + elif preserve_dtype: + self.propagate_dtype() # all outputs will have the same type as the main input raster - Returns: - self || other + # Set parameters and flush to disk + for key, output_filename in kwargs.items(): + if key in dtypes: + self.propagate_dtype(key, dtypes[key]) + self.set_parameters({key: output_filename}) + self.flush() - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return LogicalOperation("||", self, other) + def propagate_dtype(self, target_key: str = None, dtype: int = None): + """Propagate a pixel type from main input to every outputs, or to a target output key only. - def __and__(self, other: OTBObject | Output | str | int | float) -> Operation: - """Overrides the default and operator and flavours it with BandMathX. + With multiple inputs (if dtype is not provided), the type of the first input is considered. + With multiple outputs (if target_key is not provided), all outputs will be converted to the same pixel type. Args: - other: the other member of the operation - - Returns: - self && other + target_key: output param key to change pixel type + dtype: data type to use """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return LogicalOperation("&&", self, other) + if not dtype: + param = self.parameters.get(self.key_input_image) + if not param: + logger.warning("%s: could not propagate pixel type from inputs to output", self.name) + return + if isinstance(param, (list, tuple)): + param = param[0] # first image in "il" + try: + dtype = get_pixel_type(param) + except (TypeError, RuntimeError): + logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param) + return + if target_key: + keys = [target_key] + else: + keys = [k for k, v in self.out_param_types.items() if v == otb.ParameterType_OutputImage] + for key in keys: + self.app.SetParameterOutputImagePixelType(key, dtype) - # TODO: other operations ? - # e.g. __pow__... cf https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types + # Private functions + def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]: + """Gather all input arguments in kwargs dict. - def __array__(self) -> np.ndarray: - """This is called when running np.asarray(pyotb_object). + Args: + args: the list of arguments passed to set_parameters() Returns: - a numpy array + a dictionary with the right keyword depending on the object """ - return self.to_numpy() - - def __array_ufunc__(self, ufunc, method, *inputs, **kwargs) -> OTBObject: - """This is called whenever a numpy function is called on a pyotb object. + kwargs = {} + for arg in args: + if isinstance(arg, dict): + kwargs.update(arg) + elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.key_input): + kwargs.update({self.key_input: arg}) + return kwargs - Operation is performed in numpy, then imported back to pyotb with the same georeference as input. + def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]): + """Set one parameter, decide which otb.Application method to use depending on target object.""" + if obj is None or (isinstance(obj, (list, tuple)) and not obj): + self.app.ClearValue(key) + return + if key not in self.parameters_keys: + raise KeyError( + f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}" + ) + # Single-parameter cases + if isinstance(obj, OTBObject): + self.app.ConnectImage(key, obj.app, obj.key_output_image) + elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB + self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0]) + elif key == "ram": # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200 + self.app.SetParameterInt("ram", int(obj)) + elif not isinstance(obj, list): # any other parameters (str, int...) + self.app.SetParameterValue(key, obj) + # Images list + elif is_key_images_list(self, key): + # To enable possible in-memory connections, we go through the list and set the parameters one by one + for inp in obj: + if isinstance(inp, OTBObject): + self.app.ConnectImage(key, inp.app, inp.key_output_image) + elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB + self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0]) + else: # here `input` should be an image filepath + # Append `input` to the list, do not overwrite any previously set element of the image list + self.app.AddParameterStringList(key, inp) + # List of any other types (str, int...) + else: + self.app.SetParameterValue(key, obj) - Args: - ufunc: numpy function - method: an internal numpy argument - inputs: inputs, at least one being pyotb object. If there are several pyotb objects, they must all have - the same georeference and pixel size. - **kwargs: kwargs of the numpy function + def summarize(self) -> dict: + """Serialize an object and its pipeline into a dictionary. Returns: - a pyotb object + nested dictionary summarizing the pipeline """ - if method == "__call__": - # Converting potential pyotb inputs to arrays - arrays = [] - image_dic = None - for inp in inputs: - if isinstance(inp, (float, int, np.ndarray, np.generic)): - arrays.append(inp) - elif isinstance(inp, OTBObject): - if not inp.exports_dic: - inp.export() - image_dic = inp.exports_dic[inp.key_output_image] - array = image_dic["array"] - arrays.append(array) - else: - logger.debug(type(self)) - return NotImplemented - # Performing the numpy operation - result_array = ufunc(*arrays, **kwargs) - result_dic = image_dic - result_dic["array"] = result_array - # Importing back to OTB, pass the result_dic just to keep reference - app = OTBObject("ExtractROI", image_dic=result_dic, frozen=True, quiet=True) - if result_array.shape[2] == 1: - app.ImportImage("in", result_dic) - else: - app.ImportVectorImage("in", result_dic) - app.execute() - return app - return NotImplemented + params = self.parameters + for k, p in params.items(): + # In the following, we replace each parameter which is an OTBObject, with its summary. + if isinstance(p, App): # single parameter + params[k] = p.summarize() + elif isinstance(p, list): # parameter list + params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p] + return {"name": self.app.GetName(), "parameters": params} + + def __str__(self): + """Return a nice string representation with object id.""" + return f"<pyotb.App {self.name} object id {id(self)}>" -class Slicer(OTBObject): +class Slicer(App): """Slicer objects i.e. when we call something like raster[:, :, 2] from Python.""" - def __init__(self, obj: OTBObject | Output | str, rows: int, cols: int, channels: int): + def __init__(self, obj: OTBObject | str, rows: int, cols: int, channels: int): """Create a slicer object, that can be used directly for writing or inside a BandMath. It contains : @@ -907,8 +939,7 @@ class Slicer(OTBObject): channels: channels, can be slicing, list or int """ - super().__init__("ExtractROI", {"in": obj, "mode": "extent"}, quiet=True, frozen=True) - self.name = "Slicer" + super().__init__("ExtractROI", {"in": obj, "mode": "extent"}, quiet=True, frozen=True, name="Slicer") self.rows, self.cols = rows, cols parameters = {} @@ -957,7 +988,7 @@ class Slicer(OTBObject): self.execute() -class Operation(OTBObject): +class Operation(App): """Class for arithmetic/math operations done in Python. Example: @@ -978,7 +1009,7 @@ class Operation(OTBObject): """ - def __init__(self, operator: str, *inputs, nb_bands: int = None): + def __init__(self, operator: str, *inputs, nb_bands: int = None, name: str = None): """Given some inputs and an operator, this function enables to transform this into an OTB application. Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator. @@ -986,8 +1017,9 @@ class Operation(OTBObject): Args: operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? - *inputs: inputs. Can be App, Output, Input, Operation, Slicer, filepath, int or float + *inputs: inputs. Can be OTBObject, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where + name: override the Operation name """ self.operator = operator @@ -1014,11 +1046,10 @@ class Operation(OTBObject): self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)] self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands) # Execute app - name = "BandMath" if len(self.exp_bands) == 1 else "BandMathX" - super().__init__(name, il=self.unique_inputs, exp=self.exp, quiet=True) - self.name = f'Operation exp="{self.exp}"' + super().__init__("BandMath" if len(self.exp_bands) == 1 else "BandMathX", il=self.unique_inputs, + exp=self.exp, quiet=True, name=name or f'Operation exp="{self.exp}"') - def create_fake_exp(self, operator: str, inputs: list[OTBObject | Output | str | int | float], + def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None): """Create a 'fake' expression. @@ -1026,7 +1057,7 @@ class Operation(OTBObject): Args: operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? - inputs: inputs. Can be App, Output, Input, Operation, Slicer, filepath, int or float + inputs: inputs. Can be OTBObject, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ @@ -1110,7 +1141,7 @@ class Operation(OTBObject): return exp_bands, exp @staticmethod - def create_one_input_fake_exp(x: OTBObject | Output | str, + def create_one_input_fake_exp(x: OTBObject | str, band: int, keep_logical: bool = False) -> tuple(str, list[OTBObject], int): """This an internal function, only to be used by `create_fake_exp`. @@ -1190,19 +1221,18 @@ class LogicalOperation(Operation): nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ - super().__init__(operator, *inputs, nb_bands=nb_bands) + super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation") self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands) - def create_fake_exp(self, operator: str, inputs: list[OTBObject | Output | str | int | float], - nb_bands: int = None): + def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None): """Create a 'fake' expression. - E.g for the operation input1 > input2, we create a fake expression that is like + e.g for the operation input1 > input2, we create a fake expression that is like "str(input1) > str(input2) ? 1 : 0" and a logical fake expression that is like "str(input1) > str(input2)" Args: operator: str (one of >, <, >=, <=, ==, !=, &, |) - inputs: Can be App, Output, Input, Operation, Slicer, filepath, int or float + inputs: Can be OTBObject, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ @@ -1237,7 +1267,7 @@ class LogicalOperation(Operation): self.fake_exp_bands.append(fake_exp) -class Input(OTBObject): +class Input(App): """Class for transforming a filepath to pyOTB object.""" def __init__(self, path: str): @@ -1248,8 +1278,7 @@ class Input(OTBObject): """ self.path = path - super().__init__("ExtractROI", {"in": path}, frozen=True) - self.name = f"Input from {path}" + super().__init__("ExtractROI", {"in": path}, frozen=True, name=f"Input from {path}") self.propagate_dtype() self.execute() @@ -1262,7 +1291,7 @@ class Output(OTBObject): """Object that behave like a pointer to a specific application output file.""" def __init__(self, pyotb_app: OTBObject, # pylint: disable=super-init-not-called - param_key: str, filepath: str = None, mkdir: bool = True): + param_key: str = None, filepath: str = None, mkdir: bool = True): """Constructor for an Output object. Args: @@ -1272,9 +1301,9 @@ class Output(OTBObject): mkdir: create missing parent directories """ - self.pyotb_app, self.app = pyotb_app, pyotb_app.app - self.parameters = pyotb_app.parameters - self.param_key = param_key + super().__init__(name=f"Output {param_key} from {pyotb_app.name}", app=pyotb_app.app) + self.parent_pyotb_app = pyotb_app # keep trace of parent app + self.param_key = param_key or super().key_output_image self.filepath = None if filepath: if '?' in filepath: @@ -1282,7 +1311,6 @@ class Output(OTBObject): self.filepath = Path(filepath) if mkdir: self.make_parent_dirs() - self.name = f"Output {param_key} from {self.pyotb_app.name}" @property def key_output_image(self): @@ -1291,16 +1319,18 @@ class Output(OTBObject): def exists(self) -> bool: """Check file exist.""" + assert self.filepath, "Filepath not set" return self.filepath.exists() def make_parent_dirs(self): """Create missing parent directories.""" + assert self.filepath, "Filepath not set" if not self.filepath.parent.exists(): self.filepath.parent.mkdir(parents=True) def __str__(self) -> str: """Return a nice string representation with source app name and object id.""" - return f"<pyotb.Output {self.source_app.name} object, id {id(self)}>" + return f"<pyotb.Output {self.name} object, id {id(self)}>" def get_nbchannels(inp: str | OTBObject) -> int: @@ -1318,8 +1348,8 @@ def get_nbchannels(inp: str | OTBObject) -> int: else: # Executing the app, without printing its log try: - info = OTBObject("ReadImageInfo", inp, quiet=True) - nb_channels = info.GetParameterInt("numberbands") + info = App("ReadImageInfo", inp, quiet=True) + nb_channels = info.app.GetParameterInt("numberbands") except Exception as e: # this happens when we pass a str that is not a filepath raise TypeError(f'Could not get the number of channels of `{inp}`. Not a filepath or wrong filepath') from e return nb_channels @@ -1338,10 +1368,10 @@ def get_pixel_type(inp: str | OTBObject) -> str: """ if isinstance(inp, str): try: - info = OTBObject("ReadImageInfo", inp, quiet=True) + info = App("ReadImageInfo", inp, quiet=True) except Exception as info_err: # this happens when we pass a str that is not a filepath raise TypeError(f"Could not get the pixel type of `{inp}`. Not a filepath or wrong filepath") from info_err - datatype = info.GetParameterString("datatype") # which is such as short, float... + datatype = info.app.GetParameterString("datatype") # which is such as short, float... if not datatype: raise TypeError(f"Unable to read pixel type of image {inp}") datatype_to_pixeltype = { @@ -1359,7 +1389,7 @@ def get_pixel_type(inp: str | OTBObject) -> str: raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}") pixel_type = getattr(otb, f'ImagePixelType_{datatype_to_pixeltype[datatype]}') elif isinstance(inp, OTBObject): - pixel_type = inp.GetParameterOutputImagePixelType(inp.key_output_image) + pixel_type = inp.app.GetParameterOutputImagePixelType(inp.key_output_image) else: raise TypeError(f'Could not get the pixel type of {type(inp)} object {inp}') return pixel_type diff --git a/pyotb/functions.py b/pyotb/functions.py index 71c298b..d25610a 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -9,12 +9,11 @@ import textwrap import subprocess from collections import Counter -from .core import OTBObject, Input, Operation, LogicalOperation, get_nbchannels, Output +from .core import OTBObject, App, Operation, LogicalOperation, Input, get_nbchannels from .helpers import logger -def where(cond: OTBObject | Output | str, x: OTBObject | Output | str | int | float, - y: OTBObject | Output | str | int | float) -> Operation: +def where(cond: OTBObject | str, x: OTBObject | str | int | float, y: OTBObject | str | int | float) -> Operation: """Functionally similar to numpy.where. Where cond is True (!=0), returns x. Else returns y. Args: @@ -64,8 +63,7 @@ def where(cond: OTBObject | Output | str, x: OTBObject | Output | str | int | fl return operation -def clip(a: OTBObject | Output | str, a_min: OTBObject | Output | str | int | float, - a_max: OTBObject | Output | str | int | float): +def clip(a: OTBObject | str, a_min: OTBObject | str | int | float, a_max: OTBObject | str | int | float): """Clip values of image in a range of values. Args: @@ -355,9 +353,9 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ metadatas = {} for inp in inputs: if isinstance(inp, str): # this is for filepaths - metadata = Input(inp).GetImageMetaData('out') + metadata = Input(inp).app.GetImageMetaData('out') elif isinstance(inp, OTBObject): - metadata = inp.GetImageMetaData(inp.output_param) + metadata = inp.app.GetImageMetaData(inp.output_param) else: raise TypeError(f"Wrong input : {inp}") metadatas[inp] = metadata @@ -416,7 +414,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ 'mode.extent.ulx': ulx, 'mode.extent.uly': lry, # bug in OTB <= 7.3 : 'mode.extent.lrx': lrx, 'mode.extent.lry': uly, # ULY/LRY are inverted } - new_input = OTBObject('ExtractROI', params) + new_input = App('ExtractROI', params) # TODO: OTB 7.4 fixes this bug, how to handle different versions of OTB? new_inputs.append(new_input) # Potentially update the reference inputs for later resampling @@ -429,7 +427,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ inputs = new_inputs # Update metadatas - metadatas = {input: input.GetImageMetaData('out') for input in inputs} + metadatas = {input: input.app.GetImageMetaData('out') for input in inputs} # Get a metadata of an arbitrary image. This is just to compare later with other images any_metadata = next(iter(metadatas.values())) @@ -457,14 +455,14 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ new_inputs = [] for inp in inputs: if metadatas[inp]['GeoTransform'][1] != pixel_size: - superimposed = OTBObject('Superimpose', inr=reference_input, inm=inp, interpolator=interpolator) + superimposed = App('Superimpose', inr=reference_input, inm=inp, interpolator=interpolator) new_inputs.append(superimposed) else: new_inputs.append(inp) inputs = new_inputs # Update metadatas - metadatas = {inp: inp.GetImageMetaData('out') for inp in inputs} + metadatas = {inp: inp.app.GetImageMetaData('out') for inp in inputs} # Final superimposition to be sure to have the exact same image sizes # Getting the sizes of images @@ -482,7 +480,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ new_inputs = [] for inp in inputs: if image_sizes[inp] != most_common_image_size: - superimposed = OTBObject('Superimpose', inr=same_size_images[0], inm=inp, interpolator=interpolator) + superimposed = App('Superimpose', inr=same_size_images[0], inm=inp, interpolator=interpolator) new_inputs.append(superimposed) else: new_inputs.append(inp) diff --git a/tests/test_core.py b/tests/test_core.py index 436f99e..086c26e 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,13 +1,8 @@ -import os -import pyotb -from ast import literal_eval -from pathlib import Path - import pytest +import pyotb +from tests_data import INPUT -FILEPATH = os.environ["TEST_INPUT_IMAGE"] -INPUT = pyotb.Input(FILEPATH) TEST_IMAGE_STATS = { 'out.mean': [79.5505, 109.225, 115.456, 249.349], 'out.min': [33, 64, 91, 47], @@ -59,9 +54,11 @@ def test_nonraster_property(): with pytest.raises(TypeError): pyotb.ReadImageInfo(INPUT).dtype + def test_elapsed_time(): assert pyotb.ReadImageInfo(INPUT).elapsed_time < 1 + # Other functions def test_get_infos(): infos = INPUT.get_infos() diff --git a/tests/test_numpy.py b/tests/test_numpy.py index dd33425..0f42435 100644 --- a/tests/test_numpy.py +++ b/tests/test_numpy.py @@ -1,10 +1,10 @@ -import os import numpy as np import pyotb +from tests_data import INPUT +import numpy as np - -FILEPATH = os.environ["TEST_INPUT_IMAGE"] -INPUT = pyotb.Input(FILEPATH) +import pyotb +from tests_data import INPUT def test_export(): @@ -37,7 +37,7 @@ def test_convert_to_array(): def test_pixel_coords_otb_equals_numpy(): - assert INPUT[19,7] == list(INPUT.to_numpy()[19,7]) + assert INPUT[19, 7] == list(INPUT.to_numpy()[19, 7]) def test_add_noise_array(): diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 3c88153..3acc354 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,9 +1,8 @@ -import sys import os import itertools import pytest import pyotb -from pyotb.helpers import logger +from tests_data import INPUT, FILEPATH # List of buildings blocks, we can add other pyotb objects here @@ -23,8 +22,6 @@ PYOTB_BLOCKS = [ PIPELINES_LENGTH = [1, 2, 3] ALL_BLOCKS = PYOTB_BLOCKS + OTBAPPS_BLOCKS -FILEPATH = os.environ["TEST_INPUT_IMAGE"] -INPUT = pyotb.Input(FILEPATH) def generate_pipeline(inp, building_blocks): diff --git a/tests/test_serialization.py b/tests/test_serialization.py index e64f2f1..b56e447 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -1,18 +1,16 @@ -import os import pyotb - -filepath = os.environ["TEST_INPUT_IMAGE"] +from tests_data import FILEPATH def test_pipeline_simple(): # BandMath -> OrthoRectification -> ManageNoData - app1 = pyotb.BandMath({'il': [filepath], 'exp': 'im1b1'}) + app1 = pyotb.BandMath({'il': [FILEPATH], 'exp': 'im1b1'}) app2 = pyotb.OrthoRectification({'io.in': app1}) app3 = pyotb.ManageNoData({'in': app2}) summary = app3.summarize() reference = {'name': 'ManageNoData', 'parameters': {'in': { 'name': 'OrthoRectification', 'parameters': {'io.in': { - 'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}}, + 'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}}, 'map': 'utm', 'outputs.isotropic': True}}, 'mode': 'buildmask'}} @@ -21,19 +19,19 @@ def test_pipeline_simple(): def test_pipeline_diamond(): # Diamond graph - app1 = pyotb.BandMath({'il': [filepath], 'exp': 'im1b1'}) + app1 = pyotb.BandMath({'il': [FILEPATH], 'exp': 'im1b1'}) app2 = pyotb.OrthoRectification({'io.in': app1}) app3 = pyotb.ManageNoData({'in': app2}) app4 = pyotb.BandMathX({'il': [app2, app3], 'exp': 'im1+im2'}) summary = app4.summarize() reference = {'name': 'BandMathX', 'parameters': {'il': [ {'name': 'OrthoRectification', 'parameters': {'io.in': { - 'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}}, + 'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}}, 'map': 'utm', 'outputs.isotropic': True}}, {'name': 'ManageNoData', 'parameters': {'in': { 'name': 'OrthoRectification', 'parameters': { - 'io.in': {'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}}, + 'io.in': {'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}}, 'map': 'utm', 'outputs.isotropic': True}}, 'mode': 'buildmask'}} diff --git a/tests/tests_data.py b/tests/tests_data.py new file mode 100644 index 0000000..b190f40 --- /dev/null +++ b/tests/tests_data.py @@ -0,0 +1,3 @@ +import pyotb +FILEPATH = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false" +INPUT = pyotb.Input(FILEPATH) -- GitLab From 6a4ba70c3de182ff88db74ae60cb92aba3b8bfe3 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 2 Feb 2023 10:34:39 +0100 Subject: [PATCH 091/399] STYLE: linting --- pyotb/core.py | 4 ++-- pyotb/functions.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index cc3c96e..a582159 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -614,7 +614,7 @@ class App(RasterInterface): # This is when we actually call self.app.SetParameter* self.__set_param(key, obj) except (RuntimeError, TypeError, ValueError, KeyError) as e: - raise Exception( + raise RuntimeError( f"{self.name}: something went wrong before execution " f"(while setting parameter '{key}' to '{obj}')" ) from e @@ -687,7 +687,7 @@ class App(RasterInterface): try: self.app.Execute() except (RuntimeError, FileNotFoundError) as e: - raise Exception(f"{self.name}: error during during app execution") from e + raise RuntimeError(f"{self.name}: error during during app execution") from e self.frozen = False self._time_end = perf_counter() logger.debug("%s: execution ended", self.name) diff --git a/pyotb/functions.py b/pyotb/functions.py index d782a0e..6cc7889 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -118,7 +118,7 @@ def all(*inputs): # pylint: disable=redefined-builtin if isinstance(inputs[0], LogicalOperation): res = inputs[0] else: - res = (inputs[0] != 0) + res = inputs[0] != 0 for inp in inputs[1:]: if isinstance(inp, LogicalOperation): res = res & inp @@ -171,7 +171,7 @@ def any(*inputs): # pylint: disable=redefined-builtin if isinstance(inputs[0], LogicalOperation): res = inputs[0] else: - res = (inputs[0] != 0) + res = inputs[0] != 0 for inp in inputs[1:]: if isinstance(inp, LogicalOperation): res = res | inp -- GitLab From b674ff86850e48c7ddb85412f97fc14e2b4663a9 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 2 Feb 2023 21:34:05 +0100 Subject: [PATCH 092/399] FIX: bug with output --- pyotb/core.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index a582159..e8761c1 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -590,7 +590,7 @@ class App(RasterInterface): Args: *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") - - string or App, useful when the user implicitly wants to set the param "in" + - string or RasterInterface, useful when the user implicitly wants to set the param "in" - list, useful when the user implicitly wants to set the param "il" **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif' @@ -788,15 +788,15 @@ class App(RasterInterface): """ parameters = self.parameters.copy() for key, param in parameters.items(): - # In the following, we replace each parameter which is an App, with its summary. - if isinstance(param, App): # single parameter + # In the following, we replace each parameter which is an RasterInterface, with its summary. + if isinstance(param, RasterInterface): # single parameter parameters[key] = param.summarize() elif isinstance(param, list): # parameter list - parameters[key] = [p.summarize() if isinstance(p, App) else p for p in param] + parameters[key] = [p.summarize() if isinstance(p, RasterInterface) else p for p in param] return {"name": self.app.GetName(), "parameters": parameters} # Private functions - def __parse_args(self, args: list[str | App | dict | list]) -> dict[str, Any]: + def __parse_args(self, args: list[str | RasterInterface | dict | list]) -> dict[str, Any]: """Gather all input arguments in kwargs dict. Args: @@ -810,17 +810,17 @@ class App(RasterInterface): for arg in args: if isinstance(arg, dict): kwargs.update(arg) - elif isinstance(arg, (str, App)) or isinstance(arg, list) and is_key_list(self, self.key_input): + elif isinstance(arg, (str, RasterInterface)) or isinstance(arg, list) and is_key_list(self, self.key_input): kwargs.update({self.key_input: arg}) return kwargs - def __set_param(self, key: str, obj: list | tuple | App | otb.Application | list[Any]): + def __set_param(self, key: str, obj: list | tuple | RasterInterface | otb.Application | list[Any]): """Set one parameter, decide which otb.Application method to use depending on target object.""" if obj is None or (isinstance(obj, (list, tuple)) and not obj): self.app.ClearValue(key) return # Single-parameter cases - if isinstance(obj, App): + if isinstance(obj, RasterInterface): self.app.ConnectImage(key, obj.app, obj.key_output_image) elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0]) @@ -832,7 +832,7 @@ class App(RasterInterface): elif is_key_images_list(self, key): # To enable possible in-memory connections, we go through the list and set the parameters one by one for inp in obj: - if isinstance(inp, App): + if isinstance(inp, RasterInterface): self.app.ConnectImage(key, inp.app, inp.key_output_image) elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0]) -- GitLab From b71950898619af15f2b3976a1a6339cf6a7ac136 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 2 Feb 2023 22:22:12 +0100 Subject: [PATCH 093/399] REFAC: Operators --- pyotb/core.py | 240 +++++++------------------------------------------- 1 file changed, 32 insertions(+), 208 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index e8761c1..a9ea536 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -198,241 +198,65 @@ class RasterInterface(ABC): row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x return abs(int(row)), int(col) - def __add__(self, other: App | str | int | float) -> Operation: - """Overrides the default addition and flavours it with BandMathX. - - Args: - other: the other member of the operation - - Returns: - self + other - - """ - if isinstance(other, (np.ndarray, np.generic)): + @staticmethod + def _create_operator(op_cls, name, a, b): + if isinstance(b, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation("+", self, other) + return op_cls(name, a, b) - def __sub__(self, other: App | str | int | float) -> Operation: - """Overrides the default subtraction and flavours it with BandMathX. - - Args: - other: the other member of the operation - - Returns: - self - other + def __add__(self, other: App | str | int | float) -> Operation: + return self._create_operator(Operation, "+", self, other) - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation("-", self, other) + def __sub__(self, other: App | str | int | float) -> Operation: + return self._create_operator(Operation, "-", self, other) def __mul__(self, other: App | str | int | float) -> Operation: - """Overrides the default subtraction and flavours it with BandMathX. - - Args: - other: the other member of the operation - - Returns: - self * other - - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation("*", self, other) + return self._create_operator(Operation, "*", self, other) def __truediv__(self, other: App | str | int | float) -> Operation: - """Overrides the default subtraction and flavours it with BandMathX. - - Args: - other: the other member of the operation - - Returns: - self / other - - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation("/", self, other) + return self._create_operator(Operation, "/", self, other) def __radd__(self, other: App | str | int | float) -> Operation: - """Overrides the default reverse addition and flavours it with BandMathX. - - Args: - other: the other member of the operation - - Returns: - other + self - - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation("+", other, self) + return self._create_operator(Operation, "+", other, self) def __rsub__(self, other: App | str | int | float) -> Operation: - """Overrides the default subtraction and flavours it with BandMathX. - - Args: - other: the other member of the operation - - Returns: - other - self - - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation("-", other, self) + return self._create_operator(Operation, "-", other, self) def __rmul__(self, other: App | str | int | float) -> Operation: - """Overrides the default multiplication and flavours it with BandMathX. - - Args: - other: the other member of the operation - - Returns: - other * self - - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation("*", other, self) + return self._create_operator(Operation, "*", other, self) def __rtruediv__(self, other: App | str | int | float) -> Operation: - """Overrides the default division and flavours it with BandMathX. - - Args: - other: the other member of the operation - - Returns: - other / self - - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return Operation("/", other, self) + return self._create_operator(Operation, "/", other, self) def __abs__(self) -> Operation: - """Overrides the default abs operator and flavours it with BandMathX. - - Returns: - abs(self) - - """ return Operation("abs", self) - def __ge__(self, other: App | str | int | float) -> Operation: - """Overrides the default greater or equal and flavours it with BandMathX. - - Args: - other: the other member of the operation - - Returns: - self >= other - - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return LogicalOperation(">=", self, other) - - def __le__(self, other: App | str | int | float) -> Operation: - """Overrides the default less or equal and flavours it with BandMathX. - - Args: - other: the other member of the operation - - Returns: - self <= other - - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return LogicalOperation("<=", self, other) + def __ge__(self, other: App | str | int | float) -> LogicalOperation: + return self._create_operator(LogicalOperation, ">=", self, other) - def __gt__(self, other: App | str | int | float) -> Operation: - """Overrides the default greater operator and flavours it with BandMathX. + def __le__(self, other: App | str | int | float) -> LogicalOperation: + return self._create_operator(LogicalOperation, "<=", self, other) - Args: - other: the other member of the operation + def __gt__(self, other: App | str | int | float) -> LogicalOperation: + return self._create_operator(LogicalOperation, ">", self, other) - Returns: - self > other + def __lt__(self, other: App | str | int | float) -> LogicalOperation: + return self._create_operator(LogicalOperation, "<", self, other) - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return LogicalOperation(">", self, other) + def __eq__(self, other: App | str | int | float) -> LogicalOperation: + return self._create_operator(LogicalOperation, "==", self, other) - def __lt__(self, other: App | str | int | float) -> Operation: - """Overrides the default less operator and flavours it with BandMathX. + def __ne__(self, other: App | str | int | float) -> LogicalOperation: + return self._create_operator(LogicalOperation, "!=", self, other) - Args: - other: the other member of the operation + def __or__(self, other: App | str | int | float) -> LogicalOperation: + return self._create_operator(LogicalOperation, "||", self, other) - Returns: - self < other - - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return LogicalOperation("<", self, other) - - def __eq__(self, other: App | str | int | float) -> Operation: - """Overrides the default eq operator and flavours it with BandMathX. - - Args: - other: the other member of the operation - - Returns: - self == other - - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return LogicalOperation("==", self, other) - - def __ne__(self, other: App | str | int | float) -> Operation: - """Overrides the default different operator and flavours it with BandMathX. - - Args: - other: the other member of the operation - - Returns: - self != other - - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return LogicalOperation("!=", self, other) - - def __or__(self, other: App | str | int | float) -> Operation: - """Overrides the default or operator and flavours it with BandMathX. - - Args: - other: the other member of the operation - - Returns: - self || other - - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return LogicalOperation("||", self, other) - - def __and__(self, other: App | str | int | float) -> Operation: - """Overrides the default and operator and flavours it with BandMathX. - - Args: - other: the other member of the operation - - Returns: - self && other - - """ - if isinstance(other, (np.ndarray, np.generic)): - return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return LogicalOperation("&&", self, other) + def __and__(self, other: App | str | int | float) -> LogicalOperation: + return self._create_operator(LogicalOperation, "&&", self, other) - # TODO: other operations ? - # e.g. __pow__... cf https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types + # Some other operations could be implemented with the same pattern + # e.g. __pow__... cf https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types def __array__(self) -> np.ndarray: """This is called when running np.asarray(pyotb_object). -- GitLab From 805f6b9e3ba347860c43c4d65bc05ed01e119267 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 2 Feb 2023 22:36:09 +0100 Subject: [PATCH 094/399] DOC: docstring --- pyotb/core.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index a9ea536..2499aa6 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -199,60 +199,89 @@ class RasterInterface(ABC): return abs(int(row)), int(col) @staticmethod - def _create_operator(op_cls, name, a, b): - if isinstance(b, (np.ndarray, np.generic)): + def _create_operator(op_cls, name, x, y) -> Operation: + """Create an operator. + + Args: + op_cls: Operator class + name: operator expression + x: first element + y: second element + + Return: + operator + + """ + if isinstance(y, (np.ndarray, np.generic)): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ - return op_cls(name, a, b) + return op_cls(name, x, y) def __add__(self, other: App | str | int | float) -> Operation: + """Addition.""" return self._create_operator(Operation, "+", self, other) def __sub__(self, other: App | str | int | float) -> Operation: + """Subtraction.""" return self._create_operator(Operation, "-", self, other) def __mul__(self, other: App | str | int | float) -> Operation: + """Multiplication.""" return self._create_operator(Operation, "*", self, other) def __truediv__(self, other: App | str | int | float) -> Operation: + """Division.""" return self._create_operator(Operation, "/", self, other) def __radd__(self, other: App | str | int | float) -> Operation: + """Right addition.""" return self._create_operator(Operation, "+", other, self) def __rsub__(self, other: App | str | int | float) -> Operation: + """Right subtraction.""" return self._create_operator(Operation, "-", other, self) def __rmul__(self, other: App | str | int | float) -> Operation: + """Right multiplication.""" return self._create_operator(Operation, "*", other, self) def __rtruediv__(self, other: App | str | int | float) -> Operation: + """Right division.""" return self._create_operator(Operation, "/", other, self) def __abs__(self) -> Operation: + """Absolute value.""" return Operation("abs", self) def __ge__(self, other: App | str | int | float) -> LogicalOperation: + """Greater of equal than.""" return self._create_operator(LogicalOperation, ">=", self, other) def __le__(self, other: App | str | int | float) -> LogicalOperation: + """Lower of equal than.""" return self._create_operator(LogicalOperation, "<=", self, other) def __gt__(self, other: App | str | int | float) -> LogicalOperation: + """Greater than.""" return self._create_operator(LogicalOperation, ">", self, other) def __lt__(self, other: App | str | int | float) -> LogicalOperation: + """Lower than.""" return self._create_operator(LogicalOperation, "<", self, other) def __eq__(self, other: App | str | int | float) -> LogicalOperation: + """Equality.""" return self._create_operator(LogicalOperation, "==", self, other) def __ne__(self, other: App | str | int | float) -> LogicalOperation: + """Inequality.""" return self._create_operator(LogicalOperation, "!=", self, other) def __or__(self, other: App | str | int | float) -> LogicalOperation: + """Logical or.""" return self._create_operator(LogicalOperation, "||", self, other) def __and__(self, other: App | str | int | float) -> LogicalOperation: + """Logical and.""" return self._create_operator(LogicalOperation, "&&", self, other) # Some other operations could be implemented with the same pattern -- GitLab From ac5f66d82a9362606846fb2466ae0b9102f552e6 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 3 Feb 2023 10:25:39 +0100 Subject: [PATCH 095/399] DOC: ops can deal with RasterInterface objects --- pyotb/core.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 2499aa6..1781d18 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -216,35 +216,35 @@ class RasterInterface(ABC): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return op_cls(name, x, y) - def __add__(self, other: App | str | int | float) -> Operation: + def __add__(self, other: RasterInterface | str | int | float) -> Operation: """Addition.""" return self._create_operator(Operation, "+", self, other) - def __sub__(self, other: App | str | int | float) -> Operation: + def __sub__(self, other: RasterInterface | str | int | float) -> Operation: """Subtraction.""" return self._create_operator(Operation, "-", self, other) - def __mul__(self, other: App | str | int | float) -> Operation: + def __mul__(self, other: RasterInterface | str | int | float) -> Operation: """Multiplication.""" return self._create_operator(Operation, "*", self, other) - def __truediv__(self, other: App | str | int | float) -> Operation: + def __truediv__(self, other: RasterInterface | str | int | float) -> Operation: """Division.""" return self._create_operator(Operation, "/", self, other) - def __radd__(self, other: App | str | int | float) -> Operation: + def __radd__(self, other: RasterInterface | str | int | float) -> Operation: """Right addition.""" return self._create_operator(Operation, "+", other, self) - def __rsub__(self, other: App | str | int | float) -> Operation: + def __rsub__(self, other: RasterInterface | str | int | float) -> Operation: """Right subtraction.""" return self._create_operator(Operation, "-", other, self) - def __rmul__(self, other: App | str | int | float) -> Operation: + def __rmul__(self, other: RasterInterface | str | int | float) -> Operation: """Right multiplication.""" return self._create_operator(Operation, "*", other, self) - def __rtruediv__(self, other: App | str | int | float) -> Operation: + def __rtruediv__(self, other: RasterInterface | str | int | float) -> Operation: """Right division.""" return self._create_operator(Operation, "/", other, self) @@ -252,35 +252,35 @@ class RasterInterface(ABC): """Absolute value.""" return Operation("abs", self) - def __ge__(self, other: App | str | int | float) -> LogicalOperation: + def __ge__(self, other: RasterInterface | str | int | float) -> LogicalOperation: """Greater of equal than.""" return self._create_operator(LogicalOperation, ">=", self, other) - def __le__(self, other: App | str | int | float) -> LogicalOperation: + def __le__(self, other: RasterInterface | str | int | float) -> LogicalOperation: """Lower of equal than.""" return self._create_operator(LogicalOperation, "<=", self, other) - def __gt__(self, other: App | str | int | float) -> LogicalOperation: + def __gt__(self, other: RasterInterface | str | int | float) -> LogicalOperation: """Greater than.""" return self._create_operator(LogicalOperation, ">", self, other) - def __lt__(self, other: App | str | int | float) -> LogicalOperation: + def __lt__(self, other: RasterInterface | str | int | float) -> LogicalOperation: """Lower than.""" return self._create_operator(LogicalOperation, "<", self, other) - def __eq__(self, other: App | str | int | float) -> LogicalOperation: + def __eq__(self, other: RasterInterface | str | int | float) -> LogicalOperation: """Equality.""" return self._create_operator(LogicalOperation, "==", self, other) - def __ne__(self, other: App | str | int | float) -> LogicalOperation: + def __ne__(self, other: RasterInterface | str | int | float) -> LogicalOperation: """Inequality.""" return self._create_operator(LogicalOperation, "!=", self, other) - def __or__(self, other: App | str | int | float) -> LogicalOperation: + def __or__(self, other: RasterInterface | str | int | float) -> LogicalOperation: """Logical or.""" return self._create_operator(LogicalOperation, "||", self, other) - def __and__(self, other: App | str | int | float) -> LogicalOperation: + def __and__(self, other: RasterInterface | str | int | float) -> LogicalOperation: """Logical and.""" return self._create_operator(LogicalOperation, "&&", self, other) -- GitLab From 0dbe151708b3a38731f256a0a803b3b2c60f0af1 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 3 Feb 2023 11:19:11 +0100 Subject: [PATCH 096/399] REFAC: self.logical_fake_exp_bands only in LogicalOperation --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 1781d18..6b7d58c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -856,7 +856,6 @@ class Operation(App): self.inputs = [] self.nb_channels = {} self.fake_exp_bands = [] - self.logical_fake_exp_bands = [] self.build_fake_expressions(operator, inputs, nb_bands=nb_bands) # Transforming images to the adequate im#, e.g. `input1` to "im1" # creating a dictionary that is like {str(input1): 'im1', 'image2.tif': 'im2', ...}. @@ -1033,6 +1032,7 @@ class LogicalOperation(Operation): nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ + self.logical_fake_exp_bands = [] super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation") self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands) -- GitLab From c7d2a0370411b05ba799ec3ef650290cfee31a9a Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 3 Feb 2023 13:19:57 +0100 Subject: [PATCH 097/399] REFAC: rename attrs/funcs --- pyotb/core.py | 112 +++++++++++++++++++++----------------------- tests/test_core.py | 6 +-- tests/test_numpy.py | 4 +- 3 files changed, 59 insertions(+), 63 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 6b7d58c..daa4082 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -14,7 +14,7 @@ import otbApplication as otb # pylint: disable=import-error from .helpers import logger -class RasterInterface(ABC): +class ImageObject(ABC): """Abstraction of an image object.""" app: otb.Application @@ -22,7 +22,7 @@ class RasterInterface(ABC): @property @abstractmethod - def key_output_image(self): + def output_image_key(self): """Returns the name of a parameter associated to an image. Property defined in App and Output.""" @abstractmethod @@ -32,7 +32,7 @@ class RasterInterface(ABC): @property def metadata(self) -> dict[str, (str, float, list[float])]: """Return first output image metadata dictionary.""" - return dict(self.app.GetMetadataDictionary(self.key_output_image)) + return dict(self.app.GetMetadataDictionary(self.output_image_key)) @property def dtype(self) -> np.dtype: @@ -42,7 +42,7 @@ class RasterInterface(ABC): dtype: pixel type of the output image """ - enum = self.app.GetParameterOutputImagePixelType(self.key_output_image) + enum = self.app.GetParameterOutputImagePixelType(self.output_image_key) return self.app.ConvertPixelTypeToNumpy(enum) @property @@ -53,8 +53,8 @@ class RasterInterface(ABC): shape: (height, width, bands) """ - width, height = self.app.GetImageSize(self.key_output_image) - bands = self.app.GetImageNbBands(self.key_output_image) + width, height = self.app.GetImageSize(self.output_image_key) + bands = self.app.GetImageNbBands(self.output_image_key) return height, width, bands @property @@ -64,13 +64,13 @@ class RasterInterface(ABC): Returns: transform: (X spacing, X offset, X origin, Y offset, Y spacing, Y origin) """ - spacing_x, spacing_y = self.app.GetImageSpacing(self.key_output_image) - origin_x, origin_y = self.app.GetImageOrigin(self.key_output_image) + spacing_x, spacing_y = self.app.GetImageSpacing(self.output_image_key) + origin_x, origin_y = self.app.GetImageOrigin(self.output_image_key) # Shift image origin since OTB is giving coordinates of pixel center instead of corners origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2 return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y - def get_infos(self) -> dict[str, (str, float, list[float])]: + def get_info(self) -> dict[str, (str, float, list[float])]: """Return a dict output of ReadImageInfo for the first image output.""" return App("ReadImageInfo", self, quiet=True).data @@ -78,7 +78,7 @@ class RasterInterface(ABC): """Return a dict output of ComputeImagesStatistics for the first image output.""" return App("ComputeImagesStatistics", self, quiet=True).data - def read_values_at_coords(self, row: int, col: int, bands: int = None) -> list[int | float] | int | float: + def get_values_at_coords(self, row: int, col: int, bands: int = None) -> list[int | float] | int | float: """Get pixel value(s) at a given YX coordinates. Args: @@ -106,9 +106,7 @@ class RasterInterface(ABC): app.set_parameters({"cl": [f"Channel{n + 1}" for n in channels]}) app.execute() data = literal_eval(app.app.GetParameterString("value")) - if len(channels) == 1: - return data[0] - return data + return data[0] if len(channels) == 1 else data def channels_list_from_slice(self, bands: int) -> list[int]: """Get list of channels to read values at, from a slice.""" @@ -140,7 +138,7 @@ class RasterInterface(ABC): """ if key is None: - key = self.key_output_image + key = self.output_image_key if key not in self.exports_dic: self.exports_dic[key] = self.app.ExportImage(key) if preserve_dtype: @@ -163,9 +161,7 @@ class RasterInterface(ABC): """ data = self.export(key, preserve_dtype) array = data["array"] - if copy: - return array.copy() - return array + return array.copy() if copy else array def to_rasterio(self) -> tuple[np.ndarray, dict[str, Any]]: """Export image as a numpy array and its metadata compatible with rasterio. @@ -177,14 +173,14 @@ class RasterInterface(ABC): """ array = self.to_numpy(preserve_dtype=True, copy=False) height, width, count = array.shape - proj = self.app.GetImageProjection(self.key_output_image) + proj = self.app.GetImageProjection(self.output_image_key) profile = { 'crs': proj, 'dtype': array.dtype, 'transform': self.transform, 'count': count, 'height': height, 'width': width, } return np.moveaxis(array, 2, 0), profile - def xy_to_rowcol(self, x: float, y: float) -> tuple[int, int]: + def get_rowcol_from_xy(self, x: float, y: float) -> tuple[int, int]: """Find (row, col) index using (x, y) projected coordinates - image CRS is expected. Args: @@ -216,35 +212,35 @@ class RasterInterface(ABC): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return op_cls(name, x, y) - def __add__(self, other: RasterInterface | str | int | float) -> Operation: + def __add__(self, other: ImageObject | str | int | float) -> Operation: """Addition.""" return self._create_operator(Operation, "+", self, other) - def __sub__(self, other: RasterInterface | str | int | float) -> Operation: + def __sub__(self, other: ImageObject | str | int | float) -> Operation: """Subtraction.""" return self._create_operator(Operation, "-", self, other) - def __mul__(self, other: RasterInterface | str | int | float) -> Operation: + def __mul__(self, other: ImageObject | str | int | float) -> Operation: """Multiplication.""" return self._create_operator(Operation, "*", self, other) - def __truediv__(self, other: RasterInterface | str | int | float) -> Operation: + def __truediv__(self, other: ImageObject | str | int | float) -> Operation: """Division.""" return self._create_operator(Operation, "/", self, other) - def __radd__(self, other: RasterInterface | str | int | float) -> Operation: + def __radd__(self, other: ImageObject | str | int | float) -> Operation: """Right addition.""" return self._create_operator(Operation, "+", other, self) - def __rsub__(self, other: RasterInterface | str | int | float) -> Operation: + def __rsub__(self, other: ImageObject | str | int | float) -> Operation: """Right subtraction.""" return self._create_operator(Operation, "-", other, self) - def __rmul__(self, other: RasterInterface | str | int | float) -> Operation: + def __rmul__(self, other: ImageObject | str | int | float) -> Operation: """Right multiplication.""" return self._create_operator(Operation, "*", other, self) - def __rtruediv__(self, other: RasterInterface | str | int | float) -> Operation: + def __rtruediv__(self, other: ImageObject | str | int | float) -> Operation: """Right division.""" return self._create_operator(Operation, "/", other, self) @@ -252,35 +248,35 @@ class RasterInterface(ABC): """Absolute value.""" return Operation("abs", self) - def __ge__(self, other: RasterInterface | str | int | float) -> LogicalOperation: + def __ge__(self, other: ImageObject | str | int | float) -> LogicalOperation: """Greater of equal than.""" return self._create_operator(LogicalOperation, ">=", self, other) - def __le__(self, other: RasterInterface | str | int | float) -> LogicalOperation: + def __le__(self, other: ImageObject | str | int | float) -> LogicalOperation: """Lower of equal than.""" return self._create_operator(LogicalOperation, "<=", self, other) - def __gt__(self, other: RasterInterface | str | int | float) -> LogicalOperation: + def __gt__(self, other: ImageObject | str | int | float) -> LogicalOperation: """Greater than.""" return self._create_operator(LogicalOperation, ">", self, other) - def __lt__(self, other: RasterInterface | str | int | float) -> LogicalOperation: + def __lt__(self, other: ImageObject | str | int | float) -> LogicalOperation: """Lower than.""" return self._create_operator(LogicalOperation, "<", self, other) - def __eq__(self, other: RasterInterface | str | int | float) -> LogicalOperation: + def __eq__(self, other: ImageObject | str | int | float) -> LogicalOperation: """Equality.""" return self._create_operator(LogicalOperation, "==", self, other) - def __ne__(self, other: RasterInterface | str | int | float) -> LogicalOperation: + def __ne__(self, other: ImageObject | str | int | float) -> LogicalOperation: """Inequality.""" return self._create_operator(LogicalOperation, "!=", self, other) - def __or__(self, other: RasterInterface | str | int | float) -> LogicalOperation: + def __or__(self, other: ImageObject | str | int | float) -> LogicalOperation: """Logical or.""" return self._create_operator(LogicalOperation, "||", self, other) - def __and__(self, other: RasterInterface | str | int | float) -> LogicalOperation: + def __and__(self, other: ImageObject | str | int | float) -> LogicalOperation: """Logical and.""" return self._create_operator(LogicalOperation, "&&", self, other) @@ -322,7 +318,7 @@ class RasterInterface(ABC): elif isinstance(inp, App): if not inp.exports_dic: inp.export() - image_dic = inp.exports_dic[inp.key_output_image] + image_dic = inp.exports_dic[inp.output_image_key] array = image_dic["array"] arrays.append(array) else: @@ -343,7 +339,7 @@ class RasterInterface(ABC): return NotImplemented -class App(RasterInterface): +class App(ImageObject): """Base class that gathers common operations for any OTB application.""" def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, **kwargs): @@ -408,7 +404,7 @@ class App(RasterInterface): return self.get_first_key(param_types=[otb.ParameterType_InputImage, otb.ParameterType_InputImageList]) @property - def key_output_image(self) -> str: + def output_image_key(self) -> str: """Get the name of first output image parameter.""" return self.get_first_key(param_types=[otb.ParameterType_OutputImage]) @@ -443,7 +439,7 @@ class App(RasterInterface): Args: *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") - - string or RasterInterface, useful when the user implicitly wants to set the param "in" + - string or ImageObject, useful when the user implicitly wants to set the param "in" - list, useful when the user implicitly wants to set the param "il" **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif' @@ -582,8 +578,8 @@ class App(RasterInterface): kwargs.update(arg) elif isinstance(arg, str) and kwargs: logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg) - elif isinstance(arg, (str, Path)) and self.key_output_image: - kwargs.update({self.key_output_image: str(arg)}) + elif isinstance(arg, (str, Path)) and self.output_image_key: + kwargs.update({self.output_image_key: str(arg)}) # Append filename extension to filenames if filename_extension: @@ -641,15 +637,15 @@ class App(RasterInterface): """ parameters = self.parameters.copy() for key, param in parameters.items(): - # In the following, we replace each parameter which is an RasterInterface, with its summary. - if isinstance(param, RasterInterface): # single parameter + # In the following, we replace each parameter which is an ImageObject, with its summary. + if isinstance(param, ImageObject): # single parameter parameters[key] = param.summarize() elif isinstance(param, list): # parameter list - parameters[key] = [p.summarize() if isinstance(p, RasterInterface) else p for p in param] + parameters[key] = [p.summarize() if isinstance(p, ImageObject) else p for p in param] return {"name": self.app.GetName(), "parameters": parameters} # Private functions - def __parse_args(self, args: list[str | RasterInterface | dict | list]) -> dict[str, Any]: + def __parse_args(self, args: list[str | ImageObject | dict | list]) -> dict[str, Any]: """Gather all input arguments in kwargs dict. Args: @@ -663,18 +659,18 @@ class App(RasterInterface): for arg in args: if isinstance(arg, dict): kwargs.update(arg) - elif isinstance(arg, (str, RasterInterface)) or isinstance(arg, list) and is_key_list(self, self.key_input): + elif isinstance(arg, (str, ImageObject)) or isinstance(arg, list) and is_key_list(self, self.key_input): kwargs.update({self.key_input: arg}) return kwargs - def __set_param(self, key: str, obj: list | tuple | RasterInterface | otb.Application | list[Any]): + def __set_param(self, key: str, obj: list | tuple | ImageObject | otb.Application | list[Any]): """Set one parameter, decide which otb.Application method to use depending on target object.""" if obj is None or (isinstance(obj, (list, tuple)) and not obj): self.app.ClearValue(key) return # Single-parameter cases - if isinstance(obj, RasterInterface): - self.app.ConnectImage(key, obj.app, obj.key_output_image) + if isinstance(obj, ImageObject): + self.app.ConnectImage(key, obj.app, obj.output_image_key) elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0]) elif key == "ram": # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200 @@ -685,8 +681,8 @@ class App(RasterInterface): elif is_key_images_list(self, key): # To enable possible in-memory connections, we go through the list and set the parameters one by one for inp in obj: - if isinstance(inp, RasterInterface): - self.app.ConnectImage(key, inp.app, inp.key_output_image) + if isinstance(inp, ImageObject): + self.app.ConnectImage(key, inp.app, inp.output_image_key) elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0]) else: # here `input` should be an image filepath @@ -735,7 +731,7 @@ class App(RasterInterface): channels = None if len(key) == 3: channels = key[2] - return self.read_values_at_coords(row, col, channels) + return self.get_values_at_coords(row, col, channels) # Slicing if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)): raise ValueError(f'"{key}"cannot be interpreted as valid slicing. Slicing should be 2D or 3D.') @@ -1098,7 +1094,7 @@ class Input(App): return f"<pyotb.Input object from {self.filepath}>" -class Output(RasterInterface): +class Output(ImageObject): """Object that behave like a pointer to a specific application output file.""" def __init__(self, pyotb_app: App, param_key: str = None, filepath: str = None, mkdir: bool = True): @@ -1125,8 +1121,8 @@ class Output(RasterInterface): self.make_parent_dirs() @property - def key_output_image(self) -> str: - """Force the right key to be used when accessing the RasterInterface.""" + def output_image_key(self) -> str: + """Force the right key to be used when accessing the ImageObject.""" return self.param_key def exists(self) -> bool: @@ -1144,8 +1140,8 @@ class Output(RasterInterface): def write(self, filepath: None | str | Path = None, **kwargs): """Write output to disk, filepath is not required if it was provided to parent App during init.""" if filepath is None and self.filepath: - return self.parent_pyotb_app.write({self.key_output_image: self.filepath}, **kwargs) - return self.parent_pyotb_app.write({self.key_output_image: filepath}, **kwargs) + return self.parent_pyotb_app.write({self.output_image_key: self.filepath}, **kwargs) + return self.parent_pyotb_app.write({self.output_image_key: filepath}, **kwargs) def __str__(self) -> str: """Return a nice string representation with source app name and object id.""" @@ -1208,7 +1204,7 @@ def get_pixel_type(inp: str | App) -> str: raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}") pixel_type = getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[datatype]}") elif isinstance(inp, App): - pixel_type = inp.app.GetParameterOutputImagePixelType(inp.key_output_image) + pixel_type = inp.app.GetParameterOutputImagePixelType(inp.output_image_key) else: raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}") return pixel_type diff --git a/tests/test_core.py b/tests/test_core.py index 75dca37..58ddeac 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -27,7 +27,7 @@ def test_key_input(): def test_key_output(): - assert INPUT.key_output_image == "out" + assert INPUT.output_image_key == "out" def test_dtype(): @@ -62,7 +62,7 @@ def test_elapsed_time(): # Other functions def test_get_infos(): - infos = INPUT.get_infos() + infos = INPUT.get_info() assert (infos["sizex"], infos["sizey"]) == (251, 304) @@ -71,7 +71,7 @@ def test_get_statistics(): def test_xy_to_rowcol(): - assert INPUT.xy_to_rowcol(760101, 6945977) == (19, 7) + assert INPUT.get_rowcol_from_xy(760101, 6945977) == (19, 7) def test_write(): diff --git a/tests/test_numpy.py b/tests/test_numpy.py index d36dfb7..e62ffbb 100644 --- a/tests/test_numpy.py +++ b/tests/test_numpy.py @@ -5,7 +5,7 @@ from tests_data import INPUT def test_export(): INPUT.export() - array = INPUT.exports_dic[INPUT.key_output_image]["array"] + array = INPUT.exports_dic[INPUT.output_image_key]["array"] assert isinstance(array, np.ndarray) assert array.dtype == "uint8" del INPUT.exports_dic["out"] @@ -13,7 +13,7 @@ def test_export(): def test_output_export(): INPUT.out.export() - assert INPUT.out.key_output_image in INPUT.out.exports_dic + assert INPUT.out.output_image_key in INPUT.out.exports_dic def test_to_numpy(): -- GitLab From 97159e767a4b5aa7afe6ece1ddd56d9caea0379b Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 3 Feb 2023 13:52:05 +0100 Subject: [PATCH 098/399] ADD: version --> 2.0.0 --- pyotb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/__init__.py b/pyotb/__init__.py index ac6264c..8e22979 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """This module provides convenient python wrapping of otbApplications.""" -__version__ = "1.6.0" +__version__ = "2.0.0" from .helpers import logger, set_logger_level from .apps import * -- GitLab From 824a87d4c55cb5677f49c788ec7496ea1ffb815c Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 3 Feb 2023 11:42:10 +0100 Subject: [PATCH 099/399] REFAC: Use RasterInterface when possible --- pyotb/core.py | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index daa4082..8239e3a 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -315,7 +315,7 @@ class ImageObject(ABC): for inp in inputs: if isinstance(inp, (float, int, np.ndarray, np.generic)): arrays.append(inp) - elif isinstance(inp, App): + elif isinstance(inp, RasterInterface): if not inp.exports_dic: inp.export() image_dic = inp.exports_dic[inp.output_image_key] @@ -841,7 +841,7 @@ class Operation(App): Args: operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? - *inputs: inputs. Can be App, filepath, int or float + *inputs: inputs. Can be RasterInterface, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where name: override the Operation name @@ -873,14 +873,15 @@ class Operation(App): super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True) self.name = f'Operation exp="{self.exp}"' - def build_fake_expressions(self, operator: str, inputs: list[App | str | int | float], nb_bands: int = None): + def build_fake_expressions(self, operator: str, inputs: list[RasterInterface | str | int | float], + nb_bands: int = None): """Create a list of 'fake' expressions, one for each band. E.g for the operation input1 + input2, we create a fake expression that is like "str(input1) + str(input2)" Args: operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? - inputs: inputs. Can be App, filepath, int or float + inputs: inputs. Can be RasterInterface, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ @@ -955,7 +956,8 @@ class Operation(App): return exp_bands, ";".join(exp_bands) @staticmethod - def make_fake_exp(x: App | str, band: int, keep_logical: bool = False) -> tuple[str, list[App], int]: + def make_fake_exp(x: RasterInterface | str, band: int, keep_logical: bool = False) \ + -> tuple[str, list[RasterInterface], int]: """This an internal function, only to be used by `build_fake_expressions`. Enable to create a fake expression just for one input and one band. @@ -1032,7 +1034,7 @@ class LogicalOperation(Operation): super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation") self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands) - def build_fake_expressions(self, operator: str, inputs: list[App | str | int | float], nb_bands: int = None): + def build_fake_expressions(self, operator: str, inputs: list[RasterInterface | str | int | float], nb_bands: int = None): """Create a list of 'fake' expressions, one for each band. e.g for the operation input1 > input2, we create a fake expression that is like @@ -1040,7 +1042,7 @@ class LogicalOperation(Operation): Args: operator: str (one of >, <, >=, <=, ==, !=, &, |) - inputs: Can be App, filepath, int or float + inputs: Can be RasterInterface, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ @@ -1148,29 +1150,29 @@ class Output(ImageObject): return f"<pyotb.Output {self.name} object, id {id(self)}>" -def get_nbchannels(inp: str | App) -> int: +def get_nbchannels(inp: str | RasterInterface) -> int: """Get the nb of bands of input image. Args: - inp: can be filepath or pyotb object + inp: can be filepath or RasterInterface object Returns: number of bands in image """ - if isinstance(inp, App): + if isinstance(inp, RasterInterface): nb_channels = inp.shape[-1] else: # Executing the app, without printing its log try: - info = App("ReadImageInfo", inp, quiet=True) + info = RasterInterface("ReadImageInfo", inp, quiet=True) nb_channels = info.app.GetParameterInt("numberbands") except Exception as e: # this happens when we pass a str that is not a filepath raise TypeError(f"Could not get the number of channels of '{inp}'. Not a filepath or wrong filepath") from e return nb_channels -def get_pixel_type(inp: str | App) -> str: +def get_pixel_type(inp: str | RasterInterface) -> str: """Get the encoding of input image pixels. Args: @@ -1178,7 +1180,7 @@ def get_pixel_type(inp: str | App) -> str: Returns: pixel_type: OTB enum e.g. `otbApplication.ImagePixelType_uint8', which actually is an int. - For an App with several outputs, only the pixel type of the first output is returned + For an RasterInterface with several outputs, only the pixel type of the first output is returned """ if isinstance(inp, str): @@ -1203,8 +1205,8 @@ def get_pixel_type(inp: str | App) -> str: if datatype not in datatype_to_pixeltype: raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}") pixel_type = getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[datatype]}") - elif isinstance(inp, App): - pixel_type = inp.app.GetParameterOutputImagePixelType(inp.output_image_key) + elif isinstance(inp, RasterInterface): + pixel_type = inp.app.GetParameterOutputImagePixelType(inp.key_output_image) else: raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}") return pixel_type @@ -1227,8 +1229,8 @@ def parse_pixel_type(pixel_type: str | int) -> int: raise ValueError(f"Bad pixel type specification ({pixel_type})") -def is_key_list(pyotb_app: App, key: str) -> bool: - """Check if a key of the App is an input parameter list.""" +def is_key_list(pyotb_app: RasterInterface, key: str) -> bool: + """Check if a key of the RasterInterface is an input parameter list.""" types = ( otb.ParameterType_InputImageList, otb.ParameterType_StringList, @@ -1239,12 +1241,12 @@ def is_key_list(pyotb_app: App, key: str) -> bool: return pyotb_app.app.GetParameterType(key) in types -def is_key_images_list(pyotb_app: App, key: str) -> bool: - """Check if a key of the App is an input parameter image list.""" +def is_key_images_list(pyotb_app: RasterInterface, key: str) -> bool: + """Check if a key of the RasterInterface is an input parameter image list.""" types = (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList) return pyotb_app.app.GetParameterType(key) in types -def get_out_images_param_keys(app: App) -> list[str]: +def get_out_images_param_keys(app: RasterInterface) -> list[str]: """Return every output parameter keys of an OTB app.""" return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage] -- GitLab From 89158d2ebf754153da2d8b2538071fa84c851290 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 3 Feb 2023 11:47:39 +0100 Subject: [PATCH 100/399] REFAC: Use RasterInterface when possible --- pyotb/core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 8239e3a..81fb51c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1034,7 +1034,8 @@ class LogicalOperation(Operation): super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation") self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands) - def build_fake_expressions(self, operator: str, inputs: list[RasterInterface | str | int | float], nb_bands: int = None): + def build_fake_expressions(self, operator: str, inputs: list[RasterInterface | str | int | float], + nb_bands: int = None): """Create a list of 'fake' expressions, one for each band. e.g for the operation input1 > input2, we create a fake expression that is like @@ -1165,7 +1166,7 @@ def get_nbchannels(inp: str | RasterInterface) -> int: else: # Executing the app, without printing its log try: - info = RasterInterface("ReadImageInfo", inp, quiet=True) + info = App("ReadImageInfo", inp, quiet=True) nb_channels = info.app.GetParameterInt("numberbands") except Exception as e: # this happens when we pass a str that is not a filepath raise TypeError(f"Could not get the number of channels of '{inp}'. Not a filepath or wrong filepath") from e -- GitLab From 847f0fadd11648e1eee5d4d8e7d6f1944b853621 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 3 Feb 2023 13:48:28 +0100 Subject: [PATCH 101/399] FIX: sync with renamed attrs/cls --- pyotb/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 81fb51c..594fd1f 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1206,8 +1206,8 @@ def get_pixel_type(inp: str | RasterInterface) -> str: if datatype not in datatype_to_pixeltype: raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}") pixel_type = getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[datatype]}") - elif isinstance(inp, RasterInterface): - pixel_type = inp.app.GetParameterOutputImagePixelType(inp.key_output_image) + elif isinstance(inp, ImageObject): + pixel_type = inp.app.GetParameterOutputImagePixelType(inp.output_image_key) else: raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}") return pixel_type -- GitLab From 249a8a3958613f0a6bdaa24bd38d6ee7a7e3a808 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Feb 2023 19:35:33 +0100 Subject: [PATCH 102/399] FIX: merge conflicts --- pyotb/core.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 594fd1f..3705961 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -315,7 +315,7 @@ class ImageObject(ABC): for inp in inputs: if isinstance(inp, (float, int, np.ndarray, np.generic)): arrays.append(inp) - elif isinstance(inp, RasterInterface): + elif isinstance(inp, ImageObject): if not inp.exports_dic: inp.export() image_dic = inp.exports_dic[inp.output_image_key] @@ -841,7 +841,7 @@ class Operation(App): Args: operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? - *inputs: inputs. Can be RasterInterface, filepath, int or float + *inputs: inputs. Can be ImageObject, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where name: override the Operation name @@ -873,7 +873,7 @@ class Operation(App): super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True) self.name = f'Operation exp="{self.exp}"' - def build_fake_expressions(self, operator: str, inputs: list[RasterInterface | str | int | float], + def build_fake_expressions(self, operator: str, inputs: list[ImageObject | str | int | float], nb_bands: int = None): """Create a list of 'fake' expressions, one for each band. @@ -881,7 +881,7 @@ class Operation(App): Args: operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? - inputs: inputs. Can be RasterInterface, filepath, int or float + inputs: inputs. Can be ImageObject, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ @@ -956,8 +956,8 @@ class Operation(App): return exp_bands, ";".join(exp_bands) @staticmethod - def make_fake_exp(x: RasterInterface | str, band: int, keep_logical: bool = False) \ - -> tuple[str, list[RasterInterface], int]: + def make_fake_exp(x: ImageObject | str, band: int, keep_logical: bool = False) \ + -> tuple[str, list[ImageObject], int]: """This an internal function, only to be used by `build_fake_expressions`. Enable to create a fake expression just for one input and one band. @@ -1034,7 +1034,7 @@ class LogicalOperation(Operation): super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation") self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands) - def build_fake_expressions(self, operator: str, inputs: list[RasterInterface | str | int | float], + def build_fake_expressions(self, operator: str, inputs: list[ImageObject | str | int | float], nb_bands: int = None): """Create a list of 'fake' expressions, one for each band. @@ -1043,7 +1043,7 @@ class LogicalOperation(Operation): Args: operator: str (one of >, <, >=, <=, ==, !=, &, |) - inputs: Can be RasterInterface, filepath, int or float + inputs: Can be ImageObject, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ @@ -1151,17 +1151,17 @@ class Output(ImageObject): return f"<pyotb.Output {self.name} object, id {id(self)}>" -def get_nbchannels(inp: str | RasterInterface) -> int: +def get_nbchannels(inp: str | ImageObject) -> int: """Get the nb of bands of input image. Args: - inp: can be filepath or RasterInterface object + inp: can be filepath or ImageObject object Returns: number of bands in image """ - if isinstance(inp, RasterInterface): + if isinstance(inp, ImageObject): nb_channels = inp.shape[-1] else: # Executing the app, without printing its log @@ -1173,7 +1173,7 @@ def get_nbchannels(inp: str | RasterInterface) -> int: return nb_channels -def get_pixel_type(inp: str | RasterInterface) -> str: +def get_pixel_type(inp: str | ImageObject) -> str: """Get the encoding of input image pixels. Args: @@ -1181,7 +1181,7 @@ def get_pixel_type(inp: str | RasterInterface) -> str: Returns: pixel_type: OTB enum e.g. `otbApplication.ImagePixelType_uint8', which actually is an int. - For an RasterInterface with several outputs, only the pixel type of the first output is returned + For an ImageObject with several outputs, only the pixel type of the first output is returned """ if isinstance(inp, str): @@ -1230,8 +1230,8 @@ def parse_pixel_type(pixel_type: str | int) -> int: raise ValueError(f"Bad pixel type specification ({pixel_type})") -def is_key_list(pyotb_app: RasterInterface, key: str) -> bool: - """Check if a key of the RasterInterface is an input parameter list.""" +def is_key_list(pyotb_app: ImageObject, key: str) -> bool: + """Check if a key of the ImageObject is an input parameter list.""" types = ( otb.ParameterType_InputImageList, otb.ParameterType_StringList, @@ -1242,12 +1242,12 @@ def is_key_list(pyotb_app: RasterInterface, key: str) -> bool: return pyotb_app.app.GetParameterType(key) in types -def is_key_images_list(pyotb_app: RasterInterface, key: str) -> bool: - """Check if a key of the RasterInterface is an input parameter image list.""" +def is_key_images_list(pyotb_app: ImageObject, key: str) -> bool: + """Check if a key of the ImageObject is an input parameter image list.""" types = (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList) return pyotb_app.app.GetParameterType(key) in types -def get_out_images_param_keys(app: RasterInterface) -> list[str]: +def get_out_images_param_keys(app: ImageObject) -> list[str]: """Return every output parameter keys of an OTB app.""" return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage] -- GitLab From 4c3c72961a4f3cc53acba9fde70779c1b6b8ab00 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 2 Feb 2023 21:14:51 +0100 Subject: [PATCH 103/399] FIX: summarize() for RasterInterface --- pyotb/core.py | 51 ++++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 3705961..bdf4eb5 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -338,6 +338,22 @@ class ImageObject(ABC): return pyotb_app return NotImplemented + def summarize(self) -> dict[str, str | dict[str, Any]]: + """Serialize an object and its pipeline into a dictionary. + + Returns: + nested dictionary summarizing the pipeline + + """ + parameters = self.parameters.copy() + for key, param in parameters.items(): + # In the following, we replace each parameter which is an ImageObject, with its summary. + if isinstance(param, ImageObject): # single parameter + parameters[key] = param.summarize() + elif isinstance(param, list): # parameter list + parameters[key] = [p.summarize() if isinstance(p, ImageObject) else p for p in param] + return {"name": self.app.GetName(), "parameters": parameters} + class App(ImageObject): """Base class that gathers common operations for any OTB application.""" @@ -465,7 +481,7 @@ class App(ImageObject): except (RuntimeError, TypeError, ValueError, KeyError) as e: raise RuntimeError( f"{self.name}: something went wrong before execution " - f"(while setting parameter '{key}' to '{obj}')" + f"(while setting parameter '{key}' to '{obj}': {e})" ) from e # Update _parameters using values from OtbApplication object otb_params = self.app.GetParameters().items() @@ -536,7 +552,7 @@ class App(ImageObject): try: self.app.Execute() except (RuntimeError, FileNotFoundError) as e: - raise RuntimeError(f"{self.name}: error during during app execution") from e + raise RuntimeError(f"{self.name}: error during during app execution ({e}") from e self.frozen = False self._time_end = perf_counter() logger.debug("%s: execution ended", self.name) @@ -628,22 +644,6 @@ class App(ImageObject): logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) return tuple(files) - def summarize(self) -> dict[str, str | dict[str, Any]]: - """Serialize an object and its pipeline into a dictionary. - - Returns: - nested dictionary summarizing the pipeline - - """ - parameters = self.parameters.copy() - for key, param in parameters.items(): - # In the following, we replace each parameter which is an ImageObject, with its summary. - if isinstance(param, ImageObject): # single parameter - parameters[key] = param.summarize() - elif isinstance(param, list): # parameter list - parameters[key] = [p.summarize() if isinstance(p, ImageObject) else p for p in param] - return {"name": self.app.GetName(), "parameters": parameters} - # Private functions def __parse_args(self, args: list[str | ImageObject | dict | list]) -> dict[str, Any]: """Gather all input arguments in kwargs dict. @@ -1079,16 +1079,16 @@ class LogicalOperation(Operation): class Input(App): """Class for transforming a filepath to pyOTB object.""" - def __init__(self, path: str): + def __init__(self, filepath: str): """Default constructor. Args: - path: Anything supported by GDAL (local file on the filesystem, remote resource e.g. /vsicurl/.., etc.) + filepath: Anything supported by GDAL (local file on the filesystem, remote resource e.g. /vsicurl/.., etc.) """ - super().__init__("ExtractROI", {"in": path}, frozen=True) - self.name = f"Input from {path}" - self.filepath = Path(path) + super().__init__("ExtractROI", {"in": filepath}, frozen=True) + self.name = f"Input from {filepath}" + self.filepath = Path(filepath) self.propagate_dtype() self.execute() @@ -1112,6 +1112,7 @@ class Output(ImageObject): """ self.name = f"Output {param_key} from {pyotb_app.name}" self.parent_pyotb_app = pyotb_app # keep trace of parent app + self.parameters = pyotb_app.parameters self.app = pyotb_app.app self.exports_dic = pyotb_app.exports_dic self.param_key = param_key @@ -1169,7 +1170,7 @@ def get_nbchannels(inp: str | ImageObject) -> int: info = App("ReadImageInfo", inp, quiet=True) nb_channels = info.app.GetParameterInt("numberbands") except Exception as e: # this happens when we pass a str that is not a filepath - raise TypeError(f"Could not get the number of channels of '{inp}'. Not a filepath or wrong filepath") from e + raise TypeError(f"Could not get the number of channels of '{inp}' ({e})") from e return nb_channels @@ -1188,7 +1189,7 @@ def get_pixel_type(inp: str | ImageObject) -> str: try: info = App("ReadImageInfo", inp, quiet=True) except Exception as info_err: # this happens when we pass a str that is not a filepath - raise TypeError(f"Could not get the pixel type of `{inp}`. Not a filepath or wrong filepath") from info_err + raise TypeError(f"Could not get the pixel type of `{inp}` ({info_err})") from info_err datatype = info.app.GetParameterString("datatype") # which is such as short, float... if not datatype: raise TypeError(f"Unable to read pixel type of image {inp}") -- GitLab From d6de016596440d1facb081b2fa9b8059199456cd Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 2 Feb 2023 22:05:16 +0100 Subject: [PATCH 104/399] ADD: parameters property to RasterInterface --- pyotb/core.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index bdf4eb5..37a56bc 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -354,6 +354,10 @@ class ImageObject(ABC): parameters[key] = [p.summarize() if isinstance(p, ImageObject) else p for p in param] return {"name": self.app.GetName(), "parameters": parameters} + @abstractmethod + def parameters(self): + """Parameters""" + class App(ImageObject): """Base class that gathers common operations for any OTB application.""" @@ -385,7 +389,7 @@ class App(ImageObject): self.image_dic = image_dic self._time_start, self._time_end = 0, 0 self.exports_dic = {} - self.parameters = {} + self._parameters = {} # Initialize app, set parameters and execute if not frozen create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication self.app = create(name) @@ -446,6 +450,10 @@ class App(ImageObject): data_dict[str(key)] = value return data_dict + @property + def parameters(self): + return self._parameters + def set_parameters(self, *args, **kwargs): """Set some parameters of the app. @@ -487,7 +495,7 @@ class App(ImageObject): otb_params = self.app.GetParameters().items() otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params} # Update param dict and save values as object attributes - self.parameters.update({**parameters, **otb_params}) + self._parameters.update({**parameters, **otb_params}) self.save_objects() def propagate_dtype(self, target_key: str = None, dtype: int = None): @@ -1112,7 +1120,6 @@ class Output(ImageObject): """ self.name = f"Output {param_key} from {pyotb_app.name}" self.parent_pyotb_app = pyotb_app # keep trace of parent app - self.parameters = pyotb_app.parameters self.app = pyotb_app.app self.exports_dic = pyotb_app.exports_dic self.param_key = param_key @@ -1129,6 +1136,10 @@ class Output(ImageObject): """Force the right key to be used when accessing the ImageObject.""" return self.param_key + @property + def parameters(self): + return self.parent_pyotb_app.parameters + def exists(self) -> bool: """Check file exist.""" if self.filepath is None: -- GitLab From ff71b801656f56db1f7976f34a98f066c7c0f71d Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 2 Feb 2023 22:15:09 +0100 Subject: [PATCH 105/399] ADD: new test to prevent bug --- tests/test_core.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index 58ddeac..7bed905 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -166,3 +166,14 @@ def test_ndvi_comparison(): thresholded_bandmath = pyotb.where(ndvi_bandmath >= 0.3, 1, 0) assert thresholded_bandmath.exp == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)" + + +def test_output_in_arg(): + o = pyotb.Output(INPUT, "out") + t = pyotb.ReadImageInfo(o) + assert t + + +def test_output_summary(): + o = pyotb.Output(INPUT, "out") + assert o.summarize() -- GitLab From c329fc1bed3a92f25132ada7202e7d07d3d45e81 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 2 Feb 2023 22:15:23 +0100 Subject: [PATCH 106/399] STYLE: linter --- pyotb/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 37a56bc..b6e6228 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -356,7 +356,7 @@ class ImageObject(ABC): @abstractmethod def parameters(self): - """Parameters""" + """Parameters.""" class App(ImageObject): @@ -452,6 +452,7 @@ class App(ImageObject): @property def parameters(self): + """Parameters.""" return self._parameters def set_parameters(self, *args, **kwargs): @@ -1138,6 +1139,7 @@ class Output(ImageObject): @property def parameters(self): + """Parameters.""" return self.parent_pyotb_app.parameters def exists(self) -> bool: -- GitLab From 83c1d7a0db7b6aafbafe6421e5c389bf058f08bb Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 2 Feb 2023 22:16:53 +0100 Subject: [PATCH 107/399] REFAC: parameters abstract property in RasterInterface --- pyotb/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyotb/core.py b/pyotb/core.py index b6e6228..3a969f1 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -355,6 +355,7 @@ class ImageObject(ABC): return {"name": self.app.GetName(), "parameters": parameters} @abstractmethod + @property def parameters(self): """Parameters.""" -- GitLab From 3ab4fe2c0dc09e8b35fd7cb7e45394f4e4c71a63 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 2 Feb 2023 22:26:09 +0100 Subject: [PATCH 108/399] REFAC: parameters abstract property in RasterInterface --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 3a969f1..e363caf 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -354,8 +354,8 @@ class ImageObject(ABC): parameters[key] = [p.summarize() if isinstance(p, ImageObject) else p for p in param] return {"name": self.app.GetName(), "parameters": parameters} - @abstractmethod @property + @abstractmethod def parameters(self): """Parameters.""" -- GitLab From 5083e62f2a89f00b6e8b015916db1b3b6cb6e64f Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 3 Feb 2023 13:11:10 +0100 Subject: [PATCH 109/399] ADD: name in RasterInterface --- pyotb/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyotb/core.py b/pyotb/core.py index e363caf..f035979 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -17,6 +17,7 @@ from .helpers import logger class ImageObject(ABC): """Abstraction of an image object.""" + name: str app: otb.Application exports_dic: dict -- GitLab From 733e032c4b3a2ba31c1a9ee0c338c219fc82a5ae Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 3 Feb 2023 13:35:29 +0100 Subject: [PATCH 110/399] REFAC: add parameters as attribute rather than property --- pyotb/core.py | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index f035979..01f1a91 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -19,6 +19,7 @@ class ImageObject(ABC): name: str app: otb.Application + parameters: dict exports_dic: dict @property @@ -355,10 +356,6 @@ class ImageObject(ABC): parameters[key] = [p.summarize() if isinstance(p, ImageObject) else p for p in param] return {"name": self.app.GetName(), "parameters": parameters} - @property - @abstractmethod - def parameters(self): - """Parameters.""" class App(ImageObject): @@ -391,7 +388,7 @@ class App(ImageObject): self.image_dic = image_dic self._time_start, self._time_end = 0, 0 self.exports_dic = {} - self._parameters = {} + self.parameters = {} # Initialize app, set parameters and execute if not frozen create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication self.app = create(name) @@ -422,7 +419,7 @@ class App(ImageObject): @property def key_input_image(self) -> str: - """Get the name of first input image parameter.""" + """Name of the first input image parameter.""" return self.get_first_key(param_types=[otb.ParameterType_InputImage, otb.ParameterType_InputImageList]) @property @@ -452,11 +449,6 @@ class App(ImageObject): data_dict[str(key)] = value return data_dict - @property - def parameters(self): - """Parameters.""" - return self._parameters - def set_parameters(self, *args, **kwargs): """Set some parameters of the app. @@ -498,7 +490,7 @@ class App(ImageObject): otb_params = self.app.GetParameters().items() otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params} # Update param dict and save values as object attributes - self._parameters.update({**parameters, **otb_params}) + self.parameters.update({**parameters, **otb_params}) self.save_objects() def propagate_dtype(self, target_key: str = None, dtype: int = None): @@ -844,7 +836,7 @@ class Operation(App): """ - def __init__(self, operator: str, *inputs, nb_bands: int = None, name: str = None): + def __init__(self, operator: str, *inputs, nb_bands: int = None): """Given some inputs and an operator, this function enables to transform this into an OTB application. Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator. @@ -1126,6 +1118,7 @@ class Output(ImageObject): self.app = pyotb_app.app self.exports_dic = pyotb_app.exports_dic self.param_key = param_key + self.parameters = self.parent_pyotb_app.parameters self.filepath = None if filepath: if "?" in filepath: @@ -1139,11 +1132,6 @@ class Output(ImageObject): """Force the right key to be used when accessing the ImageObject.""" return self.param_key - @property - def parameters(self): - """Parameters.""" - return self.parent_pyotb_app.parameters - def exists(self) -> bool: """Check file exist.""" if self.filepath is None: -- GitLab From 683b75cfa40b48ed6ef5fd5af68f9008102a68ee Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 3 Feb 2023 13:46:33 +0100 Subject: [PATCH 111/399] FIX: sync with renamed attrs/cls --- pyotb/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 01f1a91..a174e9c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -836,7 +836,7 @@ class Operation(App): """ - def __init__(self, operator: str, *inputs, nb_bands: int = None): + def __init__(self, operator: str, *inputs, nb_bands: int = None, name: str = None): """Given some inputs and an operator, this function enables to transform this into an OTB application. Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator. @@ -874,7 +874,7 @@ class Operation(App): appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX" # Execute app super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True) - self.name = f'Operation exp="{self.exp}"' + self.name = name or f'Operation exp="{self.exp}"' def build_fake_expressions(self, operator: str, inputs: list[ImageObject | str | int | float], nb_bands: int = None): -- GitLab From bb9ce48bf66c7476c4d6d78081803396c462d49a Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Feb 2023 17:32:55 +0000 Subject: [PATCH 112/399] BUG: App init error will raise RuntimeError --- pyotb/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index a174e9c..319dc64 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1172,7 +1172,7 @@ def get_nbchannels(inp: str | ImageObject) -> int: try: info = App("ReadImageInfo", inp, quiet=True) nb_channels = info.app.GetParameterInt("numberbands") - except Exception as e: # this happens when we pass a str that is not a filepath + except RuntimeError as e: # this happens when we pass a str that is not a filepath raise TypeError(f"Could not get the number of channels of '{inp}' ({e})") from e return nb_channels @@ -1191,7 +1191,7 @@ def get_pixel_type(inp: str | ImageObject) -> str: if isinstance(inp, str): try: info = App("ReadImageInfo", inp, quiet=True) - except Exception as info_err: # this happens when we pass a str that is not a filepath + except RuntimeError as info_err: # this happens when we pass a str that is not a filepath raise TypeError(f"Could not get the pixel type of `{inp}` ({info_err})") from info_err datatype = info.app.GetParameterString("datatype") # which is such as short, float... if not datatype: -- GitLab From 819c3bf681abd096872f16f7803c9c5a92a4a5cb Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Feb 2023 17:34:53 +0000 Subject: [PATCH 113/399] ENH: rename error e to info_err --- pyotb/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 319dc64..b9fa376 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1172,8 +1172,8 @@ def get_nbchannels(inp: str | ImageObject) -> int: try: info = App("ReadImageInfo", inp, quiet=True) nb_channels = info.app.GetParameterInt("numberbands") - except RuntimeError as e: # this happens when we pass a str that is not a filepath - raise TypeError(f"Could not get the number of channels of '{inp}' ({e})") from e + except RuntimeError as info_err: # this happens when we pass a str that is not a filepath + raise TypeError(f"Could not get the number of channels of '{inp}' ({info_err})") from info_err return nb_channels -- GitLab From c9b25997804ef721cff3f05b7353500c23516ca3 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Feb 2023 20:09:32 +0000 Subject: [PATCH 114/399] Back to OTBObject --- pyotb/core.py | 97 +++++++++++++++++++++++++-------------------------- 1 file changed, 48 insertions(+), 49 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index b9fa376..a1bf285 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -14,7 +14,7 @@ import otbApplication as otb # pylint: disable=import-error from .helpers import logger -class ImageObject(ABC): +class OTBObject(ABC): """Abstraction of an image object.""" name: str @@ -214,35 +214,35 @@ class ImageObject(ABC): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return op_cls(name, x, y) - def __add__(self, other: ImageObject | str | int | float) -> Operation: + def __add__(self, other: OTBObject | str | int | float) -> Operation: """Addition.""" return self._create_operator(Operation, "+", self, other) - def __sub__(self, other: ImageObject | str | int | float) -> Operation: + def __sub__(self, other: OTBObject | str | int | float) -> Operation: """Subtraction.""" return self._create_operator(Operation, "-", self, other) - def __mul__(self, other: ImageObject | str | int | float) -> Operation: + def __mul__(self, other: OTBObject | str | int | float) -> Operation: """Multiplication.""" return self._create_operator(Operation, "*", self, other) - def __truediv__(self, other: ImageObject | str | int | float) -> Operation: + def __truediv__(self, other: OTBObject | str | int | float) -> Operation: """Division.""" return self._create_operator(Operation, "/", self, other) - def __radd__(self, other: ImageObject | str | int | float) -> Operation: + def __radd__(self, other: OTBObject | str | int | float) -> Operation: """Right addition.""" return self._create_operator(Operation, "+", other, self) - def __rsub__(self, other: ImageObject | str | int | float) -> Operation: + def __rsub__(self, other: OTBObject | str | int | float) -> Operation: """Right subtraction.""" return self._create_operator(Operation, "-", other, self) - def __rmul__(self, other: ImageObject | str | int | float) -> Operation: + def __rmul__(self, other: OTBObject | str | int | float) -> Operation: """Right multiplication.""" return self._create_operator(Operation, "*", other, self) - def __rtruediv__(self, other: ImageObject | str | int | float) -> Operation: + def __rtruediv__(self, other: OTBObject | str | int | float) -> Operation: """Right division.""" return self._create_operator(Operation, "/", other, self) @@ -250,35 +250,35 @@ class ImageObject(ABC): """Absolute value.""" return Operation("abs", self) - def __ge__(self, other: ImageObject | str | int | float) -> LogicalOperation: + def __ge__(self, other: OTBObject | str | int | float) -> LogicalOperation: """Greater of equal than.""" return self._create_operator(LogicalOperation, ">=", self, other) - def __le__(self, other: ImageObject | str | int | float) -> LogicalOperation: + def __le__(self, other: OTBObject | str | int | float) -> LogicalOperation: """Lower of equal than.""" return self._create_operator(LogicalOperation, "<=", self, other) - def __gt__(self, other: ImageObject | str | int | float) -> LogicalOperation: + def __gt__(self, other: OTBObject | str | int | float) -> LogicalOperation: """Greater than.""" return self._create_operator(LogicalOperation, ">", self, other) - def __lt__(self, other: ImageObject | str | int | float) -> LogicalOperation: + def __lt__(self, other: OTBObject | str | int | float) -> LogicalOperation: """Lower than.""" return self._create_operator(LogicalOperation, "<", self, other) - def __eq__(self, other: ImageObject | str | int | float) -> LogicalOperation: + def __eq__(self, other: OTBObject | str | int | float) -> LogicalOperation: """Equality.""" return self._create_operator(LogicalOperation, "==", self, other) - def __ne__(self, other: ImageObject | str | int | float) -> LogicalOperation: + def __ne__(self, other: OTBObject | str | int | float) -> LogicalOperation: """Inequality.""" return self._create_operator(LogicalOperation, "!=", self, other) - def __or__(self, other: ImageObject | str | int | float) -> LogicalOperation: + def __or__(self, other: OTBObject | str | int | float) -> LogicalOperation: """Logical or.""" return self._create_operator(LogicalOperation, "||", self, other) - def __and__(self, other: ImageObject | str | int | float) -> LogicalOperation: + def __and__(self, other: OTBObject | str | int | float) -> LogicalOperation: """Logical and.""" return self._create_operator(LogicalOperation, "&&", self, other) @@ -317,7 +317,7 @@ class ImageObject(ABC): for inp in inputs: if isinstance(inp, (float, int, np.ndarray, np.generic)): arrays.append(inp) - elif isinstance(inp, ImageObject): + elif isinstance(inp, OTBObject): if not inp.exports_dic: inp.export() image_dic = inp.exports_dic[inp.output_image_key] @@ -349,16 +349,15 @@ class ImageObject(ABC): """ parameters = self.parameters.copy() for key, param in parameters.items(): - # In the following, we replace each parameter which is an ImageObject, with its summary. - if isinstance(param, ImageObject): # single parameter + # In the following, we replace each parameter which is an OTBObject, with its summary. + if isinstance(param, OTBObject): # single parameter parameters[key] = param.summarize() elif isinstance(param, list): # parameter list - parameters[key] = [p.summarize() if isinstance(p, ImageObject) else p for p in param] + parameters[key] = [p.summarize() if isinstance(p, OTBObject) else p for p in param] return {"name": self.app.GetName(), "parameters": parameters} - -class App(ImageObject): +class App(OTBObject): """Base class that gathers common operations for any OTB application.""" def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, **kwargs): @@ -458,7 +457,7 @@ class App(ImageObject): Args: *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") - - string or ImageObject, useful when the user implicitly wants to set the param "in" + - string or OTBObject, useful when the user implicitly wants to set the param "in" - list, useful when the user implicitly wants to set the param "il" **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif' @@ -648,7 +647,7 @@ class App(ImageObject): return tuple(files) # Private functions - def __parse_args(self, args: list[str | ImageObject | dict | list]) -> dict[str, Any]: + def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]: """Gather all input arguments in kwargs dict. Args: @@ -662,17 +661,17 @@ class App(ImageObject): for arg in args: if isinstance(arg, dict): kwargs.update(arg) - elif isinstance(arg, (str, ImageObject)) or isinstance(arg, list) and is_key_list(self, self.key_input): + elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.key_input): kwargs.update({self.key_input: arg}) return kwargs - def __set_param(self, key: str, obj: list | tuple | ImageObject | otb.Application | list[Any]): + def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]): """Set one parameter, decide which otb.Application method to use depending on target object.""" if obj is None or (isinstance(obj, (list, tuple)) and not obj): self.app.ClearValue(key) return # Single-parameter cases - if isinstance(obj, ImageObject): + if isinstance(obj, OTBObject): self.app.ConnectImage(key, obj.app, obj.output_image_key) elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0]) @@ -684,7 +683,7 @@ class App(ImageObject): elif is_key_images_list(self, key): # To enable possible in-memory connections, we go through the list and set the parameters one by one for inp in obj: - if isinstance(inp, ImageObject): + if isinstance(inp, OTBObject): self.app.ConnectImage(key, inp.app, inp.output_image_key) elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0]) @@ -844,7 +843,7 @@ class Operation(App): Args: operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? - *inputs: inputs. Can be ImageObject, filepath, int or float + *inputs: inputs. Can be OTBObject, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where name: override the Operation name @@ -876,7 +875,7 @@ class Operation(App): super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True) self.name = name or f'Operation exp="{self.exp}"' - def build_fake_expressions(self, operator: str, inputs: list[ImageObject | str | int | float], + def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None): """Create a list of 'fake' expressions, one for each band. @@ -884,7 +883,7 @@ class Operation(App): Args: operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? - inputs: inputs. Can be ImageObject, filepath, int or float + inputs: inputs. Can be OTBObject, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ @@ -959,8 +958,8 @@ class Operation(App): return exp_bands, ";".join(exp_bands) @staticmethod - def make_fake_exp(x: ImageObject | str, band: int, keep_logical: bool = False) \ - -> tuple[str, list[ImageObject], int]: + def make_fake_exp(x: OTBObject | str, band: int, keep_logical: bool = False) \ + -> tuple[str, list[OTBObject], int]: """This an internal function, only to be used by `build_fake_expressions`. Enable to create a fake expression just for one input and one band. @@ -1037,7 +1036,7 @@ class LogicalOperation(Operation): super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation") self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands) - def build_fake_expressions(self, operator: str, inputs: list[ImageObject | str | int | float], + def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None): """Create a list of 'fake' expressions, one for each band. @@ -1046,7 +1045,7 @@ class LogicalOperation(Operation): Args: operator: str (one of >, <, >=, <=, ==, !=, &, |) - inputs: Can be ImageObject, filepath, int or float + inputs: Can be OTBObject, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where """ @@ -1100,7 +1099,7 @@ class Input(App): return f"<pyotb.Input object from {self.filepath}>" -class Output(ImageObject): +class Output(OTBObject): """Object that behave like a pointer to a specific application output file.""" def __init__(self, pyotb_app: App, param_key: str = None, filepath: str = None, mkdir: bool = True): @@ -1129,7 +1128,7 @@ class Output(ImageObject): @property def output_image_key(self) -> str: - """Force the right key to be used when accessing the ImageObject.""" + """Force the right key to be used when accessing the OTBObject.""" return self.param_key def exists(self) -> bool: @@ -1155,17 +1154,17 @@ class Output(ImageObject): return f"<pyotb.Output {self.name} object, id {id(self)}>" -def get_nbchannels(inp: str | ImageObject) -> int: +def get_nbchannels(inp: str | OTBObject) -> int: """Get the nb of bands of input image. Args: - inp: can be filepath or ImageObject object + inp: can be filepath or OTBObject object Returns: number of bands in image """ - if isinstance(inp, ImageObject): + if isinstance(inp, OTBObject): nb_channels = inp.shape[-1] else: # Executing the app, without printing its log @@ -1177,7 +1176,7 @@ def get_nbchannels(inp: str | ImageObject) -> int: return nb_channels -def get_pixel_type(inp: str | ImageObject) -> str: +def get_pixel_type(inp: str | OTBObject) -> str: """Get the encoding of input image pixels. Args: @@ -1185,7 +1184,7 @@ def get_pixel_type(inp: str | ImageObject) -> str: Returns: pixel_type: OTB enum e.g. `otbApplication.ImagePixelType_uint8', which actually is an int. - For an ImageObject with several outputs, only the pixel type of the first output is returned + For an OTBObject with several outputs, only the pixel type of the first output is returned """ if isinstance(inp, str): @@ -1210,7 +1209,7 @@ def get_pixel_type(inp: str | ImageObject) -> str: if datatype not in datatype_to_pixeltype: raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}") pixel_type = getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[datatype]}") - elif isinstance(inp, ImageObject): + elif isinstance(inp, OTBObject): pixel_type = inp.app.GetParameterOutputImagePixelType(inp.output_image_key) else: raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}") @@ -1234,8 +1233,8 @@ def parse_pixel_type(pixel_type: str | int) -> int: raise ValueError(f"Bad pixel type specification ({pixel_type})") -def is_key_list(pyotb_app: ImageObject, key: str) -> bool: - """Check if a key of the ImageObject is an input parameter list.""" +def is_key_list(pyotb_app: OTBObject, key: str) -> bool: + """Check if a key of the OTBObject is an input parameter list.""" types = ( otb.ParameterType_InputImageList, otb.ParameterType_StringList, @@ -1246,12 +1245,12 @@ def is_key_list(pyotb_app: ImageObject, key: str) -> bool: return pyotb_app.app.GetParameterType(key) in types -def is_key_images_list(pyotb_app: ImageObject, key: str) -> bool: - """Check if a key of the ImageObject is an input parameter image list.""" +def is_key_images_list(pyotb_app: OTBObject, key: str) -> bool: + """Check if a key of the OTBObject is an input parameter image list.""" types = (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList) return pyotb_app.app.GetParameterType(key) in types -def get_out_images_param_keys(app: ImageObject) -> list[str]: +def get_out_images_param_keys(app: OTBObject) -> list[str]: """Return every output parameter keys of an OTB app.""" return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage] -- GitLab From ecce6d984b46494e45b2882d95aa7c4132681df0 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 4 Feb 2023 00:23:17 +0100 Subject: [PATCH 115/399] FIX: __getitem__ in OTBObject --- pyotb/core.py | 110 +++++++++++++++++++++++++------------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index a1bf285..b80fbcb 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -196,8 +196,24 @@ class OTBObject(ABC): row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x return abs(int(row)), int(col) + def summarize(self) -> dict[str, str | dict[str, Any]]: + """Serialize an object and its pipeline into a dictionary. + + Returns: + nested dictionary summarizing the pipeline + + """ + parameters = self.parameters.copy() + for key, param in parameters.items(): + # In the following, we replace each parameter which is an OTBObject, with its summary. + if isinstance(param, OTBObject): # single parameter + parameters[key] = param.summarize() + elif isinstance(param, list): # parameter list + parameters[key] = [p.summarize() if isinstance(p, OTBObject) else p for p in param] + return {"name": self.app.GetName(), "parameters": parameters} + @staticmethod - def _create_operator(op_cls, name, x, y) -> Operation: + def __create_operator(op_cls, name, x, y) -> Operation: """Create an operator. Args: @@ -214,6 +230,44 @@ class OTBObject(ABC): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return op_cls(name, x, y) + def __getitem__(self, key) -> Any | list[int | float] | int | float | Slicer: + """Override the default __getitem__ behaviour. + + This function enables 2 things : + - access attributes like that : object['any_attribute'] + - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3] + selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]] + selecting 1000x1000 subset : object[:1000, :1000] + - access pixel value(s) at a specified row, col index + + Args: + key: attribute key + + Returns: + attribute, pixel values or Slicer + + """ + # Accessing string attributes + if isinstance(key, str): + return getattr(self, key) + # Accessing pixel value(s) using Y/X coordinates + if isinstance(key, tuple) and len(key) >= 2: + row, col = key[0], key[1] + if isinstance(row, int) and isinstance(col, int): + if row < 0 or col < 0: + raise ValueError(f"{self.name}: can't read pixel value at negative coordinates ({row}, {col})") + channels = None + if len(key) == 3: + channels = key[2] + return self.get_values_at_coords(row, col, channels) + # Slicing + if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)): + raise ValueError(f'"{key}"cannot be interpreted as valid slicing. Slicing should be 2D or 3D.') + if isinstance(key, tuple) and len(key) == 2: + # Adding a 3rd dimension + key = key + (slice(None, None, None),) + return Slicer(self, *key) + def __add__(self, other: OTBObject | str | int | float) -> Operation: """Addition.""" return self._create_operator(Operation, "+", self, other) @@ -340,22 +394,6 @@ class OTBObject(ABC): return pyotb_app return NotImplemented - def summarize(self) -> dict[str, str | dict[str, Any]]: - """Serialize an object and its pipeline into a dictionary. - - Returns: - nested dictionary summarizing the pipeline - - """ - parameters = self.parameters.copy() - for key, param in parameters.items(): - # In the following, we replace each parameter which is an OTBObject, with its summary. - if isinstance(param, OTBObject): # single parameter - parameters[key] = param.summarize() - elif isinstance(param, list): # parameter list - parameters[key] = [p.summarize() if isinstance(p, OTBObject) else p for p in param] - return {"name": self.app.GetName(), "parameters": parameters} - class App(OTBObject): """Base class that gathers common operations for any OTB application.""" @@ -704,44 +742,6 @@ class App(OTBObject): """ return id(self) - def __getitem__(self, key) -> Any | list[int | float] | int | float | Slicer: - """Override the default __getitem__ behaviour. - - This function enables 2 things : - - access attributes like that : object['any_attribute'] - - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3] - selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]] - selecting 1000x1000 subset : object[:1000, :1000] - - access pixel value(s) at a specified row, col index - - Args: - key: attribute key - - Returns: - attribute, pixel values or Slicer - - """ - # Accessing string attributes - if isinstance(key, str): - return getattr(self, key) - # Accessing pixel value(s) using Y/X coordinates - if isinstance(key, tuple) and len(key) >= 2: - row, col = key[0], key[1] - if isinstance(row, int) and isinstance(col, int): - if row < 0 or col < 0: - raise ValueError(f"{self.name}: can't read pixel value at negative coordinates ({row}, {col})") - channels = None - if len(key) == 3: - channels = key[2] - return self.get_values_at_coords(row, col, channels) - # Slicing - if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)): - raise ValueError(f'"{key}"cannot be interpreted as valid slicing. Slicing should be 2D or 3D.') - if isinstance(key, tuple) and len(key) == 2: - # Adding a 3rd dimension - key = key + (slice(None, None, None),) - return Slicer(self, *key) - def __str__(self) -> str: """Return a nice string representation with object id.""" return f"<pyotb.App {self.name} object id {id(self)}>" -- GitLab From c4aad2f7c2c648aac54f5c34fe4dfe4c10394cff Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 4 Feb 2023 16:35:08 +0100 Subject: [PATCH 116/399] ENH: input and outpout keys --- pyotb/core.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index b80fbcb..1395e4e 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -440,29 +440,39 @@ class App(OTBObject): if any(key in self.parameters for key in self._out_param_types): self.flush() # auto flush if any output param was provided during app init - def get_first_key(self, param_types: list[int]) -> str: - """Get the first output param key for specific file types.""" - for key, param_type in sorted(self._all_param_types.items()): - if param_type in param_types: - return key + def get_first_key(self, *args: tuple[list[int]]) -> str: + """Get the first param key for specific file types, try each list in args.""" + for param_types in args: + for key, param_type in sorted(self._all_param_types.items()): + if param_type in param_types: + return key return None @property - def key_input(self) -> str: + def input_key(self) -> str: """Get the name of first input parameter, raster > vector > file.""" - return self.get_first_key([otb.ParameterType_InputImage, otb.ParameterType_InputImageList]) \ - or self.get_first_key([otb.ParameterType_InputVectorData, otb.ParameterType_InputVectorDataList]) \ - or self.get_first_key([otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList]) + return self.get_first_key( + [otb.ParameterType_InputImage, otb.ParameterType_InputImageList], + [otb.ParameterType_InputVectorData, otb.ParameterType_InputVectorDataList], + [otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList] + ) @property - def key_input_image(self) -> str: + def input_image_key(self) -> str: """Name of the first input image parameter.""" - return self.get_first_key(param_types=[otb.ParameterType_InputImage, otb.ParameterType_InputImageList]) + return self.get_first_key([otb.ParameterType_InputImage, otb.ParameterType_InputImageList]) + + @property + def output_key(self) -> str: + """Name of the first output parameter, raster > vector > file.""" + return self.get_first_key( + [otb.ParameterType_OutputImage], [otb.ParameterType_OutputVectorData], [otb.ParameterType_OutputFilename] + ) @property def output_image_key(self) -> str: """Get the name of first output image parameter.""" - return self.get_first_key(param_types=[otb.ParameterType_OutputImage]) + return self.get_first_key([otb.ParameterType_OutputImage]) @property def elapsed_time(self) -> float: @@ -542,7 +552,7 @@ class App(OTBObject): """ if not dtype: - param = self.parameters.get(self.key_input_image) + param = self.parameters.get(self.input_image_key) if not param: logger.warning("%s: could not propagate pixel type from inputs to output", self.name) return @@ -699,8 +709,8 @@ class App(OTBObject): for arg in args: if isinstance(arg, dict): kwargs.update(arg) - elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.key_input): - kwargs.update({self.key_input: arg}) + elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.input_key): + kwargs.update({self.input_key: arg}) return kwargs def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]): -- GitLab From 39a715cafcb0c1fab2a064ae23db4a97083820b4 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 4 Feb 2023 16:35:20 +0100 Subject: [PATCH 117/399] ENH: add _ to _create_operator --- pyotb/core.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 1395e4e..995b975 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -270,35 +270,35 @@ class OTBObject(ABC): def __add__(self, other: OTBObject | str | int | float) -> Operation: """Addition.""" - return self._create_operator(Operation, "+", self, other) + return self.__create_operator(Operation, "+", self, other) def __sub__(self, other: OTBObject | str | int | float) -> Operation: """Subtraction.""" - return self._create_operator(Operation, "-", self, other) + return self.__create_operator(Operation, "-", self, other) def __mul__(self, other: OTBObject | str | int | float) -> Operation: """Multiplication.""" - return self._create_operator(Operation, "*", self, other) + return self.__create_operator(Operation, "*", self, other) def __truediv__(self, other: OTBObject | str | int | float) -> Operation: """Division.""" - return self._create_operator(Operation, "/", self, other) + return self.__create_operator(Operation, "/", self, other) def __radd__(self, other: OTBObject | str | int | float) -> Operation: """Right addition.""" - return self._create_operator(Operation, "+", other, self) + return self.__create_operator(Operation, "+", other, self) def __rsub__(self, other: OTBObject | str | int | float) -> Operation: """Right subtraction.""" - return self._create_operator(Operation, "-", other, self) + return self.__create_operator(Operation, "-", other, self) def __rmul__(self, other: OTBObject | str | int | float) -> Operation: """Right multiplication.""" - return self._create_operator(Operation, "*", other, self) + return self.__create_operator(Operation, "*", other, self) def __rtruediv__(self, other: OTBObject | str | int | float) -> Operation: """Right division.""" - return self._create_operator(Operation, "/", other, self) + return self.__create_operator(Operation, "/", other, self) def __abs__(self) -> Operation: """Absolute value.""" @@ -306,35 +306,35 @@ class OTBObject(ABC): def __ge__(self, other: OTBObject | str | int | float) -> LogicalOperation: """Greater of equal than.""" - return self._create_operator(LogicalOperation, ">=", self, other) + return self.__create_operator(LogicalOperation, ">=", self, other) def __le__(self, other: OTBObject | str | int | float) -> LogicalOperation: """Lower of equal than.""" - return self._create_operator(LogicalOperation, "<=", self, other) + return self.__create_operator(LogicalOperation, "<=", self, other) def __gt__(self, other: OTBObject | str | int | float) -> LogicalOperation: """Greater than.""" - return self._create_operator(LogicalOperation, ">", self, other) + return self.__create_operator(LogicalOperation, ">", self, other) def __lt__(self, other: OTBObject | str | int | float) -> LogicalOperation: """Lower than.""" - return self._create_operator(LogicalOperation, "<", self, other) + return self.__create_operator(LogicalOperation, "<", self, other) def __eq__(self, other: OTBObject | str | int | float) -> LogicalOperation: """Equality.""" - return self._create_operator(LogicalOperation, "==", self, other) + return self.__create_operator(LogicalOperation, "==", self, other) def __ne__(self, other: OTBObject | str | int | float) -> LogicalOperation: """Inequality.""" - return self._create_operator(LogicalOperation, "!=", self, other) + return self.__create_operator(LogicalOperation, "!=", self, other) def __or__(self, other: OTBObject | str | int | float) -> LogicalOperation: """Logical or.""" - return self._create_operator(LogicalOperation, "||", self, other) + return self.__create_operator(LogicalOperation, "||", self, other) def __and__(self, other: OTBObject | str | int | float) -> LogicalOperation: """Logical and.""" - return self._create_operator(LogicalOperation, "&&", self, other) + return self.__create_operator(LogicalOperation, "&&", self, other) # Some other operations could be implemented with the same pattern # e.g. __pow__... cf https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types -- GitLab From 536f24c9f4a8b2c104f0c8fc83b89cb97dbb8657 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 4 Feb 2023 19:05:54 +0100 Subject: [PATCH 118/399] ENH: refac save_objects - add __getattr__ and remove data property --- pyotb/core.py | 107 +++++++++++++++++++++++--------------------------- 1 file changed, 50 insertions(+), 57 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 995b975..358ad1e 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -230,11 +230,32 @@ class OTBObject(ABC): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return op_cls(name, x, y) - def __getitem__(self, key) -> Any | list[int | float] | int | float | Slicer: + # Special functions + def __hash__(self) -> int: + """Override the default behaviour of the hash function. + + Returns: + self hash + + """ + return id(self) + + def __getattr__(self, key: str) -> Any: + """Return object attribute, or if key is found in self.parameters or self.data.""" + if key in dir(self): + return self.__dict__[key] + if key in self._out_param_types and key in self.parameters: + return Output(self, key, self.parameters[key]) + if key in self.parameters: + return self.parameters[key] + if key in self.data: + return self.data[key] + raise AttributeError(f"{self.name}: unknown attribute '{key}'") + + def __getitem__(self, key) -> Any | list[float] | float | Slicer: """Override the default __getitem__ behaviour. This function enables 2 things : - - access attributes like that : object['any_attribute'] - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3] selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]] selecting 1000x1000 subset : object[:1000, :1000] @@ -244,12 +265,9 @@ class OTBObject(ABC): key: attribute key Returns: - attribute, pixel values or Slicer + list of pixel values if vector image, or pixel value, or Slicer """ - # Accessing string attributes - if isinstance(key, str): - return getattr(self, key) # Accessing pixel value(s) using Y/X coordinates if isinstance(key, tuple) and len(key) >= 2: row, col = key[0], key[1] @@ -372,11 +390,8 @@ class OTBObject(ABC): if isinstance(inp, (float, int, np.ndarray, np.generic)): arrays.append(inp) elif isinstance(inp, OTBObject): - if not inp.exports_dic: - inp.export() - image_dic = inp.exports_dic[inp.output_image_key] - array = image_dic["array"] - arrays.append(array) + image_dic = inp.export() + arrays.append(image_dic["array"]) else: logger.debug(type(self)) return NotImplemented @@ -424,8 +439,7 @@ class App(OTBObject): self.quiet = quiet self.image_dic = image_dic self._time_start, self._time_end = 0, 0 - self.exports_dic = {} - self.parameters = {} + self.data, self.parameters, self.exports_dic = {}, {}, {} # Initialize app, set parameters and execute if not frozen create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication self.app = create(name) @@ -484,18 +498,6 @@ class App(OTBObject): """List of used application outputs.""" return [getattr(self, key) for key in self._out_param_types if key in self.parameters] - @property - def data(self) -> dict[str, float, list[float]]: - """Expose app's output data values in a dictionary.""" - known_bad_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem") - skip_keys = known_bad_keys + tuple(self._out_param_types) + tuple(self.parameters) - data_dict = {} - for key in filter(lambda k: k not in skip_keys, self.parameters_keys): - value = self.__dict__.get(key) - if not isinstance(value, otb.ApplicationProxy) and value not in (None, "", [], ()): - data_dict[str(key)] = value - return data_dict - def set_parameters(self, *args, **kwargs): """Set some parameters of the app. @@ -533,11 +535,8 @@ class App(OTBObject): f"{self.name}: something went wrong before execution " f"(while setting parameter '{key}' to '{obj}': {e})" ) from e - # Update _parameters using values from OtbApplication object - otb_params = self.app.GetParameters().items() - otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params} # Update param dict and save values as object attributes - self.parameters.update({**parameters, **otb_params}) + self.parameters.update(parameters) self.save_objects() def propagate_dtype(self, target_key: str = None, dtype: int = None): @@ -571,29 +570,27 @@ class App(OTBObject): self.app.SetParameterOutputImagePixelType(key, dtype) def save_objects(self): - """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`. - - This is useful when the key contains reserved characters such as a point eg "io.out" - """ + """Saving OTB app parameters and outputs in data and parameters dict.""" for key in self.parameters_keys: - if key in dir(self.__class__): - continue # skip forbidden attribute since it is already used by the class - value = self.parameters.get(key) # basic parameters + value = self.parameters.get(key) if value is None: try: value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results) + if isinstance(value, otb.ApplicationProxy): + continue except RuntimeError: continue # this is when there is no value for key - # Convert output param path to Output object - if key in self._out_param_types: - value = Output(self, key, value) - elif isinstance(value, str): - try: - value = literal_eval(value) - except (ValueError, SyntaxError): - pass - # Save attribute - setattr(self, key, value) + # Save app data output or update parameters dict with otb.Application values + if isinstance(value, OTBObject) or bool(value) or value == 0: + if self.app.GetParameterRole(key) == 0: + self.parameters[key] = value + else: + if isinstance(value, str): + try: + value = literal_eval(value) + except (ValueError, SyntaxError): + pass + self.data[key] = value def execute(self): """Execute and write to disk if any output parameter has been set during init.""" @@ -615,6 +612,7 @@ class App(OTBObject): self.app.WriteOutput() except RuntimeError: logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name) + self._time_start = perf_counter() self.app.ExecuteAndWriteOutput() self._time_end = perf_counter() @@ -743,18 +741,13 @@ class App(OTBObject): self.app.SetParameterValue(key, obj) # Special functions - def __hash__(self) -> int: - """Override the default behaviour of the hash function. + def __getitem__(self, key: str) -> Any | list[int | float] | int | float | Slicer: + """This function is called when we use App()[...]. - Returns: - self hash - - """ - return id(self) - - def __str__(self) -> str: - """Return a nice string representation with object id.""" - return f"<pyotb.App {self.name} object id {id(self)}>" + We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer.""" + if isinstance(key, str) and key in self.parameters_keys: + return getattr(self, key) + return super().__getitem__(key) class Slicer(App): -- GitLab From 5c676a1b838c4a6b646176d871b45e5f636804d1 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 4 Feb 2023 20:30:43 +0100 Subject: [PATCH 119/399] ENH: update tests, move data to tests_data --- tests/test_core.py | 42 +++++++++++++++++------------------ tests/test_numpy.py | 4 ++-- tests/test_serialization.py | 25 +++------------------ tests/tests_data.py | 44 ++++++++++++++++++++++++++++++++++++- 4 files changed, 69 insertions(+), 46 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 7bed905..1b5db4c 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,14 +1,7 @@ import pytest import pyotb -from tests_data import INPUT - -TEST_IMAGE_STATS = { - 'out.mean': [79.5505, 109.225, 115.456, 249.349], - 'out.min': [33, 64, 91, 47], - 'out.max': [255, 255, 230, 255], - 'out.std': [51.0754, 35.3152, 23.4514, 20.3827] -} +from tests_data import INPUT, TEST_IMAGE_STATS # Input settings @@ -23,7 +16,7 @@ def test_wrong_key(): # OTBObject properties def test_key_input(): - assert INPUT.key_input == INPUT.key_input_image == "in" + assert INPUT.input_key == INPUT.input_image_key == "in" def test_key_output(): @@ -64,6 +57,8 @@ def test_elapsed_time(): def test_get_infos(): infos = INPUT.get_info() assert (infos["sizex"], infos["sizey"]) == (251, 304) + bm_infos = pyotb.BandMathX([INPUT], exp="im1")["out"].get_info() + assert infos == bm_infos def test_get_statistics(): @@ -76,21 +71,21 @@ def test_xy_to_rowcol(): def test_write(): INPUT.write("/tmp/test_write.tif") - assert INPUT.out.exists() - INPUT.out.filepath.unlink() + assert INPUT["out"].exists() + INPUT["out"].filepath.unlink() def test_output_write(): - INPUT.out.write("/tmp/test_output_write.tif") - assert INPUT.out.exists() - INPUT.out.filepath.unlink() + INPUT["out"].write("/tmp/test_output_write.tif") + assert INPUT["out"].exists() + INPUT["out"].filepath.unlink() # Slicer def test_slicer_shape(): extract = INPUT[:50, :60, :3] assert extract.shape == (50, 60, 3) - assert extract.parameters["cl"] == ("Channel1", "Channel2", "Channel3") + assert extract.parameters["cl"] == ["Channel1", "Channel2", "Channel3"] def test_slicer_preserve_dtype(): @@ -102,6 +97,11 @@ def test_slicer_negative_band_index(): assert INPUT[:50, :60, :-2].shape == (50, 60, 2) +def test_slicer_in_output(): + slc = pyotb.BandMath([INPUT], exp="im1b1")["out"][:50, :60, :-2] + assert isinstance(slc, pyotb.core.Slicer) + + # Arithmetic def test_operation(): op = INPUT / 255 * 128 @@ -128,8 +128,8 @@ def test_binary_mask_where(): # Essential apps def test_app_readimageinfo(): info = pyotb.ReadImageInfo(INPUT, quiet=True) - assert (info.sizex, info.sizey) == (251, 304) - assert info["numberbands"] == info.numberbands == 4 + assert (info["sizex"], info["sizey"]) == (251, 304) + assert info["numberbands"] == 4 def test_app_computeimagestats(): @@ -150,16 +150,16 @@ def test_read_values_at_coords(): # BandMath NDVI == RadiometricIndices NDVI ? def test_ndvi_comparison(): ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (INPUT[:, :, -1] + INPUT[:, :, 0]) - ndvi_indices = pyotb.RadiometricIndices(INPUT, {"list": "Vegetation:NDVI", "channels.red": 1, "channels.nir": 4}) + ndvi_indices = pyotb.RadiometricIndices(INPUT, {"list": ["Vegetation:NDVI"], "channels.red": 1, "channels.nir": 4}) assert ndvi_bandmath.exp == "((im1b4 - im1b1) / (im1b4 + im1b1))" ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", pixel_type="float") - assert ndvi_bandmath.out.filepath.exists() + assert ndvi_bandmath["out"].filepath.exists() ndvi_indices.write("/tmp/ndvi_indices.tif", pixel_type="float") - assert ndvi_indices.out.filepath.exists() + assert ndvi_indices["out"].filepath.exists() compared = pyotb.CompareImages({"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"}) - assert (compared.count, compared.mse) == (0, 0) + assert (compared["count"], compared["mse"]) == (0, 0) thresholded_indices = pyotb.where(ndvi_indices >= 0.3, 1, 0) assert thresholded_indices.exp == "((im1b1 >= 0.3) ? 1 : 0)" diff --git a/tests/test_numpy.py b/tests/test_numpy.py index e62ffbb..d9c3d7b 100644 --- a/tests/test_numpy.py +++ b/tests/test_numpy.py @@ -12,8 +12,8 @@ def test_export(): def test_output_export(): - INPUT.out.export() - assert INPUT.out.output_image_key in INPUT.out.exports_dic + INPUT["out"].export() + assert INPUT["out"].output_image_key in INPUT["out"].exports_dic def test_to_numpy(): diff --git a/tests/test_serialization.py b/tests/test_serialization.py index b56e447..cb079e1 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -1,5 +1,5 @@ import pyotb -from tests_data import FILEPATH +from tests_data import * def test_pipeline_simple(): @@ -8,13 +8,7 @@ def test_pipeline_simple(): app2 = pyotb.OrthoRectification({'io.in': app1}) app3 = pyotb.ManageNoData({'in': app2}) summary = app3.summarize() - reference = {'name': 'ManageNoData', 'parameters': {'in': { - 'name': 'OrthoRectification', 'parameters': {'io.in': { - 'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}}, - 'map': 'utm', - 'outputs.isotropic': True}}, - 'mode': 'buildmask'}} - assert summary == reference + assert summary == SIMPLE_SERIALIZATION def test_pipeline_diamond(): @@ -24,17 +18,4 @@ def test_pipeline_diamond(): app3 = pyotb.ManageNoData({'in': app2}) app4 = pyotb.BandMathX({'il': [app2, app3], 'exp': 'im1+im2'}) summary = app4.summarize() - reference = {'name': 'BandMathX', 'parameters': {'il': [ - {'name': 'OrthoRectification', 'parameters': {'io.in': { - 'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}}, - 'map': 'utm', - 'outputs.isotropic': True}}, - {'name': 'ManageNoData', 'parameters': {'in': { - 'name': 'OrthoRectification', 'parameters': { - 'io.in': {'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}}, - 'map': 'utm', - 'outputs.isotropic': True}}, - 'mode': 'buildmask'}} - ], - 'exp': 'im1+im2'}} - assert summary == reference + assert summary == COMPLEX_SERIALIZATION diff --git a/tests/tests_data.py b/tests/tests_data.py index b190f40..a002c55 100644 --- a/tests/tests_data.py +++ b/tests/tests_data.py @@ -1,3 +1,45 @@ import pyotb -FILEPATH = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false" + +FILEPATH = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" INPUT = pyotb.Input(FILEPATH) + +TEST_IMAGE_STATS = { + 'out.mean': [79.5505, 109.225, 115.456, 249.349], + 'out.min': [33, 64, 91, 47], + 'out.max': [255, 255, 230, 255], + 'out.std': [51.0754, 35.3152, 23.4514, 20.3827] +} + +SIMPLE_SERIALIZATION = {'name': 'ManageNoData', + 'parameters': {'in': {'name': 'OrthoRectification', + 'parameters': {'io.in': {'name': 'BandMath', + 'parameters': {'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'], + 'exp': 'im1b1', 'ram': 256}}, + 'map.utm.zone': 31, 'map.utm.northhem': False, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, + 'elev.default': 0.0, 'interpolator.bco.radius': 2, 'opt.rpc': 10, 'opt.ram': 256, + 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5, 'outputs.sizex': 251, 'outputs.sizey': 304, + 'outputs.spacingx': 5.997312068939209, 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}}, + 'usenan': False, 'mode.buildmask.inv': 1.0, 'mode.buildmask.outv': 0.0, 'mode.changevalue.newv': 0.0, 'mode.apply.ndval': 0.0, 'ram': 256}} + +COMPLEX_SERIALIZATION = { + 'name': 'BandMathX', + 'parameters': {'il': [{'name': 'OrthoRectification', + 'parameters': {'io.in': {'name': 'BandMath', + 'parameters': { + 'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'], + 'exp': 'im1b1', 'ram': 256}}, + 'map.utm.zone': 31, 'map.utm.northhem': False, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, 'elev.default': 0.0, + 'interpolator.bco.radius': 2, 'opt.rpc': 10, 'opt.ram': 256, 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5, + 'outputs.sizex': 251, 'outputs.sizey': 304, 'outputs.spacingx': 5.997312068939209, 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}}, + {'name': 'ManageNoData', + 'parameters': {'in': {'name': 'OrthoRectification', + 'parameters': {'io.in': {'name': 'BandMath', + 'parameters': {'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'], + 'exp': 'im1b1', 'ram': 256}}, + 'map.utm.zone': 31, 'map.utm.northhem': False, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, 'elev.default': 0.0, 'interpolator.bco.radius': 2, + 'opt.rpc': 10, 'opt.ram': 256, 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5, + 'outputs.sizex': 251, 'outputs.sizey': 304, 'outputs.spacingx': 5.997312068939209, + 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}}, + 'usenan': False, 'mode.buildmask.inv': 1.0, 'mode.buildmask.outv': 0.0, 'mode.changevalue.newv': 0.0, 'mode.apply.ndval': 0.0, 'ram': 256}}], + 'exp': 'im1+im2', 'ram': 256} +} -- GitLab From 6fec9bb996fd200dfdddc6ce36fd4670c39ca296 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 4 Feb 2023 20:30:45 +0100 Subject: [PATCH 120/399] ENH: use __repr__ with obj id in OTBObject, __str__ is left for uses --- pyotb/core.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 358ad1e..25f677b 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -240,18 +240,6 @@ class OTBObject(ABC): """ return id(self) - def __getattr__(self, key: str) -> Any: - """Return object attribute, or if key is found in self.parameters or self.data.""" - if key in dir(self): - return self.__dict__[key] - if key in self._out_param_types and key in self.parameters: - return Output(self, key, self.parameters[key]) - if key in self.parameters: - return self.parameters[key] - if key in self.data: - return self.data[key] - raise AttributeError(f"{self.name}: unknown attribute '{key}'") - def __getitem__(self, key) -> Any | list[float] | float | Slicer: """Override the default __getitem__ behaviour. @@ -286,6 +274,10 @@ class OTBObject(ABC): key = key + (slice(None, None, None),) return Slicer(self, *key) + def __repr__(self) -> str: + """Return a nice string representation with object id.""" + return f"<pyotb.{self.__class__.__name__} object id {id(self)}>" + def __add__(self, other: OTBObject | str | int | float) -> Operation: """Addition.""" return self.__create_operator(Operation, "+", self, other) @@ -670,7 +662,7 @@ class App(OTBObject): # Set parameters and flush to disk for key, output_filename in kwargs.items(): - if Path(output_filename).exists(): + if Path(output_filename.split("?")[0]).exists(): logger.warning("%s: overwriting file %s", self.name, output_filename) if key in dtypes: self.propagate_dtype(key, dtypes[key]) @@ -745,8 +737,14 @@ class App(OTBObject): """This function is called when we use App()[...]. We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer.""" - if isinstance(key, str) and key in self.parameters_keys: - return getattr(self, key) + if isinstance(key, str): + if key in self._out_param_types: + return Output(self, key, self.parameters.get(key)) + if key in self.parameters: + return self.parameters[key] + if key in self.data: + return self.data[key] + raise KeyError(f"{self.name} object has no attribute '{key}'") return super().__getitem__(key) @@ -1153,8 +1151,8 @@ class Output(OTBObject): return self.parent_pyotb_app.write({self.output_image_key: filepath}, **kwargs) def __str__(self) -> str: - """Return a nice string representation with source app name and object id.""" - return f"<pyotb.Output {self.name} object, id {id(self)}>" + """Return string representation of Output filepath.""" + return str(self.filepath) def get_nbchannels(inp: str | OTBObject) -> int: -- GitLab From 3ac03d3087b2412096e728d8a7759664781dfe6d Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 4 Feb 2023 20:41:18 +0100 Subject: [PATCH 121/399] STYE: linter --- pyotb/core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 25f677b..58b054b 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -736,7 +736,8 @@ class App(OTBObject): def __getitem__(self, key: str) -> Any | list[int | float] | int | float | Slicer: """This function is called when we use App()[...]. - We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer.""" + We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer + """ if isinstance(key, str): if key in self._out_param_types: return Output(self, key, self.parameters.get(key)) @@ -791,7 +792,7 @@ class Slicer(App): # Spatial slicing spatial_slicing = False - # TODO TBD: handle the step value in the slice so that NN undersampling is possible ? e.g. raster[::2, ::2] + # TODO: handle the step value in the slice so that NN undersampling is possible ? e.g. raster[::2, ::2] if rows.start is not None: parameters.update({"mode.extent.uly": rows.start}) spatial_slicing = True -- GitLab From 8a60776b92e1d7baea7d9dd289f0ad8b9d65ea29 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 4 Feb 2023 21:16:12 +0100 Subject: [PATCH 122/399] ENH: style and raise exceptions when get_first_key failed --- pyotb/core.py | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 58b054b..d5124ec 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -5,6 +5,7 @@ from __future__ import annotations from ast import literal_eval from pathlib import Path from time import perf_counter +from itertools import chain from typing import Any from abc import ABC, abstractmethod @@ -173,13 +174,12 @@ class OTBObject(ABC): profile: a metadata dict required to write image using rasterio """ + profile = {} array = self.to_numpy(preserve_dtype=True, copy=False) height, width, count = array.shape proj = self.app.GetImageProjection(self.output_image_key) - profile = { - 'crs': proj, 'dtype': array.dtype, 'transform': self.transform, - 'count': count, 'height': height, 'width': width, - } + profile.update({"crs": proj, "dtype": array.dtype, "transform": self.transform}) + profile.update({"count": count, "height": height, "width": width}) return np.moveaxis(array, 2, 0), profile def get_rowcol_from_xy(self, x: float, y: float) -> tuple[int, int]: @@ -449,36 +449,35 @@ class App(OTBObject): def get_first_key(self, *args: tuple[list[int]]) -> str: """Get the first param key for specific file types, try each list in args.""" for param_types in args: - for key, param_type in sorted(self._all_param_types.items()): - if param_type in param_types: + types = [getattr(otb, "ParameterType_" + key) for key in param_types] + for key, value in sorted(self._all_param_types.items()): + if value in types: return key - return None + raise TypeError(f"{self.name}: could not find any parameter of type {tuple(chain(*args))}") @property def input_key(self) -> str: """Get the name of first input parameter, raster > vector > file.""" return self.get_first_key( - [otb.ParameterType_InputImage, otb.ParameterType_InputImageList], - [otb.ParameterType_InputVectorData, otb.ParameterType_InputVectorDataList], - [otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList] + ["InputImage", "InputImageList"], + ["InputVectorData", "InputVectorDataList"], + ["InputFilename", "InputFilenameList"], ) @property def input_image_key(self) -> str: """Name of the first input image parameter.""" - return self.get_first_key([otb.ParameterType_InputImage, otb.ParameterType_InputImageList]) + return self.get_first_key(["InputImage", "InputImageList"]) @property def output_key(self) -> str: """Name of the first output parameter, raster > vector > file.""" - return self.get_first_key( - [otb.ParameterType_OutputImage], [otb.ParameterType_OutputVectorData], [otb.ParameterType_OutputFilename] - ) + return self.get_first_key(["OutputImage"], ["OutputVectorData"], ["OutputFilename"]) @property def output_image_key(self) -> str: """Get the name of first output image parameter.""" - return self.get_first_key([otb.ParameterType_OutputImage]) + return self.get_first_key(["OutputImage"]) @property def elapsed_time(self) -> float: @@ -659,7 +658,8 @@ class App(OTBObject): dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()} elif preserve_dtype: self.propagate_dtype() # all outputs will have the same type as the main input raster - + if not kwargs: + raise KeyError(f"{self.name}: at least one filepath is required, if not passed to App during init") # Set parameters and flush to disk for key, output_filename in kwargs.items(): if Path(output_filename.split("?")[0]).exists(): @@ -877,8 +877,7 @@ class Operation(App): super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True) self.name = name or f'Operation exp="{self.exp}"' - def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float], - nb_bands: int = None): + def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None): """Create a list of 'fake' expressions, one for each band. E.g for the operation input1 + input2, we create a fake expression that is like "str(input1) + str(input2)" @@ -960,8 +959,7 @@ class Operation(App): return exp_bands, ";".join(exp_bands) @staticmethod - def make_fake_exp(x: OTBObject | str, band: int, keep_logical: bool = False) \ - -> tuple[str, list[OTBObject], int]: + def make_fake_exp(x: OTBObject | str, band: int, keep_logical: bool = False) -> tuple[str, list[OTBObject], int]: """This an internal function, only to be used by `build_fake_expressions`. Enable to create a fake expression just for one input and one band. @@ -1038,8 +1036,7 @@ class LogicalOperation(Operation): super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation") self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands) - def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float], - nb_bands: int = None): + def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None): """Create a list of 'fake' expressions, one for each band. e.g for the operation input1 > input2, we create a fake expression that is like -- GitLab From d678ef5f562cffeafd03baf0b6d256624eff067a Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 4 Feb 2023 21:18:50 +0100 Subject: [PATCH 123/399] ENH: rename filename_extension to ext_fname --- pyotb/core.py | 14 +++++++------- tests/test_core.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index d5124ec..463338c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -607,7 +607,7 @@ class App(OTBObject): self.app.ExecuteAndWriteOutput() self._time_end = perf_counter() - def write(self, *args, filename_extension: str = "", pixel_type: dict[str, str] | str = None, + def write(self, *args, ext_fname: str = "", pixel_type: dict[str, str] | str = None, preserve_dtype: bool = False, **kwargs): """Set output pixel type and write the output raster files. @@ -616,7 +616,7 @@ class App(OTBObject): non-standard characters such as a point, e.g. {'io.out':'output.tif'} - string, useful when there is only one output, e.g. 'output.tif' - None if output file was passed during App init - filename_extension: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES") + ext_fname: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES") Will be used for all outputs (Default value = "") pixel_type: Can be : - dictionary {output_parameter_key: pixeltype} when specifying for several outputs - str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several @@ -637,13 +637,13 @@ class App(OTBObject): kwargs.update({self.output_image_key: str(arg)}) # Append filename extension to filenames - if filename_extension: - logger.debug("%s: using extended filename for outputs: %s", self.name, filename_extension) - if not filename_extension.startswith("?"): - filename_extension = "?" + filename_extension + if ext_fname: + logger.debug("%s: using extended filename for outputs: %s", self.name, ext_fname) + if not ext_fname.startswith("?"): + ext_fname = "?" + ext_fname for key, value in kwargs.items(): if self._out_param_types[key] == otb.ParameterType_OutputImage and "?" not in value: - kwargs[key] = value + filename_extension + kwargs[key] = value + ext_fname # Manage output pixel types dtypes = {} diff --git a/tests/test_core.py b/tests/test_core.py index 1b5db4c..eba448e 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -70,7 +70,7 @@ def test_xy_to_rowcol(): def test_write(): - INPUT.write("/tmp/test_write.tif") + INPUT.write("/tmp/test_write.tif", ext_fname="nodata=0") assert INPUT["out"].exists() INPUT["out"].filepath.unlink() -- GitLab From b198e9915522b1d018a22d9b01a75cac75472c52 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 4 Feb 2023 21:29:10 +0100 Subject: [PATCH 124/399] FIX: try to add "?&" to ext_fname test --- tests/test_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_core.py b/tests/test_core.py index eba448e..5e59d5b 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -70,7 +70,7 @@ def test_xy_to_rowcol(): def test_write(): - INPUT.write("/tmp/test_write.tif", ext_fname="nodata=0") + INPUT.write("/tmp/test_write.tif", ext_fname="?&nodata=0") assert INPUT["out"].exists() INPUT["out"].filepath.unlink() -- GitLab From d29d17c6d973c216384cc82bce0bdc5e8868cf2d Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 4 Feb 2023 21:35:48 +0100 Subject: [PATCH 125/399] FIX: force use of "?&" in ext_fname when "?" is missing --- pyotb/core.py | 2 +- tests/test_core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 463338c..4eac666 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -640,7 +640,7 @@ class App(OTBObject): if ext_fname: logger.debug("%s: using extended filename for outputs: %s", self.name, ext_fname) if not ext_fname.startswith("?"): - ext_fname = "?" + ext_fname + ext_fname = "?&" + ext_fname for key, value in kwargs.items(): if self._out_param_types[key] == otb.ParameterType_OutputImage and "?" not in value: kwargs[key] = value + ext_fname diff --git a/tests/test_core.py b/tests/test_core.py index 5e59d5b..eba448e 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -70,7 +70,7 @@ def test_xy_to_rowcol(): def test_write(): - INPUT.write("/tmp/test_write.tif", ext_fname="?&nodata=0") + INPUT.write("/tmp/test_write.tif", ext_fname="nodata=0") assert INPUT["out"].exists() INPUT["out"].filepath.unlink() -- GitLab From bdbefb2b278c1e85349c4aaec8876fc901d514df Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 4 Feb 2023 21:51:36 +0100 Subject: [PATCH 126/399] STYLE: identation --- tests/tests_data.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/tests_data.py b/tests/tests_data.py index a002c55..f69edac 100644 --- a/tests/tests_data.py +++ b/tests/tests_data.py @@ -9,18 +9,18 @@ TEST_IMAGE_STATS = { 'out.max': [255, 255, 230, 255], 'out.std': [51.0754, 35.3152, 23.4514, 20.3827] } - -SIMPLE_SERIALIZATION = {'name': 'ManageNoData', - 'parameters': {'in': {'name': 'OrthoRectification', - 'parameters': {'io.in': {'name': 'BandMath', - 'parameters': {'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'], - 'exp': 'im1b1', 'ram': 256}}, - 'map.utm.zone': 31, 'map.utm.northhem': False, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, - 'elev.default': 0.0, 'interpolator.bco.radius': 2, 'opt.rpc': 10, 'opt.ram': 256, - 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5, 'outputs.sizex': 251, 'outputs.sizey': 304, - 'outputs.spacingx': 5.997312068939209, 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}}, - 'usenan': False, 'mode.buildmask.inv': 1.0, 'mode.buildmask.outv': 0.0, 'mode.changevalue.newv': 0.0, 'mode.apply.ndval': 0.0, 'ram': 256}} - +SIMPLE_SERIALIZATION = { + 'name': 'ManageNoData', + 'parameters': {'in': {'name': 'OrthoRectification', + 'parameters': {'io.in': {'name': 'BandMath', + 'parameters': {'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'], + 'exp': 'im1b1', 'ram': 256}}, + 'map.utm.zone': 31, 'map.utm.northhem': False, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, + 'elev.default': 0.0, 'interpolator.bco.radius': 2, 'opt.rpc': 10, 'opt.ram': 256, + 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5, 'outputs.sizex': 251, 'outputs.sizey': 304, + 'outputs.spacingx': 5.997312068939209, 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}}, + 'usenan': False, 'mode.buildmask.inv': 1.0, 'mode.buildmask.outv': 0.0, 'mode.changevalue.newv': 0.0, 'mode.apply.ndval': 0.0, 'ram': 256} +} COMPLEX_SERIALIZATION = { 'name': 'BandMathX', 'parameters': {'il': [{'name': 'OrthoRectification', -- GitLab From 249103b2e54c1d3aadae27b8e99c5055a46a3932 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 4 Feb 2023 21:56:02 +0100 Subject: [PATCH 127/399] ENH: better exception when accessing App[param] with wrong name --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 4eac666..3512131 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -745,7 +745,7 @@ class App(OTBObject): return self.parameters[key] if key in self.data: return self.data[key] - raise KeyError(f"{self.name} object has no attribute '{key}'") + raise KeyError(f"{self.name}: unknown parameter '{key}'") return super().__getitem__(key) -- GitLab From 5e15559891adf2550529e514c2da1fe22cbec091 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 4 Feb 2023 22:11:17 +0100 Subject: [PATCH 128/399] STYLE: tests autoformat, use Output.exists() --- tests/test_core.py | 5 ++--- tests/test_pipeline.py | 8 +++----- tests/test_serialization.py | 14 +++++++------- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index eba448e..1b6441f 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -52,7 +52,6 @@ def test_elapsed_time(): assert 0 < pyotb.ReadImageInfo(INPUT).elapsed_time < 1 - # Other functions def test_get_infos(): infos = INPUT.get_info() @@ -154,9 +153,9 @@ def test_ndvi_comparison(): assert ndvi_bandmath.exp == "((im1b4 - im1b1) / (im1b4 + im1b1))" ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", pixel_type="float") - assert ndvi_bandmath["out"].filepath.exists() + assert ndvi_bandmath["out"].exists() ndvi_indices.write("/tmp/ndvi_indices.tif", pixel_type="float") - assert ndvi_indices["out"].filepath.exists() + assert ndvi_indices["out"].exists() compared = pyotb.CompareImages({"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"}) assert (compared["count"], compared["mse"]) == (0, 0) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 76f9872..19160c5 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -12,7 +12,7 @@ OTBAPPS_BLOCKS = [ lambda inp: pyotb.DynamicConvert({"in": inp}), lambda inp: pyotb.Mosaic({"il": [inp]}), lambda inp: pyotb.BandMath({"il": [inp], "exp": "im1b1 + 1"}), - lambda inp: pyotb.BandMathX({"il": [inp], "exp": "im1"}) + lambda inp: pyotb.BandMathX({"il": [inp], "exp": "im1"}), ] PYOTB_BLOCKS = [ @@ -74,14 +74,12 @@ def pipeline2str(pipeline): a string """ - return " > ".join([INPUT.__class__.__name__] + [f"{i}.{app.name.split()[0]}" - for i, app in enumerate(pipeline)]) + return " > ".join([INPUT.__class__.__name__] + [f"{i}.{app.name.split()[0]}" for i, app in enumerate(pipeline)]) def make_pipelines_list(): """Create a list of pipelines using different lengths and blocks""" - blocks = {FILEPATH: OTBAPPS_BLOCKS, # for filepath, we can't use Slicer or Operation - INPUT: ALL_BLOCKS} + blocks = {FILEPATH: OTBAPPS_BLOCKS, INPUT: ALL_BLOCKS} # for filepath, we can't use Slicer or Operation pipelines = [] names = [] for inp, blocks in blocks.items(): diff --git a/tests/test_serialization.py b/tests/test_serialization.py index cb079e1..28aa3a7 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -4,18 +4,18 @@ from tests_data import * def test_pipeline_simple(): # BandMath -> OrthoRectification -> ManageNoData - app1 = pyotb.BandMath({'il': [FILEPATH], 'exp': 'im1b1'}) - app2 = pyotb.OrthoRectification({'io.in': app1}) - app3 = pyotb.ManageNoData({'in': app2}) + app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"}) + app2 = pyotb.OrthoRectification({"io.in": app1}) + app3 = pyotb.ManageNoData({"in": app2}) summary = app3.summarize() assert summary == SIMPLE_SERIALIZATION def test_pipeline_diamond(): # Diamond graph - app1 = pyotb.BandMath({'il': [FILEPATH], 'exp': 'im1b1'}) - app2 = pyotb.OrthoRectification({'io.in': app1}) - app3 = pyotb.ManageNoData({'in': app2}) - app4 = pyotb.BandMathX({'il': [app2, app3], 'exp': 'im1+im2'}) + app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"}) + app2 = pyotb.OrthoRectification({"io.in": app1}) + app3 = pyotb.ManageNoData({"in": app2}) + app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"}) summary = app4.summarize() assert summary == COMPLEX_SERIALIZATION -- GitLab From 0addfdf8f5f759b79f17277597ca188d8454a6db Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 5 Feb 2023 14:27:02 +0100 Subject: [PATCH 129/399] STYLE: var names, types and exceptions --- pyotb/core.py | 123 ++++++++++++++++++++++++++------------------------ 1 file changed, 63 insertions(+), 60 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 3512131..364c7c7 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -259,16 +259,15 @@ class OTBObject(ABC): # Accessing pixel value(s) using Y/X coordinates if isinstance(key, tuple) and len(key) >= 2: row, col = key[0], key[1] - if isinstance(row, int) and isinstance(col, int): - if row < 0 or col < 0: - raise ValueError(f"{self.name}: can't read pixel value at negative coordinates ({row}, {col})") - channels = None - if len(key) == 3: - channels = key[2] - return self.get_values_at_coords(row, col, channels) + if not isinstance(row, int) or not isinstance(col, int): + raise TypeError(f"{self.name}: cannnot read pixel values using {type(row)} and {type(col)}") + if row < 0 or col < 0: + raise ValueError(f"{self.name} cannot read pixel value at negative coordinates ({row}, {col})") + channels = key[2] if len(key) == 3 else None + return self.get_values_at_coords(row, col, channels) # Slicing if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)): - raise ValueError(f'"{key}"cannot be interpreted as valid slicing. Slicing should be 2D or 3D.') + raise ValueError(f'"{key}" cannot be interpreted as valid slicing. Slicing should be 2D or 3D.') if isinstance(key, tuple) and len(key) == 2: # Adding a 3rd dimension key = key + (slice(None, None, None),) @@ -430,7 +429,7 @@ class App(OTBObject): self.frozen = frozen self.quiet = quiet self.image_dic = image_dic - self._time_start, self._time_end = 0, 0 + self._time_start, self._time_end = 0., 0. self.data, self.parameters, self.exports_dic = {}, {}, {} # Initialize app, set parameters and execute if not frozen create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication @@ -446,14 +445,14 @@ class App(OTBObject): if any(key in self.parameters for key in self._out_param_types): self.flush() # auto flush if any output param was provided during app init - def get_first_key(self, *args: tuple[list[int]]) -> str: + def get_first_key(self, *type_lists: tuple[list[int]]) -> str: """Get the first param key for specific file types, try each list in args.""" - for param_types in args: + for param_types in type_lists: types = [getattr(otb, "ParameterType_" + key) for key in param_types] for key, value in sorted(self._all_param_types.items()): if value in types: return key - raise TypeError(f"{self.name}: could not find any parameter of type {tuple(chain(*args))}") + raise TypeError(f"{self.name}: could not find any parameter of type {tuple(chain(*type_lists))}") @property def input_key(self) -> str: @@ -608,17 +607,17 @@ class App(OTBObject): self._time_end = perf_counter() def write(self, *args, ext_fname: str = "", pixel_type: dict[str, str] | str = None, - preserve_dtype: bool = False, **kwargs): + preserve_dtype: bool = False, **kwargs, ) -> bool: """Set output pixel type and write the output raster files. Args: *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key contains non-standard characters such as a point, e.g. {'io.out':'output.tif'} - - string, useful when there is only one output, e.g. 'output.tif' + - filepath, useful when there is only one output, e.g. 'output.tif' - None if output file was passed during App init ext_fname: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES") Will be used for all outputs (Default value = "") - pixel_type: Can be : - dictionary {output_parameter_key: pixeltype} when specifying for several outputs + pixel_type: Can be : - dictionary {out_param_key: pixeltype} when specifying for several outputs - str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several outputs, all outputs are written with this unique type. Valid pixel types are uint8, uint16, uint32, int16, int32, float, double, @@ -626,6 +625,9 @@ class App(OTBObject): preserve_dtype: propagate main input pixel type to outputs, in case pixel_type is None **kwargs: keyword arguments e.g. out='output.tif' + Returns: + True if all files are found on disk + """ # Gather all input arguments in kwargs dict for arg in args: @@ -633,39 +635,43 @@ class App(OTBObject): kwargs.update(arg) elif isinstance(arg, str) and kwargs: logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg) - elif isinstance(arg, (str, Path)) and self.output_image_key: - kwargs.update({self.output_image_key: str(arg)}) + elif isinstance(arg, (str, Path)) and self.output_key: + kwargs.update({self.output_key: str(arg)}) + if not kwargs: + raise KeyError(f"{self.name}: at least one filepath is required, if not passed to App during init") + parameters = kwargs.copy() # Append filename extension to filenames if ext_fname: logger.debug("%s: using extended filename for outputs: %s", self.name, ext_fname) if not ext_fname.startswith("?"): ext_fname = "?&" + ext_fname + elif not ext_fname.startswith("?&"): + ext_fname = "?&" + ext_fname[1:] for key, value in kwargs.items(): if self._out_param_types[key] == otb.ParameterType_OutputImage and "?" not in value: - kwargs[key] = value + ext_fname - + parameters[key] = value + ext_fname # Manage output pixel types - dtypes = {} - if pixel_type: + data_types = {} + if pixel_type is not None: if isinstance(pixel_type, str): - type_name = self.app.ConvertPixelTypeToNumpy(parse_pixel_type(pixel_type)) + dtype = parse_pixel_type(pixel_type) + type_name = self.app.ConvertPixelTypeToNumpy(dtype) logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name) - for key in kwargs: + for key in parameters: if self._out_param_types[key] == otb.ParameterType_OutputImage: - dtypes[key] = parse_pixel_type(pixel_type) + data_types[key] = dtype elif isinstance(pixel_type, dict): - dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()} + data_types = {key: parse_pixel_type(dtype) for key, dtype in pixel_type.items()} elif preserve_dtype: self.propagate_dtype() # all outputs will have the same type as the main input raster - if not kwargs: - raise KeyError(f"{self.name}: at least one filepath is required, if not passed to App during init") + # Set parameters and flush to disk - for key, output_filename in kwargs.items(): + for key, output_filename in parameters.items(): if Path(output_filename.split("?")[0]).exists(): logger.warning("%s: overwriting file %s", self.name, output_filename) - if key in dtypes: - self.propagate_dtype(key, dtypes[key]) + if key in data_types: + self.propagate_dtype(key, data_types[key]) self.set_parameters({key: output_filename}) self.flush() @@ -1153,7 +1159,7 @@ class Output(OTBObject): return str(self.filepath) -def get_nbchannels(inp: str | OTBObject) -> int: +def get_nbchannels(inp: str | Path | OTBObject) -> int: """Get the nb of bands of input image. Args: @@ -1164,18 +1170,18 @@ def get_nbchannels(inp: str | OTBObject) -> int: """ if isinstance(inp, OTBObject): - nb_channels = inp.shape[-1] - else: + return inp.shape[-1] + if isinstance(inp, (str, Path)): # Executing the app, without printing its log try: info = App("ReadImageInfo", inp, quiet=True) - nb_channels = info.app.GetParameterInt("numberbands") + return info["numberbands"] except RuntimeError as info_err: # this happens when we pass a str that is not a filepath - raise TypeError(f"Could not get the number of channels of '{inp}' ({info_err})") from info_err - return nb_channels + raise TypeError(f"Could not get the number of channels file '{inp}' ({info_err})") from info_err + raise TypeError(f"Can't read number of channels of type '{type(inp)}' object {inp}") -def get_pixel_type(inp: str | OTBObject) -> str: +def get_pixel_type(inp: str | Path | OTBObject) -> str: """Get the encoding of input image pixels. Args: @@ -1186,33 +1192,30 @@ def get_pixel_type(inp: str | OTBObject) -> str: For an OTBObject with several outputs, only the pixel type of the first output is returned """ - if isinstance(inp, str): + if isinstance(inp, OTBObject): + return inp.app.GetParameterOutputImagePixelType(inp.output_image_key) + if isinstance(inp, (str, Path)): try: info = App("ReadImageInfo", inp, quiet=True) + datatype = info["datatype"] # which is such as short, float... except RuntimeError as info_err: # this happens when we pass a str that is not a filepath raise TypeError(f"Could not get the pixel type of `{inp}` ({info_err})") from info_err - datatype = info.app.GetParameterString("datatype") # which is such as short, float... - if not datatype: - raise TypeError(f"Unable to read pixel type of image {inp}") - datatype_to_pixeltype = { - "unsigned_char": "uint8", - "short": "int16", - "unsigned_short": "uint16", - "int": "int32", - "unsigned_int": "uint32", - "long": "int32", - "ulong": "uint32", - "float": "float", - "double": "double", - } - if datatype not in datatype_to_pixeltype: - raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}") - pixel_type = getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[datatype]}") - elif isinstance(inp, OTBObject): - pixel_type = inp.app.GetParameterOutputImagePixelType(inp.output_image_key) - else: - raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}") - return pixel_type + if datatype: + datatype_to_pixeltype = { + "unsigned_char": "uint8", + "short": "int16", + "unsigned_short": "uint16", + "int": "int32", + "unsigned_int": "uint32", + "long": "int32", + "ulong": "uint32", + "float": "float", + "double": "double", + } + if datatype not in datatype_to_pixeltype: + raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}") + return getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[datatype]}") + raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}") def parse_pixel_type(pixel_type: str | int) -> int: -- GitLab From b2cf422165144f872fbdd3cd8d515a1ce6bf4cc2 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 5 Feb 2023 14:38:40 +0100 Subject: [PATCH 130/399] FIX: Outputs, save_objects and getittem, add App.outputs dict --- pyotb/core.py | 63 ++++++++++++++++++++------------------------- tests/tests_data.py | 6 ++--- 2 files changed, 31 insertions(+), 38 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 364c7c7..f7447ad 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -259,12 +259,11 @@ class OTBObject(ABC): # Accessing pixel value(s) using Y/X coordinates if isinstance(key, tuple) and len(key) >= 2: row, col = key[0], key[1] - if not isinstance(row, int) or not isinstance(col, int): - raise TypeError(f"{self.name}: cannnot read pixel values using {type(row)} and {type(col)}") - if row < 0 or col < 0: - raise ValueError(f"{self.name} cannot read pixel value at negative coordinates ({row}, {col})") - channels = key[2] if len(key) == 3 else None - return self.get_values_at_coords(row, col, channels) + if isinstance(row, int) and isinstance(col, int): + if row < 0 or col < 0: + raise ValueError(f"{self.name} cannot read pixel value at negative coordinates ({row}, {col})") + channels = key[2] if len(key) == 3 else None + return self.get_values_at_coords(row, col, channels) # Slicing if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)): raise ValueError(f'"{key}" cannot be interpreted as valid slicing. Slicing should be 2D or 3D.') @@ -425,12 +424,10 @@ class App(OTBObject): e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif' """ - self.name = name - self.frozen = frozen - self.quiet = quiet - self.image_dic = image_dic + self.name, self.image_dic = name, image_dic + self.quiet, self.frozen = quiet, frozen self._time_start, self._time_end = 0., 0. - self.data, self.parameters, self.exports_dic = {}, {}, {} + self.data, self.parameters, self.outputs, self.exports_dic = {}, {}, {}, {} # Initialize app, set parameters and execute if not frozen create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication self.app = create(name) @@ -486,7 +483,7 @@ class App(OTBObject): @property def used_outputs(self) -> list[str]: """List of used application outputs.""" - return [getattr(self, key) for key in self._out_param_types if key in self.parameters] + return [self.outputs[key] for key in self._out_param_types if key in self.parameters] def set_parameters(self, *args, **kwargs): """Set some parameters of the app. @@ -527,7 +524,7 @@ class App(OTBObject): ) from e # Update param dict and save values as object attributes self.parameters.update(parameters) - self.save_objects() + self.save_objects(list(parameters)) def propagate_dtype(self, target_key: str = None, dtype: int = None): """Propagate a pixel type from main input to every outputs, or to a target output key only. @@ -559,18 +556,20 @@ class App(OTBObject): for key in keys: self.app.SetParameterOutputImagePixelType(key, dtype) - def save_objects(self): - """Saving OTB app parameters and outputs in data and parameters dict.""" - for key in self.parameters_keys: + def save_objects(self, keys: list[str] = None): + """Save OTB app values in data, parameters and outputs dict, for a list of keys or all parameters.""" + keys = keys or self.parameters_keys + for key in keys: value = self.parameters.get(key) if value is None: try: value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results) - if isinstance(value, otb.ApplicationProxy): - continue except RuntimeError: continue # this is when there is no value for key - # Save app data output or update parameters dict with otb.Application values + if value is None or isinstance(value, otb.ApplicationProxy): + continue + if self._out_param_types.get(key) == otb.ParameterType_OutputImage: + self.outputs[key] = Output(self, key, value) if isinstance(value, OTBObject) or bool(value) or value == 0: if self.app.GetParameterRole(key) == 0: self.parameters[key] = value @@ -674,21 +673,14 @@ class App(OTBObject): self.propagate_dtype(key, data_types[key]) self.set_parameters({key: output_filename}) self.flush() - - def find_outputs(self) -> tuple[str]: - """Find output files on disk using path found in parameters. - - Returns: - list of files found on disk - - """ files, missing = [], [] for out in self.used_outputs: dest = files if out.exists() else missing dest.append(str(out.filepath.absolute())) for filename in missing: logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) - return tuple(files) + raise Exception + return bool(files) and not missing # Private functions def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]: @@ -744,16 +736,17 @@ class App(OTBObject): We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer """ + if isinstance(key, tuple): + return super().__getitem__(key) if isinstance(key, str): - if key in self._out_param_types: - return Output(self, key, self.parameters.get(key)) - if key in self.parameters: - return self.parameters[key] if key in self.data: return self.data[key] - raise KeyError(f"{self.name}: unknown parameter '{key}'") - return super().__getitem__(key) - + if key in self.outputs: + return self.outputs[key] + if key in self.parameters: + return self.parameters[key] + raise KeyError(f"{self.name}: unknown or undefined parameter '{key}'") + raise TypeError(f"{self.name}: cannot access object item or slice using {type(key)} object") class Slicer(App): """Slicer objects i.e. when we call something like raster[:, :, 2] from Python.""" diff --git a/tests/tests_data.py b/tests/tests_data.py index f69edac..de5d727 100644 --- a/tests/tests_data.py +++ b/tests/tests_data.py @@ -15,7 +15,7 @@ SIMPLE_SERIALIZATION = { 'parameters': {'io.in': {'name': 'BandMath', 'parameters': {'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'], 'exp': 'im1b1', 'ram': 256}}, - 'map.utm.zone': 31, 'map.utm.northhem': False, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, + 'map.utm.zone': 31, 'map.utm.northhem': True, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, 'elev.default': 0.0, 'interpolator.bco.radius': 2, 'opt.rpc': 10, 'opt.ram': 256, 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5, 'outputs.sizex': 251, 'outputs.sizey': 304, 'outputs.spacingx': 5.997312068939209, 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}}, @@ -28,7 +28,7 @@ COMPLEX_SERIALIZATION = { 'parameters': { 'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'], 'exp': 'im1b1', 'ram': 256}}, - 'map.utm.zone': 31, 'map.utm.northhem': False, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, 'elev.default': 0.0, + 'map.utm.zone': 31, 'map.utm.northhem': True, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, 'elev.default': 0.0, 'interpolator.bco.radius': 2, 'opt.rpc': 10, 'opt.ram': 256, 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5, 'outputs.sizex': 251, 'outputs.sizey': 304, 'outputs.spacingx': 5.997312068939209, 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}}, {'name': 'ManageNoData', @@ -36,7 +36,7 @@ COMPLEX_SERIALIZATION = { 'parameters': {'io.in': {'name': 'BandMath', 'parameters': {'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'], 'exp': 'im1b1', 'ram': 256}}, - 'map.utm.zone': 31, 'map.utm.northhem': False, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, 'elev.default': 0.0, 'interpolator.bco.radius': 2, + 'map.utm.zone': 31, 'map.utm.northhem': True, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, 'elev.default': 0.0, 'interpolator.bco.radius': 2, 'opt.rpc': 10, 'opt.ram': 256, 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5, 'outputs.sizex': 251, 'outputs.sizey': 304, 'outputs.spacingx': 5.997312068939209, 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}}, -- GitLab From cbb49030e994c9f64e7aa128504a958e86c56b90 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 5 Feb 2023 14:43:39 +0100 Subject: [PATCH 131/399] STYLE: remove test Exception --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index f7447ad..c8ff8eb 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -679,7 +679,6 @@ class App(OTBObject): dest.append(str(out.filepath.absolute())) for filename in missing: logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) - raise Exception return bool(files) and not missing # Private functions @@ -748,6 +747,7 @@ class App(OTBObject): raise KeyError(f"{self.name}: unknown or undefined parameter '{key}'") raise TypeError(f"{self.name}: cannot access object item or slice using {type(key)} object") + class Slicer(App): """Slicer objects i.e. when we call something like raster[:, :, 2] from Python.""" -- GitLab From f008eb0550450b38914303320383343b8935a863 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 5 Feb 2023 15:01:41 +0100 Subject: [PATCH 132/399] ENH: remove find_outputs, just check file exists during write() --- pyotb/core.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index c8ff8eb..695039e 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -638,8 +638,8 @@ class App(OTBObject): kwargs.update({self.output_key: str(arg)}) if not kwargs: raise KeyError(f"{self.name}: at least one filepath is required, if not passed to App during init") - parameters = kwargs.copy() + # Append filename extension to filenames if ext_fname: logger.debug("%s: using extended filename for outputs: %s", self.name, ext_fname) @@ -673,10 +673,11 @@ class App(OTBObject): self.propagate_dtype(key, data_types[key]) self.set_parameters({key: output_filename}) self.flush() + # Search and log missing files files, missing = [], [] - for out in self.used_outputs: - dest = files if out.exists() else missing - dest.append(str(out.filepath.absolute())) + for key, output_filename in parameters.items(): + dest = files if Path(output_filename).exists() else missing + dest.append(str(Path(output_filename).absolute())) for filename in missing: logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) return bool(files) and not missing -- GitLab From 427c44aed88c231903d77fefcbb68769bac04123 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 5 Feb 2023 15:08:01 +0100 Subject: [PATCH 133/399] ENH: update tests, now write returns True if file exist --- pyotb/core.py | 5 +++-- tests/test_core.py | 23 +++++++---------------- tests/test_pipeline.py | 11 ++++------- 3 files changed, 14 insertions(+), 25 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 695039e..d6b66a8 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -676,8 +676,9 @@ class App(OTBObject): # Search and log missing files files, missing = [], [] for key, output_filename in parameters.items(): - dest = files if Path(output_filename).exists() else missing - dest.append(str(Path(output_filename).absolute())) + filepath = Path(output_filename.split("?")[0]) + dest = files if filepath.exists() else missing + dest.append(str(filepath.absolute())) for filename in missing: logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) return bool(files) and not missing diff --git a/tests/test_core.py b/tests/test_core.py index 1b6441f..824d6ad 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -69,14 +69,12 @@ def test_xy_to_rowcol(): def test_write(): - INPUT.write("/tmp/test_write.tif", ext_fname="nodata=0") - assert INPUT["out"].exists() + assert INPUT.write("/tmp/test_write.tif", ext_fname="nodata=0") INPUT["out"].filepath.unlink() def test_output_write(): - INPUT["out"].write("/tmp/test_output_write.tif") - assert INPUT["out"].exists() + assert INPUT["out"].write("/tmp/test_output_write.tif") INPUT["out"].filepath.unlink() @@ -151,28 +149,21 @@ def test_ndvi_comparison(): ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (INPUT[:, :, -1] + INPUT[:, :, 0]) ndvi_indices = pyotb.RadiometricIndices(INPUT, {"list": ["Vegetation:NDVI"], "channels.red": 1, "channels.nir": 4}) assert ndvi_bandmath.exp == "((im1b4 - im1b1) / (im1b4 + im1b1))" - - ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", pixel_type="float") - assert ndvi_bandmath["out"].exists() - ndvi_indices.write("/tmp/ndvi_indices.tif", pixel_type="float") - assert ndvi_indices["out"].exists() + assert ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", pixel_type="float") + assert ndvi_indices.write("/tmp/ndvi_indices.tif", pixel_type="float") compared = pyotb.CompareImages({"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"}) assert (compared["count"], compared["mse"]) == (0, 0) - thresholded_indices = pyotb.where(ndvi_indices >= 0.3, 1, 0) assert thresholded_indices.exp == "((im1b1 >= 0.3) ? 1 : 0)" - thresholded_bandmath = pyotb.where(ndvi_bandmath >= 0.3, 1, 0) assert thresholded_bandmath.exp == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)" def test_output_in_arg(): - o = pyotb.Output(INPUT, "out") - t = pyotb.ReadImageInfo(o) - assert t + t = pyotb.ReadImageInfo(INPUT["out"]) + assert t.data def test_output_summary(): - o = pyotb.Output(INPUT, "out") - assert o.summarize() + assert INPUT["out"].summarize() diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 19160c5..eb3b51f 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -123,18 +123,16 @@ def test_pipeline_write(pipe): out = f"/tmp/out_{i}.tif" if os.path.isfile(out): os.remove(out) - app.write(out) - assert os.path.isfile(out) + assert app.write(out) @pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES) def test_pipeline_write_nointermediate(pipe): app = [pipe[-1]][0] - out = f"/tmp/out_0.tif" + out = "/tmp/out_0.tif" if os.path.isfile(out): os.remove(out) - app.write(out) - assert os.path.isfile(out) + assert app.write(out) @pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES) @@ -143,5 +141,4 @@ def test_pipeline_write_backward(pipe): out = f"/tmp/out_{i}.tif" if os.path.isfile(out): os.remove(out) - app.write(out) - assert os.path.isfile(out) + assert app.write(out) -- GitLab From c109323c9974cf35fd1dd543b6d3eec445f257ed Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 5 Feb 2023 15:12:06 +0100 Subject: [PATCH 134/399] ENH: move summarize back to App, fix Output summarize parent app --- pyotb/core.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index d6b66a8..2092d4a 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -196,22 +196,6 @@ class OTBObject(ABC): row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x return abs(int(row)), int(col) - def summarize(self) -> dict[str, str | dict[str, Any]]: - """Serialize an object and its pipeline into a dictionary. - - Returns: - nested dictionary summarizing the pipeline - - """ - parameters = self.parameters.copy() - for key, param in parameters.items(): - # In the following, we replace each parameter which is an OTBObject, with its summary. - if isinstance(param, OTBObject): # single parameter - parameters[key] = param.summarize() - elif isinstance(param, list): # parameter list - parameters[key] = [p.summarize() if isinstance(p, OTBObject) else p for p in param] - return {"name": self.app.GetName(), "parameters": parameters} - @staticmethod def __create_operator(op_cls, name, x, y) -> Operation: """Create an operator. @@ -683,6 +667,22 @@ class App(OTBObject): logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) return bool(files) and not missing + def summarize(self) -> dict[str, str | dict[str, Any]]: + """Serialize an object and its pipeline into a dictionary. + + Returns: + nested dictionary summarizing the pipeline + + """ + parameters = self.parameters.copy() + for key, param in parameters.items(): + # In the following, we replace each parameter which is an OTBObject, with its summary. + if isinstance(param, OTBObject): # single parameter + parameters[key] = param.summarize() + elif isinstance(param, list): # parameter list + parameters[key] = [p.summarize() if isinstance(p, OTBObject) else p for p in param] + return {"name": self.app.GetName(), "parameters": parameters} + # Private functions def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]: """Gather all input arguments in kwargs dict. @@ -1149,6 +1149,10 @@ class Output(OTBObject): return self.parent_pyotb_app.write({self.output_image_key: self.filepath}, **kwargs) return self.parent_pyotb_app.write({self.output_image_key: filepath}, **kwargs) + def summarize(self): + """Summarize parent pyotb App.""" + return self.parent_pyotb_app.summarize() + def __str__(self) -> str: """Return string representation of Output filepath.""" return str(self.filepath) -- GitLab From 394e8d148bfa7a9111c73696cc02f6e5ff6cdbb8 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 5 Feb 2023 15:29:16 +0100 Subject: [PATCH 135/399] ENH: save Output obj before skipping empty key --- pyotb/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 2092d4a..7a77c30 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -549,11 +549,11 @@ class App(OTBObject): try: value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results) except RuntimeError: - continue # this is when there is no value for key - if value is None or isinstance(value, otb.ApplicationProxy): - continue + pass # this is when there is no value for key if self._out_param_types.get(key) == otb.ParameterType_OutputImage: self.outputs[key] = Output(self, key, value) + if value is None or isinstance(value, otb.ApplicationProxy): + continue if isinstance(value, OTBObject) or bool(value) or value == 0: if self.app.GetParameterRole(key) == 0: self.parameters[key] = value -- GitLab From 0ff7abd62c031ebdc4910dec285e00f8fd4a3a9a Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 5 Feb 2023 15:51:54 +0100 Subject: [PATCH 136/399] CI: linter --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 7a77c30..17db15f 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -549,7 +549,7 @@ class App(OTBObject): try: value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results) except RuntimeError: - pass # this is when there is no value for key + pass # undefined parameter if self._out_param_types.get(key) == otb.ParameterType_OutputImage: self.outputs[key] = Output(self, key, value) if value is None or isinstance(value, otb.ApplicationProxy): -- GitLab From 9082d0b7da3acd5411e0a75650387fbf1ad37b30 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 5 Feb 2023 16:51:37 +0100 Subject: [PATCH 137/399] ENH: remove used_outputs --- pyotb/core.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 17db15f..170ee9a 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -464,11 +464,6 @@ class App(OTBObject): """Get elapsed time between app init and end of exec or file writing.""" return self._time_end - self._time_start - @property - def used_outputs(self) -> list[str]: - """List of used application outputs.""" - return [self.outputs[key] for key in self._out_param_types if key in self.parameters] - def set_parameters(self, *args, **kwargs): """Set some parameters of the app. -- GitLab From bd4b6e1ec936186b5fc0e78e864ebeb1e8d1bb0d Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 5 Feb 2023 17:35:16 +0100 Subject: [PATCH 138/399] ENH: remove image_dic --- pyotb/core.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 170ee9a..bd3013a 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -163,8 +163,7 @@ class OTBObject(ABC): """ data = self.export(key, preserve_dtype) - array = data["array"] - return array.copy() if copy else array + return data["array"].copy() if copy else data["array"] def to_rasterio(self) -> tuple[np.ndarray, dict[str, Any]]: """Export image as a numpy array and its metadata compatible with rasterio. @@ -374,7 +373,7 @@ class OTBObject(ABC): result_dic = image_dic result_dic["array"] = result_array # Importing back to OTB, pass the result_dic just to keep reference - pyotb_app = App("ExtractROI", image_dic=result_dic, frozen=True, quiet=True) + pyotb_app = App("ExtractROI", frozen=True, quiet=True) if result_array.shape[2] == 1: pyotb_app.app.ImportImage("in", result_dic) else: @@ -387,7 +386,7 @@ class OTBObject(ABC): class App(OTBObject): """Base class that gathers common operations for any OTB application.""" - def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, **kwargs): + def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, **kwargs): """Common constructor for OTB applications. Handles in-memory connection between apps. Args: @@ -399,32 +398,28 @@ class App(OTBObject): - list, useful when the user wants to specify the input list 'il' frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___ quiet: whether to print logs of the OTB app - image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as - the result of app.ExportImage(). Use it when the app takes a numpy array as input. - See this related issue for why it is necessary to keep reference of object: - https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824 **kwargs: used for passing application parameters. e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif' """ - self.name, self.image_dic = name, image_dic + self.name = name self.quiet, self.frozen = quiet, frozen - self._time_start, self._time_end = 0., 0. - self.data, self.parameters, self.outputs, self.exports_dic = {}, {}, {}, {} - # Initialize app, set parameters and execute if not frozen - create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication - self.app = create(name) + self.data, self.parameters = {}, {} # params from self.app.GetParameterValue() + self.outputs, self.exports_dic = {}, {} # Outputs objects and numpy arrays exports + self.app = otb.Registry.CreateApplicationWithoutLogger(name) if quiet else otb.Registry.CreateApplication(name) self.parameters_keys = tuple(self.app.GetParametersKeys()) self._all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys} types = (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename) self._out_param_types = {k: v for k, v in self._all_param_types.items() if v in types} + # Init, execute and write (auto flush only when output param was provided) + self._time_start, self._time_end = 0., 0. if args or kwargs: self.set_parameters(*args, **kwargs) if not self.frozen: self.execute() if any(key in self.parameters for key in self._out_param_types): - self.flush() # auto flush if any output param was provided during app init + self.flush() def get_first_key(self, *type_lists: tuple[list[int]]) -> str: """Get the first param key for specific file types, try each list in args.""" @@ -733,7 +728,7 @@ class App(OTBObject): We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer """ if isinstance(key, tuple): - return super().__getitem__(key) + return super().__getitem__(key) # to read pixel values, or slice if isinstance(key, str): if key in self.data: return self.data[key] -- GitLab From 791da78558f2678489b3e6acef0df826bba5b31e Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 5 Feb 2023 17:38:19 +0100 Subject: [PATCH 139/399] STYLE: move functions --- pyotb/core.py | 91 +++++++++++++++++++++++++-------------------------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index bd3013a..298367c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -213,52 +213,6 @@ class OTBObject(ABC): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return op_cls(name, x, y) - # Special functions - def __hash__(self) -> int: - """Override the default behaviour of the hash function. - - Returns: - self hash - - """ - return id(self) - - def __getitem__(self, key) -> Any | list[float] | float | Slicer: - """Override the default __getitem__ behaviour. - - This function enables 2 things : - - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3] - selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]] - selecting 1000x1000 subset : object[:1000, :1000] - - access pixel value(s) at a specified row, col index - - Args: - key: attribute key - - Returns: - list of pixel values if vector image, or pixel value, or Slicer - - """ - # Accessing pixel value(s) using Y/X coordinates - if isinstance(key, tuple) and len(key) >= 2: - row, col = key[0], key[1] - if isinstance(row, int) and isinstance(col, int): - if row < 0 or col < 0: - raise ValueError(f"{self.name} cannot read pixel value at negative coordinates ({row}, {col})") - channels = key[2] if len(key) == 3 else None - return self.get_values_at_coords(row, col, channels) - # Slicing - if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)): - raise ValueError(f'"{key}" cannot be interpreted as valid slicing. Slicing should be 2D or 3D.') - if isinstance(key, tuple) and len(key) == 2: - # Adding a 3rd dimension - key = key + (slice(None, None, None),) - return Slicer(self, *key) - - def __repr__(self) -> str: - """Return a nice string representation with object id.""" - return f"<pyotb.{self.__class__.__name__} object id {id(self)}>" - def __add__(self, other: OTBObject | str | int | float) -> Operation: """Addition.""" return self.__create_operator(Operation, "+", self, other) @@ -382,6 +336,51 @@ class OTBObject(ABC): return pyotb_app return NotImplemented + def __hash__(self) -> int: + """Override the default behaviour of the hash function. + + Returns: + self hash + + """ + return id(self) + + def __getitem__(self, key) -> Any | list[float] | float | Slicer: + """Override the default __getitem__ behaviour. + + This function enables 2 things : + - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3] + selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]] + selecting 1000x1000 subset : object[:1000, :1000] + - access pixel value(s) at a specified row, col index + + Args: + key: attribute key + + Returns: + list of pixel values if vector image, or pixel value, or Slicer + + """ + # Accessing pixel value(s) using Y/X coordinates + if isinstance(key, tuple) and len(key) >= 2: + row, col = key[0], key[1] + if isinstance(row, int) and isinstance(col, int): + if row < 0 or col < 0: + raise ValueError(f"{self.name} cannot read pixel value at negative coordinates ({row}, {col})") + channels = key[2] if len(key) == 3 else None + return self.get_values_at_coords(row, col, channels) + # Slicing + if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)): + raise ValueError(f'"{key}" cannot be interpreted as valid slicing. Slicing should be 2D or 3D.') + if isinstance(key, tuple) and len(key) == 2: + # Adding a 3rd dimension + key = key + (slice(None, None, None),) + return Slicer(self, *key) + + def __repr__(self) -> str: + """Return a nice string representation with object id.""" + return f"<pyotb.{self.__class__.__name__} object id {id(self)}>" + class App(OTBObject): """Base class that gathers common operations for any OTB application.""" -- GitLab From c8c74094254196bc5929b816028ab0a90e4edc7b Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 5 Feb 2023 18:57:39 +0100 Subject: [PATCH 140/399] FIX: Operation expressions are stored using OTBObject repr with id, instead of str --- pyotb/core.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 298367c..30b67e0 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -379,7 +379,7 @@ class OTBObject(ABC): def __repr__(self) -> str: """Return a nice string representation with object id.""" - return f"<pyotb.{self.__class__.__name__} object id {id(self)}>" + return f"<pyotb.{self.__class__.__name__} object, id {id(self)}>" class App(OTBObject): @@ -852,15 +852,15 @@ class Operation(App): # NB: the keys of the dictionary are strings-only, instead of 'complex' objects, to enable easy serialization self.im_dic = {} self.im_count = 1 - mapping_str_to_input = {} # to be able to retrieve the real python object from its string representation + map_repr_to_input = {} # to be able to retrieve the real python object from its string representation for inp in self.inputs: if not isinstance(inp, (int, float)): if str(inp) not in self.im_dic: - self.im_dic[str(inp)] = f"im{self.im_count}" - mapping_str_to_input[str(inp)] = inp + self.im_dic[repr(inp)] = f"im{self.im_count}" + map_repr_to_input[repr(inp)] = inp self.im_count += 1 # Getting unique image inputs, in the order im1, im2, im3 ... - self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)] + self.unique_inputs = [map_repr_to_input[id_str] for id_str in sorted(self.im_dic, key=self.im_dic.get)] self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands) appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX" # Execute app @@ -943,7 +943,7 @@ class Operation(App): one_band_exp = one_band_fake_exp for inp in self.inputs: # Replace the name of in-memory object (e.g. '<pyotb.App object>b1' by 'im1b1') - one_band_exp = one_band_exp.replace(str(inp), self.im_dic[str(inp)]) + one_band_exp = one_band_exp.replace(repr(inp), self.im_dic[repr(inp)]) exp_bands.append(one_band_exp) # Form the final expression (e.g. 'im1b1 + 1; im1b2 + 1') return exp_bands, ";".join(exp_bands) @@ -979,8 +979,8 @@ class Operation(App): inputs, nb_channels = x.input.inputs, x.input.nb_channels else: # Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1') - fake_exp = f"{x.input}b{x.one_band_sliced}" - inputs, nb_channels = [x.input], {x.input: 1} + fake_exp = f"{repr(x.input)}b{x.one_band_sliced}" + inputs, nb_channels = [x.input], {repr(x.input): 1} # For LogicalOperation, we save almost the same attributes as an Operation elif keep_logical and isinstance(x, LogicalOperation): fake_exp = x.logical_fake_exp_bands[band - 1] @@ -995,12 +995,12 @@ class Operation(App): # We go on with other inputs, i.e. pyotb objects, filepaths... else: # Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1') - fake_exp = f"{x}b{band}" - inputs, nb_channels = [x], {x: get_nbchannels(x)} + fake_exp = f"{repr(x)}b{band}" + inputs, nb_channels = [x], {repr(x): get_nbchannels(x)} return fake_exp, inputs, nb_channels - def __str__(self) -> str: + def __repr__(self) -> str: """Return a nice string representation with operator and object id.""" return f"<pyotb.Operation `{self.operator}` object, id {id(self)}>" @@ -1083,9 +1083,9 @@ class Input(App): self.propagate_dtype() self.execute() - def __str__(self) -> str: - """Return a nice string representation with file path.""" - return f"<pyotb.Input object from {self.filepath}>" + def __repr__(self) -> str: + """Return a string representation with file path, used in Operation to store file ref.""" + return f"<pyotb.Input object, from {self.filepath}>" class Output(OTBObject): -- GitLab From 0270376d522c0701958db9f7f2ccbe250a3430aa Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 5 Feb 2023 19:35:58 +0100 Subject: [PATCH 141/399] CI: merge serialize tests in test_core, add json for summarize() data --- .gitlab-ci.yml | 5 -- tests/serialized_apps.json | 132 ++++++++++++++++++++++++++++++++++++ tests/test_core.py | 33 +++++++-- tests/test_serialization.py | 21 ------ tests/tests_data.py | 43 +++--------- 5 files changed, 166 insertions(+), 68 deletions(-) create mode 100644 tests/serialized_apps.json delete mode 100644 tests/test_serialization.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a38149e..a60518f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -104,11 +104,6 @@ test_pipeline: script: - python3 -m pytest --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py -test_serialization: - extends: .tests - script: - - python3 -m pytest --color=yes --junitxml=test-serialization.xml tests/test_serialization.py - # -------------------------------------- Ship --------------------------------------- pages: diff --git a/tests/serialized_apps.json b/tests/serialized_apps.json new file mode 100644 index 0000000..088e853 --- /dev/null +++ b/tests/serialized_apps.json @@ -0,0 +1,132 @@ +{ + "SIMPLE": { + "name": "ManageNoData", + "parameters": { + "in": { + "name": "OrthoRectification", + "parameters": { + "io.in": { + "name": "BandMath", + "parameters": { + "il": [ + "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" + ], + "exp": "im1b1", + "ram": 256 + } + }, + "map.utm.zone": 31, + "map.utm.northhem": true, + "map.epsg.code": 4326, + "outputs.isotropic": true, + "outputs.default": 0.0, + "elev.default": 0.0, + "interpolator.bco.radius": 2, + "opt.rpc": 10, + "opt.ram": 256, + "opt.gridspacing": 4.0, + "outputs.ulx": 560000.8125, + "outputs.uly": 5495732.5, + "outputs.sizex": 251, + "outputs.sizey": 304, + "outputs.spacingx": 5.997312068939209, + "outputs.spacingy": -5.997312068939209, + "outputs.lrx": 561506.125, + "outputs.lry": 5493909.5 + } + }, + "usenan": false, + "mode.buildmask.inv": 1.0, + "mode.buildmask.outv": 0.0, + "mode.changevalue.newv": 0.0, + "mode.apply.ndval": 0.0, + "ram": 256 + } + }, + "COMPLEX": { + "name": "BandMathX", + "parameters": { + "il": [ + { + "name": "OrthoRectification", + "parameters": { + "io.in": { + "name": "BandMath", + "parameters": { + "il": [ + "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" + ], + "exp": "im1b1", + "ram": 256 + } + }, + "map.utm.zone": 31, + "map.utm.northhem": true, + "map.epsg.code": 4326, + "outputs.isotropic": true, + "outputs.default": 0.0, + "elev.default": 0.0, + "interpolator.bco.radius": 2, + "opt.rpc": 10, + "opt.ram": 256, + "opt.gridspacing": 4.0, + "outputs.ulx": 560000.8125, + "outputs.uly": 5495732.5, + "outputs.sizex": 251, + "outputs.sizey": 304, + "outputs.spacingx": 5.997312068939209, + "outputs.spacingy": -5.997312068939209, + "outputs.lrx": 561506.125, + "outputs.lry": 5493909.5 + } + }, + { + "name": "ManageNoData", + "parameters": { + "in": { + "name": "OrthoRectification", + "parameters": { + "io.in": { + "name": "BandMath", + "parameters": { + "il": [ + "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" + ], + "exp": "im1b1", + "ram": 256 + } + }, + "map.utm.zone": 31, + "map.utm.northhem": true, + "map.epsg.code": 4326, + "outputs.isotropic": true, + "outputs.default": 0.0, + "elev.default": 0.0, + "interpolator.bco.radius": 2, + "opt.rpc": 10, + "opt.ram": 256, + "opt.gridspacing": 4.0, + "outputs.ulx": 560000.8125, + "outputs.uly": 5495732.5, + "outputs.sizex": 251, + "outputs.sizey": 304, + "outputs.spacingx": 5.997312068939209, + "outputs.spacingy": -5.997312068939209, + "outputs.lrx": 561506.125, + "outputs.lry": 5493909.5 + } + }, + "usenan": false, + "mode.buildmask.inv": 1.0, + "mode.buildmask.outv": 0.0, + "mode.changevalue.newv": 0.0, + "mode.apply.ndval": 0.0, + "ram": 256 + } + } + ], + "exp": "im1+im2", + "ram": 256 + } + } +} \ No newline at end of file diff --git a/tests/test_core.py b/tests/test_core.py index 824d6ad..d3c7d59 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,7 +1,7 @@ import pytest import pyotb -from tests_data import INPUT, TEST_IMAGE_STATS +from tests_data import * # Input settings @@ -53,7 +53,7 @@ def test_elapsed_time(): # Other functions -def test_get_infos(): +def test_get_info(): infos = INPUT.get_info() assert (infos["sizex"], infos["sizey"]) == (251, 304) bm_infos = pyotb.BandMathX([INPUT], exp="im1")["out"].get_info() @@ -78,6 +78,15 @@ def test_output_write(): INPUT["out"].filepath.unlink() +def test_output_in_arg(): + t = pyotb.ReadImageInfo(INPUT["out"]) + assert t.data + + +def test_output_summary(): + assert INPUT["out"].summarize() + + # Slicer def test_slicer_shape(): extract = INPUT[:50, :60, :3] @@ -160,10 +169,20 @@ def test_ndvi_comparison(): assert thresholded_bandmath.exp == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)" -def test_output_in_arg(): - t = pyotb.ReadImageInfo(INPUT["out"]) - assert t.data +def test_pipeline_simple(): + # BandMath -> OrthoRectification -> ManageNoData + app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"}) + app2 = pyotb.OrthoRectification({"io.in": app1}) + app3 = pyotb.ManageNoData({"in": app2}) + summary = app3.summarize() + assert summary == SIMPLE_SERIALIZATION -def test_output_summary(): - assert INPUT["out"].summarize() +def test_pipeline_diamond(): + # Diamond graph + app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"}) + app2 = pyotb.OrthoRectification({"io.in": app1}) + app3 = pyotb.ManageNoData({"in": app2}) + app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"}) + summary = app4.summarize() + assert summary == COMPLEX_SERIALIZATION diff --git a/tests/test_serialization.py b/tests/test_serialization.py deleted file mode 100644 index 28aa3a7..0000000 --- a/tests/test_serialization.py +++ /dev/null @@ -1,21 +0,0 @@ -import pyotb -from tests_data import * - - -def test_pipeline_simple(): - # BandMath -> OrthoRectification -> ManageNoData - app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"}) - app2 = pyotb.OrthoRectification({"io.in": app1}) - app3 = pyotb.ManageNoData({"in": app2}) - summary = app3.summarize() - assert summary == SIMPLE_SERIALIZATION - - -def test_pipeline_diamond(): - # Diamond graph - app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"}) - app2 = pyotb.OrthoRectification({"io.in": app1}) - app3 = pyotb.ManageNoData({"in": app2}) - app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"}) - summary = app4.summarize() - assert summary == COMPLEX_SERIALIZATION diff --git a/tests/tests_data.py b/tests/tests_data.py index de5d727..bfb774b 100644 --- a/tests/tests_data.py +++ b/tests/tests_data.py @@ -1,45 +1,18 @@ +import json +from pathlib import Path import pyotb FILEPATH = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" INPUT = pyotb.Input(FILEPATH) - TEST_IMAGE_STATS = { 'out.mean': [79.5505, 109.225, 115.456, 249.349], 'out.min': [33, 64, 91, 47], 'out.max': [255, 255, 230, 255], 'out.std': [51.0754, 35.3152, 23.4514, 20.3827] } -SIMPLE_SERIALIZATION = { - 'name': 'ManageNoData', - 'parameters': {'in': {'name': 'OrthoRectification', - 'parameters': {'io.in': {'name': 'BandMath', - 'parameters': {'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'], - 'exp': 'im1b1', 'ram': 256}}, - 'map.utm.zone': 31, 'map.utm.northhem': True, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, - 'elev.default': 0.0, 'interpolator.bco.radius': 2, 'opt.rpc': 10, 'opt.ram': 256, - 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5, 'outputs.sizex': 251, 'outputs.sizey': 304, - 'outputs.spacingx': 5.997312068939209, 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}}, - 'usenan': False, 'mode.buildmask.inv': 1.0, 'mode.buildmask.outv': 0.0, 'mode.changevalue.newv': 0.0, 'mode.apply.ndval': 0.0, 'ram': 256} -} -COMPLEX_SERIALIZATION = { - 'name': 'BandMathX', - 'parameters': {'il': [{'name': 'OrthoRectification', - 'parameters': {'io.in': {'name': 'BandMath', - 'parameters': { - 'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'], - 'exp': 'im1b1', 'ram': 256}}, - 'map.utm.zone': 31, 'map.utm.northhem': True, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, 'elev.default': 0.0, - 'interpolator.bco.radius': 2, 'opt.rpc': 10, 'opt.ram': 256, 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5, - 'outputs.sizex': 251, 'outputs.sizey': 304, 'outputs.spacingx': 5.997312068939209, 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}}, - {'name': 'ManageNoData', - 'parameters': {'in': {'name': 'OrthoRectification', - 'parameters': {'io.in': {'name': 'BandMath', - 'parameters': {'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'], - 'exp': 'im1b1', 'ram': 256}}, - 'map.utm.zone': 31, 'map.utm.northhem': True, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, 'elev.default': 0.0, 'interpolator.bco.radius': 2, - 'opt.rpc': 10, 'opt.ram': 256, 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5, - 'outputs.sizex': 251, 'outputs.sizey': 304, 'outputs.spacingx': 5.997312068939209, - 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}}, - 'usenan': False, 'mode.buildmask.inv': 1.0, 'mode.buildmask.outv': 0.0, 'mode.changevalue.newv': 0.0, 'mode.apply.ndval': 0.0, 'ram': 256}}], - 'exp': 'im1+im2', 'ram': 256} -} + +json_file = Path(__file__).parent / "serialized_apps.json" +with json_file.open("r") as js: + data = json.load(js) +SIMPLE_SERIALIZATION = data["SIMPLE"] +COMPLEX_SERIALIZATION = data["COMPLEX"] -- GitLab From 938de5798bc8f464f5f61f880697b53ba923aff4 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 5 Feb 2023 20:56:38 +0100 Subject: [PATCH 142/399] STYLE: docstrings and type hints --- pyotb/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 30b67e0..eebae77 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -175,9 +175,9 @@ class OTBObject(ABC): """ profile = {} array = self.to_numpy(preserve_dtype=True, copy=False) - height, width, count = array.shape proj = self.app.GetImageProjection(self.output_image_key) profile.update({"crs": proj, "dtype": array.dtype, "transform": self.transform}) + height, width, count = array.shape profile.update({"count": count, "height": height, "width": width}) return np.moveaxis(array, 2, 0), profile @@ -378,7 +378,7 @@ class OTBObject(ABC): return Slicer(self, *key) def __repr__(self) -> str: - """Return a nice string representation with object id.""" + """Return a string representation with object id, this is a key used to store image ref in Operation dicts.""" return f"<pyotb.{self.__class__.__name__} object, id {id(self)}>" @@ -742,7 +742,7 @@ class App(OTBObject): class Slicer(App): """Slicer objects i.e. when we call something like raster[:, :, 2] from Python.""" - def __init__(self, obj: App | str, rows: int, cols: int, channels: int): + def __init__(self, obj: App | str, rows: slice, cols: slice, channels: slice | list[int] | int): """Create a slicer object, that can be used directly for writing or inside a BandMath. It contains : -- GitLab From 10346ab018152f418b07b04b85a5c811b9ac75fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr> Date: Mon, 6 Feb 2023 09:51:38 +0000 Subject: [PATCH 143/399] Apply 1 suggestion(s) to 1 file(s) --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index eebae77..309fda6 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -625,7 +625,7 @@ class App(OTBObject): parameters[key] = value + ext_fname # Manage output pixel types data_types = {} - if pixel_type is not None: + if pixel_type: if isinstance(pixel_type, str): dtype = parse_pixel_type(pixel_type) type_name = self.app.ConvertPixelTypeToNumpy(dtype) -- GitLab From b1b52e2c528518dd1bf8171446168fc689b99d6d Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Feb 2023 10:52:18 +0100 Subject: [PATCH 144/399] ENH: reuse parse_pixel_type in get_pixel_type, or raise exception --- pyotb/core.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 309fda6..c78b3dc 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1189,20 +1189,7 @@ def get_pixel_type(inp: str | Path | OTBObject) -> str: except RuntimeError as info_err: # this happens when we pass a str that is not a filepath raise TypeError(f"Could not get the pixel type of `{inp}` ({info_err})") from info_err if datatype: - datatype_to_pixeltype = { - "unsigned_char": "uint8", - "short": "int16", - "unsigned_short": "uint16", - "int": "int32", - "unsigned_int": "uint32", - "long": "int32", - "ulong": "uint32", - "float": "float", - "double": "double", - } - if datatype not in datatype_to_pixeltype: - raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}") - return getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[datatype]}") + return parse_pixel_type(datatype) raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}") @@ -1216,11 +1203,26 @@ def parse_pixel_type(pixel_type: str | int) -> int: pixel_type integer value """ - if isinstance(pixel_type, str): # this correspond to 'uint8' etc... - return getattr(otb, f"ImagePixelType_{pixel_type}") - if isinstance(pixel_type, int): + if isinstance(pixel_type, int): # normal OTB int enum return pixel_type - raise ValueError(f"Bad pixel type specification ({pixel_type})") + if isinstance(pixel_type, str): # correspond to 'uint8' etc... + datatype_to_pixeltype = { + "unsigned_char": "uint8", + "short": "int16", + "unsigned_short": "uint16", + "int": "int32", + "unsigned_int": "uint32", + "long": "int32", + "ulong": "uint32", + "float": "float", + "double": "double", + } + if pixel_type in datatype_to_pixeltype.keys(): + return getattr(otb, f"ImagePixelType_{pixel_type}") + if pixel_type in datatype_to_pixeltype: + return getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[pixel_type]}") + raise KeyError(f"Unknown data type `{pixel_type}`. Available ones: {datatype_to_pixeltype}") + raise TypeError(f"Bad pixel type specification ({pixel_type} of type {type(pixel_type)})") def is_key_list(pyotb_app: OTBObject, key: str) -> bool: -- GitLab From 212011170ab1f894fcd2df301f78ca63acd406ef Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Feb 2023 10:57:25 +0100 Subject: [PATCH 145/399] ENH: fix parse_pixel_type --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index c78b3dc..859c2e6 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1217,7 +1217,7 @@ def parse_pixel_type(pixel_type: str | int) -> int: "float": "float", "double": "double", } - if pixel_type in datatype_to_pixeltype.keys(): + if pixel_type in datatype_to_pixeltype.values(): return getattr(otb, f"ImagePixelType_{pixel_type}") if pixel_type in datatype_to_pixeltype: return getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[pixel_type]}") -- GitLab From 8072693f42847568e11a2e496baab68b966057d3 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Feb 2023 11:07:36 +0100 Subject: [PATCH 146/399] ENH: fix potential problems with Path objects and /vsi URLs --- pyotb/core.py | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 859c2e6..66a6906 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -639,19 +639,20 @@ class App(OTBObject): self.propagate_dtype() # all outputs will have the same type as the main input raster # Set parameters and flush to disk - for key, output_filename in parameters.items(): - if Path(output_filename.split("?")[0]).exists(): - logger.warning("%s: overwriting file %s", self.name, output_filename) + for key, filepath in parameters.items(): + if Path(filepath.split("?")[0]).exists(): + logger.warning("%s: overwriting file %s", self.name, filepath) if key in data_types: self.propagate_dtype(key, data_types[key]) - self.set_parameters({key: output_filename}) + self.set_parameters({key: filepath}) self.flush() # Search and log missing files files, missing = [], [] - for key, output_filename in parameters.items(): - filepath = Path(output_filename.split("?")[0]) - dest = files if filepath.exists() else missing - dest.append(str(filepath.absolute())) + for key, filepath in parameters.items(): + if not filepath.startswith("/vsi"): + filepath = Path(filepath.split("?")[0]) + dest = files if filepath.exists() else missing + dest.append(str(filepath.absolute())) for filename in missing: logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) return bool(files) and not missing @@ -1079,7 +1080,7 @@ class Input(App): """ super().__init__("ExtractROI", {"in": filepath}, frozen=True) self.name = f"Input from {filepath}" - self.filepath = Path(filepath) + self.filepath = Path(filepath) if not filepath.startswith("/vsi") else filepath self.propagate_dtype() self.execute() @@ -1091,7 +1092,7 @@ class Input(App): class Output(OTBObject): """Object that behave like a pointer to a specific application output file.""" - def __init__(self, pyotb_app: App, param_key: str = None, filepath: str = None, mkdir: bool = True): + def __init__(self, pyotb_app: App, param_key: str = None, filepath: str = "", mkdir: bool = True): """Constructor for an Output object. Args: @@ -1107,11 +1108,9 @@ class Output(OTBObject): self.exports_dic = pyotb_app.exports_dic self.param_key = param_key self.parameters = self.parent_pyotb_app.parameters - self.filepath = None - if filepath: - if "?" in filepath: - filepath = filepath.split("?")[0] - self.filepath = Path(filepath) + self.filepath = filepath + if not filepath.startswith("/vsi"): + self.filepath = Path(filepath.split("?")[0]) if mkdir: self.make_parent_dirs() @@ -1122,14 +1121,14 @@ class Output(OTBObject): def exists(self) -> bool: """Check file exist.""" - if self.filepath is None: - raise ValueError("Filepath is not set") + if not isinstance(self.filepath, Path): + raise ValueError("Filepath is not set or points to a remote URL") return self.filepath.exists() def make_parent_dirs(self): """Create missing parent directories.""" - if self.filepath is None: - raise ValueError("Filepath is not set") + if not isinstance(self.filepath, Path): + raise ValueError("Filepath is not set or points to a remote URL") self.filepath.parent.mkdir(parents=True, exist_ok=True) def write(self, filepath: None | str | Path = None, **kwargs): -- GitLab From 70bb5e74bf047cb0d51b52d8e748976c7c0bbf64 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Feb 2023 12:00:43 +0100 Subject: [PATCH 147/399] ENH: add case filepath is not set in Output --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 66a6906..7fd543c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1109,7 +1109,7 @@ class Output(OTBObject): self.param_key = param_key self.parameters = self.parent_pyotb_app.parameters self.filepath = filepath - if not filepath.startswith("/vsi"): + if filepath and not filepath.startswith("/vsi"): self.filepath = Path(filepath.split("?")[0]) if mkdir: self.make_parent_dirs() -- GitLab From f98710283180d2b09d461bf996dfc4a2f8caa429 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Feb 2023 12:15:34 +0100 Subject: [PATCH 148/399] ENH: back to OTB enum in get_first_key --- pyotb/core.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 7fd543c..465dbbe 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -423,35 +423,36 @@ class App(OTBObject): def get_first_key(self, *type_lists: tuple[list[int]]) -> str: """Get the first param key for specific file types, try each list in args.""" for param_types in type_lists: - types = [getattr(otb, "ParameterType_" + key) for key in param_types] for key, value in sorted(self._all_param_types.items()): - if value in types: + if value in param_types: return key - raise TypeError(f"{self.name}: could not find any parameter of type {tuple(chain(*type_lists))}") + raise KeyError(f"{self.name}: could not find any parameter key matching the provided types") @property def input_key(self) -> str: """Get the name of first input parameter, raster > vector > file.""" return self.get_first_key( - ["InputImage", "InputImageList"], - ["InputVectorData", "InputVectorDataList"], - ["InputFilename", "InputFilenameList"], + [otb.ParameterType_InputImage, otb.ParameterType_InputImageList], + [otb.ParameterType_InputVectorData, otb.ParameterType_InputVectorDataList], + [otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList], ) @property def input_image_key(self) -> str: """Name of the first input image parameter.""" - return self.get_first_key(["InputImage", "InputImageList"]) + return self.get_first_key([otb.ParameterType_InputImage, otb.ParameterType_InputImageList]) @property def output_key(self) -> str: """Name of the first output parameter, raster > vector > file.""" - return self.get_first_key(["OutputImage"], ["OutputVectorData"], ["OutputFilename"]) + return self.get_first_key( + [otb.ParameterType_OutputImage], [otb.ParameterType_OutputVectorData], [otb.ParameterType_OutputFilename] + ) @property def output_image_key(self) -> str: """Get the name of first output image parameter.""" - return self.get_first_key(["OutputImage"]) + return self.get_first_key([otb.ParameterType_OutputImage]) @property def elapsed_time(self) -> float: -- GitLab From d18442bc406b9f4ab10e5140d6737d0fcbf6f080 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Feb 2023 12:16:20 +0100 Subject: [PATCH 149/399] ENH: gest_first_key raises TypeError --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 465dbbe..4f320f2 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -426,7 +426,7 @@ class App(OTBObject): for key, value in sorted(self._all_param_types.items()): if value in param_types: return key - raise KeyError(f"{self.name}: could not find any parameter key matching the provided types") + raise TypeError(f"{self.name}: could not find any parameter key matching the provided types") @property def input_key(self) -> str: -- GitLab From e216c2f8dcae089045528cd0e31bcde0c11d001b Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Feb 2023 12:25:26 +0100 Subject: [PATCH 150/399] STYLE: remove unused import from itertools --- pyotb/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 4f320f2..94b5f1e 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -5,7 +5,6 @@ from __future__ import annotations from ast import literal_eval from pathlib import Path from time import perf_counter -from itertools import chain from typing import Any from abc import ABC, abstractmethod -- GitLab From 55e34924dbd5ec718422b8b3b56e09c8f7274567 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Feb 2023 14:38:41 +0100 Subject: [PATCH 151/399] ENH: move summarize back to OTBObject --- pyotb/core.py | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 94b5f1e..b64c877 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -72,6 +72,22 @@ class OTBObject(ABC): origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2 return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y + def summarize(self) -> dict[str, str | dict[str, Any]]: + """Serialize an object and its pipeline into a dictionary. + + Returns: + nested dictionary summarizing the pipeline + + """ + parameters = self.parameters.copy() + for key, param in parameters.items(): + # In the following, we replace each parameter which is an OTBObject, with its summary. + if isinstance(param, OTBObject): # single parameter + parameters[key] = param.summarize() + elif isinstance(param, list): # parameter list + parameters[key] = [p.summarize() if isinstance(p, OTBObject) else p for p in param] + return {"name": self.app.GetName(), "parameters": parameters} + def get_info(self) -> dict[str, (str, float, list[float])]: """Return a dict output of ReadImageInfo for the first image output.""" return App("ReadImageInfo", self, quiet=True).data @@ -657,22 +673,6 @@ class App(OTBObject): logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) return bool(files) and not missing - def summarize(self) -> dict[str, str | dict[str, Any]]: - """Serialize an object and its pipeline into a dictionary. - - Returns: - nested dictionary summarizing the pipeline - - """ - parameters = self.parameters.copy() - for key, param in parameters.items(): - # In the following, we replace each parameter which is an OTBObject, with its summary. - if isinstance(param, OTBObject): # single parameter - parameters[key] = param.summarize() - elif isinstance(param, list): # parameter list - parameters[key] = [p.summarize() if isinstance(p, OTBObject) else p for p in param] - return {"name": self.app.GetName(), "parameters": parameters} - # Private functions def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]: """Gather all input arguments in kwargs dict. @@ -1137,10 +1137,6 @@ class Output(OTBObject): return self.parent_pyotb_app.write({self.output_image_key: self.filepath}, **kwargs) return self.parent_pyotb_app.write({self.output_image_key: filepath}, **kwargs) - def summarize(self): - """Summarize parent pyotb App.""" - return self.parent_pyotb_app.summarize() - def __str__(self) -> str: """Return string representation of Output filepath.""" return str(self.filepath) -- GitLab From 22b5e9746d338411aa545c4d869b62794709bbd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr> Date: Wed, 8 Feb 2023 13:27:04 +0000 Subject: [PATCH 152/399] More operators tests --- tests/test_core.py | 56 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index d3c7d59..4e98d50 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -45,7 +45,7 @@ def test_metadata(): def test_nonraster_property(): with pytest.raises(TypeError): - pyotb.ReadImageInfo(INPUT).dtype + assert pyotb.ReadImageInfo(INPUT).dtype == "uint8" def test_elapsed_time(): @@ -79,12 +79,8 @@ def test_output_write(): def test_output_in_arg(): - t = pyotb.ReadImageInfo(INPUT["out"]) - assert t.data - - -def test_output_summary(): - assert INPUT["out"].summarize() + info = pyotb.ReadImageInfo(INPUT["out"]) + assert info.data # Slicer @@ -109,6 +105,44 @@ def test_slicer_in_output(): # Arithmetic +def test_rational_operators(): + def _test(func, exp): + meas = func(INPUT) + ref = pyotb.BandMathX({"il": [FILEPATH], "exp": exp}) + for i in range(1, 5): + compared = pyotb.CompareImages({"ref.in": ref, "meas.in": meas, "ref.channel": i, "meas.channel": i}) + assert (compared["count"], compared["mse"]) == (0, 0) + + _test(lambda x: x + x, "im1 + im1") + _test(lambda x: x - x, "im1 - im1") + _test(lambda x: x / x, "im1 div im1") + _test(lambda x: x * x, "im1 mult im1") + _test(lambda x: x + FILEPATH, "im1 + im1") + _test(lambda x: x - FILEPATH, "im1 - im1") + _test(lambda x: x / FILEPATH, "im1 div im1") + _test(lambda x: x * FILEPATH, "im1 mult im1") + _test(lambda x: FILEPATH + x, "im1 + im1") + _test(lambda x: FILEPATH - x, "im1 - im1") + _test(lambda x: FILEPATH / x, "im1 div im1") + _test(lambda x: FILEPATH * x, "im1 mult im1") + _test(lambda x: x + 2, "im1 + {2;2;2;2}") + _test(lambda x: x - 2, "im1 - {2;2;2;2}") + _test(lambda x: x / 2, "0.5 * im1") + _test(lambda x: x * 2, "im1 * 2") + _test(lambda x: x + 2.0, "im1 + {2.0;2.0;2.0;2.0}") + _test(lambda x: x - 2.0, "im1 - {2.0;2.0;2.0;2.0}") + _test(lambda x: x / 2.0, "0.5 * im1") + _test(lambda x: x * 2.0, "im1 * 2.0") + _test(lambda x: 2 + x, "{2;2;2;2} + im1") + _test(lambda x: 2 - x, "{2;2;2;2} - im1") + _test(lambda x: 2 / x, "{2;2;2;2} div im1") + _test(lambda x: 2 * x, "2 * im1") + _test(lambda x: 2.0 + x, "{2.0;2.0;2.0;2.0} + im1") + _test(lambda x: 2.0 - x, "{2.0;2.0;2.0;2.0} - im1") + _test(lambda x: 2.0 / x, "{2.0;2.0;2.0;2.0} div im1") + _test(lambda x: 2.0 * x, "2.0 * im1") + + def test_operation(): op = INPUT / 255 * 128 assert op.exp == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)" @@ -164,9 +198,13 @@ def test_ndvi_comparison(): compared = pyotb.CompareImages({"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"}) assert (compared["count"], compared["mse"]) == (0, 0) thresholded_indices = pyotb.where(ndvi_indices >= 0.3, 1, 0) - assert thresholded_indices.exp == "((im1b1 >= 0.3) ? 1 : 0)" + assert thresholded_indices["exp"] == "((im1b1 >= 0.3) ? 1 : 0)" thresholded_bandmath = pyotb.where(ndvi_bandmath >= 0.3, 1, 0) - assert thresholded_bandmath.exp == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)" + assert thresholded_bandmath["exp"] == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)" + + +def test_output_summary(): + assert INPUT["out"].summarize() def test_pipeline_simple(): -- GitLab From 47fb1f471b4f948bc6d16008575da7f7826cbfdd Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 8 Feb 2023 18:48:47 +0000 Subject: [PATCH 153/399] Add more OTBObject abstract properties --- pyotb/core.py | 134 +++++++++++++++++++++++++++++++-------------- tests/test_core.py | 5 ++ 2 files changed, 98 insertions(+), 41 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index b64c877..4676c59 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -16,16 +16,30 @@ from .helpers import logger class OTBObject(ABC): """Abstraction of an image object.""" + @property + @abstractmethod + def name(self) -> str: + """By default, should return the application name, but a custom name may be passed during init.""" - name: str - app: otb.Application - parameters: dict - exports_dic: dict + @property + @abstractmethod + def app(self) -> otb.Application: + """Reference to the main (or last in pipeline) otb.Application instance linked to this object.""" @property @abstractmethod - def output_image_key(self): - """Returns the name of a parameter associated to an image. Property defined in App and Output.""" + def parameters(self) -> dict[str, Any]: + """Should return every input parameters of the main otb.Application instance linked to this object.""" + + @property + @abstractmethod + def output_image_key(self) -> str: + """Return the name of a parameter key associated to the main output image of the object.""" + + @property + @abstractmethod + def exports_dic(self) -> dict[str, dict]: + """Return an internal dict object containing np.array exports, to avoid duplicated ExportImage() calls.""" @abstractmethod def write(self): @@ -118,7 +132,7 @@ class OTBObject(ABC): elif isinstance(bands, slice): channels = self.channels_list_from_slice(bands) elif not isinstance(bands, list): - raise TypeError(f"{self.app.GetName()}: type '{type(bands)}' cannot be interpreted as a valid slicing") + raise TypeError(f"{self.name}: type '{type(bands)}' cannot be interpreted as a valid slicing") if channels: app.app.Execute() app.set_parameters({"cl": [f"Channel{n + 1}" for n in channels]}) @@ -141,7 +155,7 @@ class OTBObject(ABC): return list(range(0, stop, step)) if start is None and stop is None: return list(range(0, nb_channels, step)) - raise ValueError(f"{self.app.GetName()}: '{bands}' cannot be interpreted as valid slicing.") + raise ValueError(f"{self.name}: '{bands}' cannot be interpreted as valid slicing.") def export(self, key: str = None, preserve_dtype: bool = True) -> dict[str, dict[str, np.ndarray]]: """Export a specific output image as numpy array and store it in object exports_dic. @@ -400,11 +414,11 @@ class OTBObject(ABC): class App(OTBObject): """Base class that gathers common operations for any OTB application.""" - def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, **kwargs): + def __init__(self, appname: str, *args, frozen: bool = False, quiet: bool = False, name: str = "", **kwargs): """Common constructor for OTB applications. Handles in-memory connection between apps. Args: - name: name of the app, e.g. 'BandMath' + appname: name of the OTB application to initialize, e.g. 'BandMath' *args: used for passing application parameters. Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") @@ -412,22 +426,26 @@ class App(OTBObject): - list, useful when the user wants to specify the input list 'il' frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___ quiet: whether to print logs of the OTB app + name: custom name that will show up in logs, appname will be used if not provided **kwargs: used for passing application parameters. e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif' """ - self.name = name + # Attributes and data structures used by properties + create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication + self._app = create(appname) + self._name = name or appname + self._time_start, self._time_end = 0.0, 0.0 + self._parameters, self._exports_dic = {}, {} + self.data, self.outputs = {}, {} self.quiet, self.frozen = quiet, frozen - self.data, self.parameters = {}, {} # params from self.app.GetParameterValue() - self.outputs, self.exports_dic = {}, {} # Outputs objects and numpy arrays exports - self.app = otb.Registry.CreateApplicationWithoutLogger(name) if quiet else otb.Registry.CreateApplication(name) + # Param keys and types self.parameters_keys = tuple(self.app.GetParametersKeys()) self._all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys} types = (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename) self._out_param_types = {k: v for k, v in self._all_param_types.items() if v in types} # Init, execute and write (auto flush only when output param was provided) - self._time_start, self._time_end = 0., 0. if args or kwargs: self.set_parameters(*args, **kwargs) if not self.frozen: @@ -435,13 +453,30 @@ class App(OTBObject): if any(key in self.parameters for key in self._out_param_types): self.flush() - def get_first_key(self, *type_lists: tuple[list[int]]) -> str: - """Get the first param key for specific file types, try each list in args.""" - for param_types in type_lists: - for key, value in sorted(self._all_param_types.items()): - if value in param_types: - return key - raise TypeError(f"{self.name}: could not find any parameter key matching the provided types") + @property + def name(self) -> str: + """Returns appname by default, or a custom name if passed during App init.""" + return self._name + + @property + def app(self) -> otb.Application: + """Property to return an internal _app instance.""" + return self._app + + @property + def parameters(self) -> dict[str, Any]: + """Property to return an internal _parameters dict instance.""" + return self._parameters + + @property + def output_image_key(self) -> str: + """Get the name of first output image parameter.""" + return self.get_first_key([otb.ParameterType_OutputImage]) + + @property + def exports_dic(self) -> dict[str, dict]: + """Returns internal _exports_dic object that contains numpy array exports.""" + return self._exports_dic @property def input_key(self) -> str: @@ -464,16 +499,19 @@ class App(OTBObject): [otb.ParameterType_OutputImage], [otb.ParameterType_OutputVectorData], [otb.ParameterType_OutputFilename] ) - @property - def output_image_key(self) -> str: - """Get the name of first output image parameter.""" - return self.get_first_key([otb.ParameterType_OutputImage]) - @property def elapsed_time(self) -> float: """Get elapsed time between app init and end of exec or file writing.""" return self._time_end - self._time_start + def get_first_key(self, *type_lists: tuple[list[int]]) -> str: + """Get the first param key for specific file types, try each list in args.""" + for param_types in type_lists: + for key, value in sorted(self._all_param_types.items()): + if value in param_types: + return key + raise TypeError(f"{self.name}: could not find any parameter key matching the provided types") + def set_parameters(self, *args, **kwargs): """Set some parameters of the app. @@ -512,7 +550,7 @@ class App(OTBObject): f"(while setting parameter '{key}' to '{obj}': {e})" ) from e # Update param dict and save values as object attributes - self.parameters.update(parameters) + self._parameters.update(parameters) self.save_objects(list(parameters)) def propagate_dtype(self, target_key: str = None, dtype: int = None): @@ -561,7 +599,7 @@ class App(OTBObject): continue if isinstance(value, OTBObject) or bool(value) or value == 0: if self.app.GetParameterRole(key) == 0: - self.parameters[key] = value + self._parameters[key] = value else: if isinstance(value, str): try: @@ -743,7 +781,7 @@ class App(OTBObject): class Slicer(App): """Slicer objects i.e. when we call something like raster[:, :, 2] from Python.""" - def __init__(self, obj: App | str, rows: slice, cols: slice, channels: slice | list[int] | int): + def __init__(self, obj: App, rows: slice, cols: slice, channels: slice | list[int] | int): """Create a slicer object, that can be used directly for writing or inside a BandMath. It contains : @@ -757,8 +795,7 @@ class Slicer(App): channels: channels, can be slicing, list or int """ - super().__init__("ExtractROI", obj, mode="extent", quiet=True, frozen=True) - self.name = "Slicer" + super().__init__("ExtractROI", obj, mode="extent", quiet=True, frozen=True, name=f"Slicer from {obj.name}") self.rows, self.cols = rows, cols parameters = {} @@ -864,9 +901,8 @@ class Operation(App): self.unique_inputs = [map_repr_to_input[id_str] for id_str in sorted(self.im_dic, key=self.im_dic.get)] self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands) appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX" - # Execute app - super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True) - self.name = name or f'Operation exp="{self.exp}"' + name = f'Operation exp="{self.exp}"' + super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True, name=name) def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None): """Create a list of 'fake' expressions, one for each band. @@ -1079,7 +1115,7 @@ class Input(App): """ super().__init__("ExtractROI", {"in": filepath}, frozen=True) - self.name = f"Input from {filepath}" + self._name = f"Input from {filepath}" self.filepath = Path(filepath) if not filepath.startswith("/vsi") else filepath self.propagate_dtype() self.execute() @@ -1102,18 +1138,34 @@ class Output(OTBObject): mkdir: create missing parent directories """ - self.name = f"Output {param_key} from {pyotb_app.name}" self.parent_pyotb_app = pyotb_app # keep trace of parent app - self.app = pyotb_app.app - self.exports_dic = pyotb_app.exports_dic self.param_key = param_key - self.parameters = self.parent_pyotb_app.parameters self.filepath = filepath if filepath and not filepath.startswith("/vsi"): self.filepath = Path(filepath.split("?")[0]) if mkdir: self.make_parent_dirs() + @property + def name(self): + """Return Output name containing filepath.""" + return f"Output {self.param_key} from {self.parent_pyotb_app.name}" + + @property + def app(self) -> otb.Application: + """Reference to the parent pyotb otb.Application instance.""" + return self.parent_pyotb_app.app + + @property + def parameters(self) -> dict[str, Any]: + """Reference to the parent pyotb App parameters.""" + return self.parent_pyotb_app.parameters + + @property + def exports_dic(self) -> dict[str, dict]: + """Returns internal _exports_dic object that contains numpy array exports.""" + return self.parent_pyotb_app.exports_dic + @property def output_image_key(self) -> str: """Force the right key to be used when accessing the OTBObject.""" @@ -1131,7 +1183,7 @@ class Output(OTBObject): raise ValueError("Filepath is not set or points to a remote URL") self.filepath.parent.mkdir(parents=True, exist_ok=True) - def write(self, filepath: None | str | Path = None, **kwargs): + def write(self, filepath: None | str | Path = None, **kwargs) -> bool: """Write output to disk, filepath is not required if it was provided to parent App during init.""" if filepath is None and self.filepath: return self.parent_pyotb_app.write({self.output_image_key: self.filepath}, **kwargs) diff --git a/tests/test_core.py b/tests/test_core.py index 4e98d50..143bf93 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -15,6 +15,11 @@ def test_wrong_key(): # OTBObject properties +def test_name(): + app = pyotb.App("BandMath", [INPUT], exp="im1b1", name="TestName") + assert app.name == "TestName" + + def test_key_input(): assert INPUT.input_key == INPUT.input_image_key == "in" -- GitLab From 724f7e897723bdb477c8e7936e08cdd2ca335438 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 14 Feb 2023 11:39:40 +0000 Subject: [PATCH 154/399] Remove unused params from App.parameters and summarize + create App.outputs objects only once --- pyotb/__init__.py | 3 +- pyotb/core.py | 164 +++++++++++++++++++------------------ tests/serialized_apps.json | 123 +++++++++++----------------- tests/test_core.py | 8 +- 4 files changed, 141 insertions(+), 157 deletions(-) diff --git a/pyotb/__init__.py b/pyotb/__init__.py index 8e22979..d4efd39 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -9,7 +9,8 @@ from .core import ( Input, Output, get_nbchannels, - get_pixel_type + get_pixel_type, + summarize ) from .functions import ( # pylint: disable=redefined-builtin all, diff --git a/pyotb/core.py b/pyotb/core.py index 4676c59..cd144f6 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -26,11 +26,6 @@ class OTBObject(ABC): def app(self) -> otb.Application: """Reference to the main (or last in pipeline) otb.Application instance linked to this object.""" - @property - @abstractmethod - def parameters(self) -> dict[str, Any]: - """Should return every input parameters of the main otb.Application instance linked to this object.""" - @property @abstractmethod def output_image_key(self) -> str: @@ -86,22 +81,6 @@ class OTBObject(ABC): origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2 return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y - def summarize(self) -> dict[str, str | dict[str, Any]]: - """Serialize an object and its pipeline into a dictionary. - - Returns: - nested dictionary summarizing the pipeline - - """ - parameters = self.parameters.copy() - for key, param in parameters.items(): - # In the following, we replace each parameter which is an OTBObject, with its summary. - if isinstance(param, OTBObject): # single parameter - parameters[key] = param.summarize() - elif isinstance(param, list): # parameter list - parameters[key] = [p.summarize() if isinstance(p, OTBObject) else p for p in param] - return {"name": self.app.GetName(), "parameters": parameters} - def get_info(self) -> dict[str, (str, float, list[float])]: """Return a dict output of ReadImageInfo for the first image output.""" return App("ReadImageInfo", self, quiet=True).data @@ -436,8 +415,9 @@ class App(OTBObject): create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication self._app = create(appname) self._name = name or appname + self._exports_dic = {} + self._settings, self._auto_parameters = {}, {} self._time_start, self._time_end = 0.0, 0.0 - self._parameters, self._exports_dic = {}, {} self.data, self.outputs = {}, {} self.quiet, self.frozen = quiet, frozen # Param keys and types @@ -445,12 +425,14 @@ class App(OTBObject): self._all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys} types = (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename) self._out_param_types = {k: v for k, v in self._all_param_types.items() if v in types} + for key in self._out_param_types: + self.outputs[key] = Output(self, key, self._settings.get(key)) # Init, execute and write (auto flush only when output param was provided) if args or kwargs: self.set_parameters(*args, **kwargs) if not self.frozen: self.execute() - if any(key in self.parameters for key in self._out_param_types): + if any(key in self._settings for key in self._out_param_types): self.flush() @property @@ -464,20 +446,23 @@ class App(OTBObject): return self._app @property - def parameters(self) -> dict[str, Any]: - """Property to return an internal _parameters dict instance.""" - return self._parameters - - @property - def output_image_key(self) -> str: - """Get the name of first output image parameter.""" - return self.get_first_key([otb.ParameterType_OutputImage]) + def parameters(self): + """Return used OTB applications parameters.""" + return {**self.app.GetParameters(), **self._auto_parameters, **self._settings} @property def exports_dic(self) -> dict[str, dict]: """Returns internal _exports_dic object that contains numpy array exports.""" return self._exports_dic + def get_first_key(self, *type_lists: tuple[list[int]]) -> str: + """Get the first param key for specific file types, try each list in args.""" + for param_types in type_lists: + for key, value in sorted(self._all_param_types.items()): + if value in param_types: + return key + raise TypeError(f"{self.name}: could not find any parameter key matching the provided types") + @property def input_key(self) -> str: """Get the name of first input parameter, raster > vector > file.""" @@ -499,19 +484,16 @@ class App(OTBObject): [otb.ParameterType_OutputImage], [otb.ParameterType_OutputVectorData], [otb.ParameterType_OutputFilename] ) + @property + def output_image_key(self) -> str: + """Get the name of first output image parameter.""" + return self.get_first_key([otb.ParameterType_OutputImage]) + @property def elapsed_time(self) -> float: """Get elapsed time between app init and end of exec or file writing.""" return self._time_end - self._time_start - def get_first_key(self, *type_lists: tuple[list[int]]) -> str: - """Get the first param key for specific file types, try each list in args.""" - for param_types in type_lists: - for key, value in sorted(self._all_param_types.items()): - if value in param_types: - return key - raise TypeError(f"{self.name}: could not find any parameter key matching the provided types") - def set_parameters(self, *args, **kwargs): """Set some parameters of the app. @@ -546,12 +528,12 @@ class App(OTBObject): self.__set_param(key, obj) except (RuntimeError, TypeError, ValueError, KeyError) as e: raise RuntimeError( - f"{self.name}: something went wrong before execution " - f"(while setting parameter '{key}' to '{obj}': {e})" + f"{self.name}: error before execution, while setting parameter '{key}' to '{obj}': {e})" ) from e - # Update param dict and save values as object attributes - self._parameters.update(parameters) - self.save_objects(list(parameters)) + # Save / update setting value and update the Output object initialized in __init__ without a filepath + self._settings[key] = obj + if key in self._out_param_types: + self.outputs[key].filepath = obj def propagate_dtype(self, target_key: str = None, dtype: int = None): """Propagate a pixel type from main input to every outputs, or to a target output key only. @@ -565,7 +547,7 @@ class App(OTBObject): """ if not dtype: - param = self.parameters.get(self.input_image_key) + param = self._settings.get(self.input_image_key) if not param: logger.warning("%s: could not propagate pixel type from inputs to output", self.name) return @@ -583,30 +565,28 @@ class App(OTBObject): for key in keys: self.app.SetParameterOutputImagePixelType(key, dtype) - def save_objects(self, keys: list[str] = None): + def save_objects(self): """Save OTB app values in data, parameters and outputs dict, for a list of keys or all parameters.""" - keys = keys or self.parameters_keys - for key in keys: - value = self.parameters.get(key) - if value is None: + for key in self.parameters_keys: + if not self.app.HasValue(key): + continue + value = self.app.GetParameterValue(key) + # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken + if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(key): try: - value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results) + value = str(value) # some default str values like "mode" or "interpolator" + self._auto_parameters[key] = value + continue except RuntimeError: - pass # undefined parameter - if self._out_param_types.get(key) == otb.ParameterType_OutputImage: - self.outputs[key] = Output(self, key, value) - if value is None or isinstance(value, otb.ApplicationProxy): - continue - if isinstance(value, OTBObject) or bool(value) or value == 0: - if self.app.GetParameterRole(key) == 0: - self._parameters[key] = value - else: - if isinstance(value, str): - try: - value = literal_eval(value) - except (ValueError, SyntaxError): - pass - self.data[key] = value + continue # grouped parameters + # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.) + elif self.app.GetParameterRole(key) == 1 and bool(value) or value == 0: + if isinstance(value, str): + try: + value = literal_eval(value) + except (ValueError, SyntaxError): + pass + self.data[key] = value def execute(self): """Execute and write to disk if any output parameter has been set during init.""" @@ -781,7 +761,7 @@ class App(OTBObject): class Slicer(App): """Slicer objects i.e. when we call something like raster[:, :, 2] from Python.""" - def __init__(self, obj: App, rows: slice, cols: slice, channels: slice | list[int] | int): + def __init__(self, obj: OTBObject, rows: slice, cols: slice, channels: slice | list[int] | int): """Create a slicer object, that can be used directly for writing or inside a BandMath. It contains : @@ -1127,8 +1107,9 @@ class Input(App): class Output(OTBObject): """Object that behave like a pointer to a specific application output file.""" + _filepath: str | Path = None - def __init__(self, pyotb_app: App, param_key: str = None, filepath: str = "", mkdir: bool = True): + def __init__(self, pyotb_app: App, param_key: str = None, filepath: str = None, mkdir: bool = True): """Constructor for an Output object. Args: @@ -1141,13 +1122,11 @@ class Output(OTBObject): self.parent_pyotb_app = pyotb_app # keep trace of parent app self.param_key = param_key self.filepath = filepath - if filepath and not filepath.startswith("/vsi"): - self.filepath = Path(filepath.split("?")[0]) - if mkdir: - self.make_parent_dirs() + if mkdir and filepath is not None: + self.make_parent_dirs() @property - def name(self): + def name(self) -> str: """Return Output name containing filepath.""" return f"Output {self.param_key} from {self.parent_pyotb_app.name}" @@ -1156,11 +1135,6 @@ class Output(OTBObject): """Reference to the parent pyotb otb.Application instance.""" return self.parent_pyotb_app.app - @property - def parameters(self) -> dict[str, Any]: - """Reference to the parent pyotb App parameters.""" - return self.parent_pyotb_app.parameters - @property def exports_dic(self) -> dict[str, dict]: """Returns internal _exports_dic object that contains numpy array exports.""" @@ -1171,6 +1145,20 @@ class Output(OTBObject): """Force the right key to be used when accessing the OTBObject.""" return self.param_key + @property + def filepath(self) -> str | Path: + """Property to manage output URL.""" + if self._filepath is None: + raise ValueError("Filepath is not set") + return self._filepath + + @filepath.setter + def filepath(self, path: str): + if isinstance(path, str): + if path and not path.startswith("/vsi"): + path = Path(path.split("?")[0]) + self._filepath = path + def exists(self) -> bool: """Check file exist.""" if not isinstance(self.filepath, Path): @@ -1185,7 +1173,7 @@ class Output(OTBObject): def write(self, filepath: None | str | Path = None, **kwargs) -> bool: """Write output to disk, filepath is not required if it was provided to parent App during init.""" - if filepath is None and self.filepath: + if filepath is None: return self.parent_pyotb_app.write({self.output_image_key: self.filepath}, **kwargs) return self.parent_pyotb_app.write({self.output_image_key: filepath}, **kwargs) @@ -1293,3 +1281,21 @@ def is_key_images_list(pyotb_app: OTBObject, key: str) -> bool: def get_out_images_param_keys(app: OTBObject) -> list[str]: """Return every output parameter keys of an OTB app.""" return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage] + + +def summarize(obj: App | Output | Any) -> dict[str, str | dict[str, Any]]: + """Recursively summarize application parameters, and every App or Output found upstream in the pipeline. + + Returns: + nested dictionary with serialized App(s) containing name and parameters of an app and its parents + + """ + if isinstance(obj, Output): + return summarize(obj.parent_pyotb_app) + if not isinstance(obj, App): + return obj + parameters = { + key: [summarize(p) for p in param] if isinstance(param, list) else summarize(param) + for key, param in obj.parameters.items() + } + return {"name": obj.app.GetName(), "parameters": parameters} diff --git a/tests/serialized_apps.json b/tests/serialized_apps.json index 088e853..c413645 100644 --- a/tests/serialized_apps.json +++ b/tests/serialized_apps.json @@ -2,29 +2,13 @@ "SIMPLE": { "name": "ManageNoData", "parameters": { + "mode": "buildmask", "in": { "name": "OrthoRectification", "parameters": { - "io.in": { - "name": "BandMath", - "parameters": { - "il": [ - "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" - ], - "exp": "im1b1", - "ram": 256 - } - }, + "map": "utm", "map.utm.zone": 31, "map.utm.northhem": true, - "map.epsg.code": 4326, - "outputs.isotropic": true, - "outputs.default": 0.0, - "elev.default": 0.0, - "interpolator.bco.radius": 2, - "opt.rpc": 10, - "opt.ram": 256, - "opt.gridspacing": 4.0, "outputs.ulx": 560000.8125, "outputs.uly": 5495732.5, "outputs.sizex": 251, @@ -32,15 +16,22 @@ "outputs.spacingx": 5.997312068939209, "outputs.spacingy": -5.997312068939209, "outputs.lrx": 561506.125, - "outputs.lry": 5493909.5 + "outputs.lry": 5493909.5, + "outputs.isotropic": true, + "opt.gridspacing": 4.0, + "outputs.mode": "auto", + "interpolator": "bco", + "io.in": { + "name": "BandMath", + "parameters": { + "il": [ + "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" + ], + "exp": "im1b1" + } + } } - }, - "usenan": false, - "mode.buildmask.inv": 1.0, - "mode.buildmask.outv": 0.0, - "mode.changevalue.newv": 0.0, - "mode.apply.ndval": 0.0, - "ram": 256 + } } }, "COMPLEX": { @@ -50,26 +41,9 @@ { "name": "OrthoRectification", "parameters": { - "io.in": { - "name": "BandMath", - "parameters": { - "il": [ - "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" - ], - "exp": "im1b1", - "ram": 256 - } - }, + "map": "utm", "map.utm.zone": 31, "map.utm.northhem": true, - "map.epsg.code": 4326, - "outputs.isotropic": true, - "outputs.default": 0.0, - "elev.default": 0.0, - "interpolator.bco.radius": 2, - "opt.rpc": 10, - "opt.ram": 256, - "opt.gridspacing": 4.0, "outputs.ulx": 560000.8125, "outputs.uly": 5495732.5, "outputs.sizex": 251, @@ -77,35 +51,32 @@ "outputs.spacingx": 5.997312068939209, "outputs.spacingy": -5.997312068939209, "outputs.lrx": 561506.125, - "outputs.lry": 5493909.5 + "outputs.lry": 5493909.5, + "outputs.isotropic": true, + "opt.gridspacing": 4.0, + "outputs.mode": "auto", + "interpolator": "bco", + "io.in": { + "name": "BandMath", + "parameters": { + "il": [ + "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" + ], + "exp": "im1b1" + } + } } }, { "name": "ManageNoData", "parameters": { + "mode": "buildmask", "in": { "name": "OrthoRectification", "parameters": { - "io.in": { - "name": "BandMath", - "parameters": { - "il": [ - "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" - ], - "exp": "im1b1", - "ram": 256 - } - }, + "map": "utm", "map.utm.zone": 31, "map.utm.northhem": true, - "map.epsg.code": 4326, - "outputs.isotropic": true, - "outputs.default": 0.0, - "elev.default": 0.0, - "interpolator.bco.radius": 2, - "opt.rpc": 10, - "opt.ram": 256, - "opt.gridspacing": 4.0, "outputs.ulx": 560000.8125, "outputs.uly": 5495732.5, "outputs.sizex": 251, @@ -113,20 +84,26 @@ "outputs.spacingx": 5.997312068939209, "outputs.spacingy": -5.997312068939209, "outputs.lrx": 561506.125, - "outputs.lry": 5493909.5 + "outputs.lry": 5493909.5, + "outputs.isotropic": true, + "opt.gridspacing": 4.0, + "outputs.mode": "auto", + "interpolator": "bco", + "io.in": { + "name": "BandMath", + "parameters": { + "il": [ + "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" + ], + "exp": "im1b1" + } + } } - }, - "usenan": false, - "mode.buildmask.inv": 1.0, - "mode.buildmask.outv": 0.0, - "mode.changevalue.newv": 0.0, - "mode.apply.ndval": 0.0, - "ram": 256 + } } } ], - "exp": "im1+im2", - "ram": 256 + "exp": "im1+im2" } } } \ No newline at end of file diff --git a/tests/test_core.py b/tests/test_core.py index 143bf93..8d9acf8 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -208,8 +208,8 @@ def test_ndvi_comparison(): assert thresholded_bandmath["exp"] == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)" -def test_output_summary(): - assert INPUT["out"].summarize() +def test_summarize_output(): + assert pyotb.summarize(INPUT["out"]) def test_pipeline_simple(): @@ -217,7 +217,7 @@ def test_pipeline_simple(): app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"}) app2 = pyotb.OrthoRectification({"io.in": app1}) app3 = pyotb.ManageNoData({"in": app2}) - summary = app3.summarize() + summary = pyotb.summarize(app3) assert summary == SIMPLE_SERIALIZATION @@ -227,5 +227,5 @@ def test_pipeline_diamond(): app2 = pyotb.OrthoRectification({"io.in": app1}) app3 = pyotb.ManageNoData({"in": app2}) app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"}) - summary = app4.summarize() + summary = pyotb.summarize(app4) assert summary == COMPLEX_SERIALIZATION -- GitLab From 03533380701fda8ccd41003303cd80e757662caa Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 14 Feb 2023 21:17:19 +0100 Subject: [PATCH 155/399] FIX: #89 --- pyotb/core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index cd144f6..6a0a3e4 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -425,11 +425,12 @@ class App(OTBObject): self._all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys} types = (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename) self._out_param_types = {k: v for k, v in self._all_param_types.items() if v in types} - for key in self._out_param_types: - self.outputs[key] = Output(self, key, self._settings.get(key)) # Init, execute and write (auto flush only when output param was provided) if args or kwargs: self.set_parameters(*args, **kwargs) + # Create Output image objects + for key in filter(lambda k: self._out_param_types[k] == types[0], self._out_param_types.keys()): + self.outputs[key] = Output(self, key, self._settings.get(key)) if not self.frozen: self.execute() if any(key in self._settings for key in self._out_param_types): -- GitLab From cb40648ba425756a1fb3929e9534f5c6f0b2a5db Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 14 Feb 2023 21:24:29 +0100 Subject: [PATCH 156/399] ENH: remove OTBObject.write --- pyotb/core.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 6a0a3e4..9992f81 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -36,10 +36,6 @@ class OTBObject(ABC): def exports_dic(self) -> dict[str, dict]: """Return an internal dict object containing np.array exports, to avoid duplicated ExportImage() calls.""" - @abstractmethod - def write(self): - """Write image, this is defined in App. Output will use App.write for a specific key.""" - @property def metadata(self) -> dict[str, (str, float, list[float])]: """Return first output image metadata dictionary.""" -- GitLab From a87f6f0cb955b17e36304066086cfbd858c8a450 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 14 Feb 2023 22:42:46 +0100 Subject: [PATCH 157/399] ENH: remove app.write(*args) argument - write only accept 1 arg, or **kwargs --- pyotb/core.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 9992f81..3cdf469 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -609,15 +609,15 @@ class App(OTBObject): self.app.ExecuteAndWriteOutput() self._time_end = perf_counter() - def write(self, *args, ext_fname: str = "", pixel_type: dict[str, str] | str = None, - preserve_dtype: bool = False, **kwargs, ) -> bool: + def write(self, path: str | Path | dict[str, str] = None, ext_fname: str = "", + pixel_type: dict[str, str] | str = None, preserve_dtype: bool = False, **kwargs, ) -> bool: """Set output pixel type and write the output raster files. Args: - *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key contains - non-standard characters such as a point, e.g. {'io.out':'output.tif'} - - filepath, useful when there is only one output, e.g. 'output.tif' - - None if output file was passed during App init + path: Can be : - filepath, useful when there is only one output, e.g. 'output.tif' + - dictionary containing key-arguments enumeration. Useful when a key contains + non-standard characters such as a point, e.g. {'io.out':'output.tif'} + - None if output file was passed during App init ext_fname: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES") Will be used for all outputs (Default value = "") pixel_type: Can be : - dictionary {out_param_key: pixeltype} when specifying for several outputs @@ -633,14 +633,14 @@ class App(OTBObject): """ # Gather all input arguments in kwargs dict - for arg in args: - if isinstance(arg, dict): - kwargs.update(arg) - elif isinstance(arg, str) and kwargs: - logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg) - elif isinstance(arg, (str, Path)) and self.output_key: - kwargs.update({self.output_key: str(arg)}) - if not kwargs: + if path is not None: + if isinstance(path, dict): + kwargs.update(path) + elif isinstance(path, str) and kwargs: + logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, path) + elif isinstance(path, (str, Path)) and self.output_key: + kwargs.update({self.output_key: str(path)}) + if not (kwargs or self.output_key in self._settings): raise KeyError(f"{self.name}: at least one filepath is required, if not passed to App during init") parameters = kwargs.copy() -- GitLab From 0e865c9049c524079675cd90b8bdf6704f9ce04d Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 14 Feb 2023 22:58:15 +0100 Subject: [PATCH 158/399] FIX: avoid possible bugs with Output and write --- pyotb/core.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 3cdf469..df97fad 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -529,7 +529,7 @@ class App(OTBObject): ) from e # Save / update setting value and update the Output object initialized in __init__ without a filepath self._settings[key] = obj - if key in self._out_param_types: + if key in self.outputs: self.outputs[key].filepath = obj def propagate_dtype(self, target_key: str = None, dtype: int = None): @@ -640,7 +640,7 @@ class App(OTBObject): logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, path) elif isinstance(path, (str, Path)) and self.output_key: kwargs.update({self.output_key: str(path)}) - if not (kwargs or self.output_key in self._settings): + if not (kwargs or any(k in self._settings for k in self._out_param_types)): raise KeyError(f"{self.name}: at least one filepath is required, if not passed to App during init") parameters = kwargs.copy() @@ -676,7 +676,11 @@ class App(OTBObject): if key in data_types: self.propagate_dtype(key, data_types[key]) self.set_parameters({key: filepath}) + if self.frozen: + self.execute() self.flush() + if not parameters: + return True # Search and log missing files files, missing = [], [] for key, filepath in parameters.items(): -- GitLab From b99fbe8a660b93bb592d7ab747e573a122b1fe7e Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 14 Feb 2023 23:04:01 +0100 Subject: [PATCH 159/399] ENH: raise TypeError when bad filepath is passed to write --- pyotb/core.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index df97fad..1c3d86f 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -633,15 +633,16 @@ class App(OTBObject): """ # Gather all input arguments in kwargs dict - if path is not None: - if isinstance(path, dict): - kwargs.update(path) - elif isinstance(path, str) and kwargs: - logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, path) - elif isinstance(path, (str, Path)) and self.output_key: - kwargs.update({self.output_key: str(path)}) + if isinstance(path, dict): + kwargs.update(path) + elif isinstance(path, str) and kwargs: + logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, path) + elif isinstance(path, (str, Path)) and self.output_key: + kwargs.update({self.output_key: str(path)}) + elif path is not None: + raise TypeError(f"{self.name}: unsupported filepath type ({type(path)})") if not (kwargs or any(k in self._settings for k in self._out_param_types)): - raise KeyError(f"{self.name}: at least one filepath is required, if not passed to App during init") + raise KeyError(f"{self.name}: at least one filepath is required, if not provided during App init") parameters = kwargs.copy() # Append filename extension to filenames -- GitLab From 111c14143d5ae1d61943707f5fc31d582703a6b0 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 15 Feb 2023 10:17:57 +0100 Subject: [PATCH 160/399] STYLE: rename save_objects to private and more accurate __sync_parameters --- pyotb/core.py | 54 +++++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 1c3d86f..e02a5f7 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -444,8 +444,8 @@ class App(OTBObject): @property def parameters(self): - """Return used OTB applications parameters.""" - return {**self.app.GetParameters(), **self._auto_parameters, **self._settings} + """Return used OTB application parameters.""" + return {**self._auto_parameters, **self.app.GetParameters(), **self._settings} @property def exports_dic(self) -> dict[str, dict]: @@ -562,29 +562,6 @@ class App(OTBObject): for key in keys: self.app.SetParameterOutputImagePixelType(key, dtype) - def save_objects(self): - """Save OTB app values in data, parameters and outputs dict, for a list of keys or all parameters.""" - for key in self.parameters_keys: - if not self.app.HasValue(key): - continue - value = self.app.GetParameterValue(key) - # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken - if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(key): - try: - value = str(value) # some default str values like "mode" or "interpolator" - self._auto_parameters[key] = value - continue - except RuntimeError: - continue # grouped parameters - # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.) - elif self.app.GetParameterRole(key) == 1 and bool(value) or value == 0: - if isinstance(value, str): - try: - value = literal_eval(value) - except (ValueError, SyntaxError): - pass - self.data[key] = value - def execute(self): """Execute and write to disk if any output parameter has been set during init.""" logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters) @@ -596,7 +573,7 @@ class App(OTBObject): self.frozen = False self._time_end = perf_counter() logger.debug("%s: execution ended", self.name) - self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics + self.__sync_parameters() # this is required for apps like ReadImageInfo or ComputeImagesStatistics def flush(self): """Flush data to disk, this is when WriteOutput is actually called.""" @@ -610,7 +587,7 @@ class App(OTBObject): self._time_end = perf_counter() def write(self, path: str | Path | dict[str, str] = None, ext_fname: str = "", - pixel_type: dict[str, str] | str = None, preserve_dtype: bool = False, **kwargs, ) -> bool: + pixel_type: dict[str, str] | str = None, preserve_dtype: bool = False, **kwargs) -> bool: """Set output pixel type and write the output raster files. Args: @@ -741,6 +718,29 @@ class App(OTBObject): else: self.app.SetParameterValue(key, obj) + def __sync_parameters(self): + """Save OTB parameters in _settings, data and outputs dict, for a list of keys or all parameters.""" + for key in self.parameters_keys: + if not self.app.HasValue(key): + continue + value = self.app.GetParameterValue(key) + # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken + if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(key): + try: + value = str(value) # some default str values like "mode" or "interpolator" + self._auto_parameters[key] = value + continue + except RuntimeError: + continue # grouped parameters + # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.) + elif self.app.GetParameterRole(key) == 1 and bool(value) or value == 0: + if isinstance(value, str): + try: + value = literal_eval(value) + except (ValueError, SyntaxError): + pass + self.data[key] = value + # Special functions def __getitem__(self, key: str) -> Any | list[int | float] | int | float | Slicer: """This function is called when we use App()[...]. -- GitLab From 789913a50f302bbfe8776bc279325ec12cf96b29 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 15 Feb 2023 21:07:57 +0100 Subject: [PATCH 161/399] REFAC: move function back to its original position, for review purpose --- pyotb/core.py | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index e02a5f7..4d5f7b2 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -562,6 +562,29 @@ class App(OTBObject): for key in keys: self.app.SetParameterOutputImagePixelType(key, dtype) + def __sync_parameters(self): + """Save OTB parameters in _settings, data and outputs dict, for a list of keys or all parameters.""" + for key in self.parameters_keys: + if not self.app.HasValue(key): + continue + value = self.app.GetParameterValue(key) + # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken + if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(key): + try: + value = str(value) # some default str values like "mode" or "interpolator" + self._auto_parameters[key] = value + continue + except RuntimeError: + continue # grouped parameters + # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.) + elif self.app.GetParameterRole(key) == 1 and bool(value) or value == 0: + if isinstance(value, str): + try: + value = literal_eval(value) + except (ValueError, SyntaxError): + pass + self.data[key] = value + def execute(self): """Execute and write to disk if any output parameter has been set during init.""" logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters) @@ -718,29 +741,6 @@ class App(OTBObject): else: self.app.SetParameterValue(key, obj) - def __sync_parameters(self): - """Save OTB parameters in _settings, data and outputs dict, for a list of keys or all parameters.""" - for key in self.parameters_keys: - if not self.app.HasValue(key): - continue - value = self.app.GetParameterValue(key) - # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken - if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(key): - try: - value = str(value) # some default str values like "mode" or "interpolator" - self._auto_parameters[key] = value - continue - except RuntimeError: - continue # grouped parameters - # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.) - elif self.app.GetParameterRole(key) == 1 and bool(value) or value == 0: - if isinstance(value, str): - try: - value = literal_eval(value) - except (ValueError, SyntaxError): - pass - self.data[key] = value - # Special functions def __getitem__(self, key: str) -> Any | list[int | float] | int | float | Slicer: """This function is called when we use App()[...]. -- GitLab From 183decee7e245ea0bfc3e3725821a7f9144eeee2 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 16 Mar 2023 14:19:57 +0000 Subject: [PATCH 162/399] ENH: better filter expression for parameter keys, use enum --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 4d5f7b2..8821e55 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -425,7 +425,7 @@ class App(OTBObject): if args or kwargs: self.set_parameters(*args, **kwargs) # Create Output image objects - for key in filter(lambda k: self._out_param_types[k] == types[0], self._out_param_types.keys()): + for key in filter(lambda k: self._out_param_types[k] == otb.ParameterType_OutputImage, self._out_param_types): self.outputs[key] = Output(self, key, self._settings.get(key)) if not self.frozen: self.execute() -- GitLab From 3156a3a3f9c9197050328dbd18e3f08e5a2be5cb Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 16 Mar 2023 15:21:21 +0100 Subject: [PATCH 163/399] ENH: add 2 write() tests for frozen apps --- tests/test_core.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index 8d9acf8..ebb26cf 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -78,11 +78,27 @@ def test_write(): INPUT["out"].filepath.unlink() +def test_frozen_app_write(): + app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True) + assert app.write("/tmp/test_frozen_app_write.tif") + app["out"].filepath.unlink() + + app = pyotb.BandMath(INPUT, exp="im1b1", out="/tmp/test_frozen_app_write.tif", frozen=True) + assert app.write() + app["out"].filepath.unlink() + + def test_output_write(): assert INPUT["out"].write("/tmp/test_output_write.tif") INPUT["out"].filepath.unlink() +def test_frozen_output_write(): + app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True) + assert app["out"].write("/tmp/test_frozen_app_write.tif") + app["out"].filepath.unlink() + + def test_output_in_arg(): info = pyotb.ReadImageInfo(INPUT["out"]) assert info.data -- GitLab From fd963531465af35d6a66afcbc6808187b2cfdf91 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 24 Mar 2023 11:32:35 +0100 Subject: [PATCH 164/399] Revert "REFAC: move function back to its original position, for review purpose" This reverts commit 789913a50f302bbfe8776bc279325ec12cf96b29. --- pyotb/core.py | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 8821e55..ad18769 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -562,29 +562,6 @@ class App(OTBObject): for key in keys: self.app.SetParameterOutputImagePixelType(key, dtype) - def __sync_parameters(self): - """Save OTB parameters in _settings, data and outputs dict, for a list of keys or all parameters.""" - for key in self.parameters_keys: - if not self.app.HasValue(key): - continue - value = self.app.GetParameterValue(key) - # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken - if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(key): - try: - value = str(value) # some default str values like "mode" or "interpolator" - self._auto_parameters[key] = value - continue - except RuntimeError: - continue # grouped parameters - # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.) - elif self.app.GetParameterRole(key) == 1 and bool(value) or value == 0: - if isinstance(value, str): - try: - value = literal_eval(value) - except (ValueError, SyntaxError): - pass - self.data[key] = value - def execute(self): """Execute and write to disk if any output parameter has been set during init.""" logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters) @@ -741,6 +718,29 @@ class App(OTBObject): else: self.app.SetParameterValue(key, obj) + def __sync_parameters(self): + """Save OTB parameters in _settings, data and outputs dict, for a list of keys or all parameters.""" + for key in self.parameters_keys: + if not self.app.HasValue(key): + continue + value = self.app.GetParameterValue(key) + # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken + if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(key): + try: + value = str(value) # some default str values like "mode" or "interpolator" + self._auto_parameters[key] = value + continue + except RuntimeError: + continue # grouped parameters + # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.) + elif self.app.GetParameterRole(key) == 1 and bool(value) or value == 0: + if isinstance(value, str): + try: + value = literal_eval(value) + except (ValueError, SyntaxError): + pass + self.data[key] = value + # Special functions def __getitem__(self, key: str) -> Any | list[int | float] | int | float | Slicer: """This function is called when we use App()[...]. -- GitLab From 236b80ef2ff92a7dc3fadd8c9f9a4645f1947241 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Mon, 22 May 2023 11:11:13 +0200 Subject: [PATCH 165/399] ADD: strip_input_paths and strip_output_paths in summarize() + subtle refactoring to achieve this elegantly --- pyotb/core.py | 136 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 119 insertions(+), 17 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index ad18769..8c30da8 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -389,6 +389,33 @@ class OTBObject(ABC): class App(OTBObject): """Base class that gathers common operations for any OTB application.""" + INPUT_IMAGE_TYPES = [ + # Images only + otb.ParameterType_InputImage, + otb.ParameterType_InputImageList + ] + + INPUT_PARAM_TYPES = INPUT_IMAGE_TYPES + [ + # Vectors + otb.ParameterType_InputVectorData, + otb.ParameterType_InputVectorDataList, + # Filenames + otb.ParameterType_InputFilename, + otb.ParameterType_InputFilenameList, + ] + + OUTPUT_IMAGES_TYPES = [ + # Images only + otb.ParameterType_OutputImage + ] + + OUTPUT_PARAM_TYPES = OUTPUT_IMAGES_TYPES + [ + # Vectors + otb.ParameterType_OutputVectorData, + # Filenames + otb.ParameterType_OutputFilename, + ] + def __init__(self, appname: str, *args, frozen: bool = False, quiet: bool = False, name: str = "", **kwargs): """Common constructor for OTB applications. Handles in-memory connection between apps. @@ -452,39 +479,78 @@ class App(OTBObject): """Returns internal _exports_dic object that contains numpy array exports.""" return self._exports_dic - def get_first_key(self, *type_lists: tuple[list[int]]) -> str: + def _is_one_of_types(self, key: str, param_types: list[int]) -> bool: + """ + Helper to factor is_input and is_output + """ + if key not in self._all_param_types: + raise KeyError( + f"key {key} not found in the application parameters types" + ) + return self._all_param_types[key] in param_types + + @property + def is_input(self, key) -> bool: + """ + Returns True if the key is an input + + Args: + key: parameter key + + Returns: + True if the parameter is an input, else False + + """ + return self._is_one_of_types( + key=key, + param_types=self.INPUT_PARAM_TYPES + ) + + @property + def is_output(self, key) -> bool: + """ + Returns True if the key is an output + + Args: + key: parameter key + + Returns: + True if the parameter is an output, else False + + """ + return self._is_one_of_types( + key=key, + param_types=self.OUTPUT_PARAM_TYPES + ) + def get_first_key(self, param_types: list[int]) -> str: """Get the first param key for specific file types, try each list in args.""" - for param_types in type_lists: + for param_type in param_types: + # Return the first key, from the alphabetically sorted keys of the + # application, which has the parameter type matching param_type. for key, value in sorted(self._all_param_types.items()): - if value in param_types: + if value == param_type: return key raise TypeError(f"{self.name}: could not find any parameter key matching the provided types") @property def input_key(self) -> str: """Get the name of first input parameter, raster > vector > file.""" - return self.get_first_key( - [otb.ParameterType_InputImage, otb.ParameterType_InputImageList], - [otb.ParameterType_InputVectorData, otb.ParameterType_InputVectorDataList], - [otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList], - ) + return self.get_first_key(self.INPUT_PARAM_TYPES) @property def input_image_key(self) -> str: """Name of the first input image parameter.""" - return self.get_first_key([otb.ParameterType_InputImage, otb.ParameterType_InputImageList]) + return self.get_first_key(self.INPUT_IMAGE_TYPES) @property def output_key(self) -> str: """Name of the first output parameter, raster > vector > file.""" - return self.get_first_key( - [otb.ParameterType_OutputImage], [otb.ParameterType_OutputVectorData], [otb.ParameterType_OutputFilename] - ) + return self.get_first_key(self.OUTPUT_PARAM_TYPES) @property def output_image_key(self) -> str: """Get the name of first output image parameter.""" - return self.get_first_key([otb.ParameterType_OutputImage]) + return self.get_first_key(self.OUTPUT_IMAGES_TYPES) @property def elapsed_time(self) -> float: @@ -1285,19 +1351,55 @@ def get_out_images_param_keys(app: OTBObject) -> list[str]: return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage] -def summarize(obj: App | Output | Any) -> dict[str, str | dict[str, Any]]: - """Recursively summarize application parameters, and every App or Output found upstream in the pipeline. +def summarize( + obj: App | Output | Any, + strip_input_paths: bool = False, + strip_output_paths: bool = False, +) -> dict[str, str | dict[str, Any]]: + """Recursively summarize application parameters, and every App or Output + found upstream in the pipeline. + + Args: + obj: input object to summarize + strip_input_paths: strip all input paths: If enabled, paths related to + inputs are truncated after the first "?" character. Can be useful + to remove URLs tokens (e.g. SAS or S3 credentials). + strip_output_paths: strip all output paths: If enabled, paths related + to outputs are truncated after the first "?" character. Can be + useful to remove extended filenames. Returns: - nested dictionary with serialized App(s) containing name and parameters of an app and its parents + nested dictionary with serialized App(s) containing name and + parameters of an app and its parents """ if isinstance(obj, Output): return summarize(obj.parent_pyotb_app) if not isinstance(obj, App): return obj + + # If we are here, "obj" is an App + + def _summarize_single_param(key, param): + """ + This function truncates inputs or outputs paths, before the first + occurrence of character "?". + """ + param_summary = summarize(param) + if isinstance(param, str): + # If we are here, "param" could be any str parameter value + if strip_input_paths and obj.is_input( + key) or strip_output_paths and obj.is_output(key): + # If we are here, "param" is a path to an input or output + # image, vector, or filename + param_summary = param_summary.split("?")[0] + + return param_summary + parameters = { - key: [summarize(p) for p in param] if isinstance(param, list) else summarize(param) + key: [ + _summarize_single_param(key, p) for p in param + ] if isinstance(param, list) else _summarize_single_param(key, param) for key, param in obj.parameters.items() } return {"name": obj.app.GetName(), "parameters": parameters} -- GitLab From 9174c412029bb506b837246564c201b19af3c1f6 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Mon, 22 May 2023 11:20:49 +0200 Subject: [PATCH 166/399] ADD: fix property-with-parameters --- pyotb/core.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 8c30da8..6b4ef8d 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -489,7 +489,6 @@ class App(OTBObject): ) return self._all_param_types[key] in param_types - @property def is_input(self, key) -> bool: """ Returns True if the key is an input @@ -506,7 +505,6 @@ class App(OTBObject): param_types=self.INPUT_PARAM_TYPES ) - @property def is_output(self, key) -> bool: """ Returns True if the key is an output -- GitLab From 9156b59250d88670347161fd5d6c960c828de7d7 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 12:07:33 +0200 Subject: [PATCH 167/399] ENH: smaller summarize function, check '?' is in path before split --- pyotb/core.py | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 6b4ef8d..db392d6 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1350,9 +1350,9 @@ def get_out_images_param_keys(app: OTBObject) -> list[str]: def summarize( - obj: App | Output | Any, - strip_input_paths: bool = False, - strip_output_paths: bool = False, + obj: App | Output | Any, + strip_input_paths: bool = False, + strip_output_paths: bool = False, ) -> dict[str, str | dict[str, Any]]: """Recursively summarize application parameters, and every App or Output found upstream in the pipeline. @@ -1371,33 +1371,22 @@ def summarize( parameters of an app and its parents """ - if isinstance(obj, Output): - return summarize(obj.parent_pyotb_app) - if not isinstance(obj, App): - return obj - # If we are here, "obj" is an App - - def _summarize_single_param(key, param): - """ - This function truncates inputs or outputs paths, before the first - occurrence of character "?". - """ + def strip_path(key: str, param: str | Any): + """Truncate text after the first "?" character in filepath.""" param_summary = summarize(param) - if isinstance(param, str): - # If we are here, "param" could be any str parameter value - if strip_input_paths and obj.is_input( - key) or strip_output_paths and obj.is_output(key): - # If we are here, "param" is a path to an input or output - # image, vector, or filename + if strip_input_paths and obj.is_input(key) or strip_output_paths and obj.is_output(key): + if isinstance(param, str) and "?" in param_summary: param_summary = param_summary.split("?")[0] - return param_summary + if isinstance(obj, Output): + return summarize(obj.parent_pyotb_app) + if not isinstance(obj, App): + return obj + # If we are here, "obj" is an App parameters = { - key: [ - _summarize_single_param(key, p) for p in param - ] if isinstance(param, list) else _summarize_single_param(key, param) + key: [strip_path(key, p) for p in param] if isinstance(param, list) else strip_path(key, param) for key, param in obj.parameters.items() } return {"name": obj.app.GetName(), "parameters": parameters} -- GitLab From 58d7401ce16d86a8e4fc44060b59750798d0d673 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 12:12:44 +0200 Subject: [PATCH 168/399] STYLE: add missing underscore and type hints --- pyotb/core.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index db392d6..d0a3b48 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -479,7 +479,7 @@ class App(OTBObject): """Returns internal _exports_dic object that contains numpy array exports.""" return self._exports_dic - def _is_one_of_types(self, key: str, param_types: list[int]) -> bool: + def __is_one_of_types(self, key: str, param_types: list[int]) -> bool: """ Helper to factor is_input and is_output """ @@ -489,7 +489,7 @@ class App(OTBObject): ) return self._all_param_types[key] in param_types - def is_input(self, key) -> bool: + def is_input(self, key: str) -> bool: """ Returns True if the key is an input @@ -500,12 +500,11 @@ class App(OTBObject): True if the parameter is an input, else False """ - return self._is_one_of_types( - key=key, - param_types=self.INPUT_PARAM_TYPES + return self.__is_one_of_types( + key=key, param_types=self.INPUT_PARAM_TYPES ) - def is_output(self, key) -> bool: + def is_output(self, key: str) -> bool: """ Returns True if the key is an output @@ -516,10 +515,10 @@ class App(OTBObject): True if the parameter is an output, else False """ - return self._is_one_of_types( - key=key, - param_types=self.OUTPUT_PARAM_TYPES + return self.__is_one_of_types( + key=key, param_types=self.OUTPUT_PARAM_TYPES ) + def get_first_key(self, param_types: list[int]) -> str: """Get the first param key for specific file types, try each list in args.""" for param_type in param_types: -- GitLab From 26593adab359f38ecafdc6f619c99f063095dca2 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 12:23:56 +0200 Subject: [PATCH 169/399] CI: docstrings --- pyotb/core.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index d0a3b48..b3f2278 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -480,9 +480,7 @@ class App(OTBObject): return self._exports_dic def __is_one_of_types(self, key: str, param_types: list[int]) -> bool: - """ - Helper to factor is_input and is_output - """ + """Helper to factor is_input and is_output.""" if key not in self._all_param_types: raise KeyError( f"key {key} not found in the application parameters types" @@ -490,8 +488,7 @@ class App(OTBObject): return self._all_param_types[key] in param_types def is_input(self, key: str) -> bool: - """ - Returns True if the key is an input + """Returns True if the key is an input. Args: key: parameter key @@ -505,8 +502,7 @@ class App(OTBObject): ) def is_output(self, key: str) -> bool: - """ - Returns True if the key is an output + """Returns True if the key is an output. Args: key: parameter key @@ -1353,8 +1349,7 @@ def summarize( strip_input_paths: bool = False, strip_output_paths: bool = False, ) -> dict[str, str | dict[str, Any]]: - """Recursively summarize application parameters, and every App or Output - found upstream in the pipeline. + """Recursively summarize parameters of an App or Output object and its parents. Args: obj: input object to summarize -- GitLab From 0894ff58ce1178f4a80e495b856a684e01f02d9e Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Mon, 22 May 2023 13:53:10 +0200 Subject: [PATCH 170/399] TEST: add a test for the path strippping feature of summarize() --- tests/test_core.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index ebb26cf..7b2d53d 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -228,6 +228,26 @@ def test_summarize_output(): assert pyotb.summarize(INPUT["out"]) +def test_summarize_strip_output(): + in_fn = FILEPATH + in_fn_w_ext = FILEPATH + "?&skipcarto=1" + out_fn = "/tmp/output.tif" + out_fn_w_ext = out_fn + "?&box=10:10:10:10" + + baseline = [ + (in_fn, out_fn_w_ext, "out", {}, out_fn_w_ext), + (in_fn, out_fn_w_ext, "out", {"strip_output_paths": True}, out_fn), + (in_fn_w_ext, out_fn, "in", {}, in_fn_w_ext), + (in_fn_w_ext, out_fn, "in", {"strip_input_paths": True}, in_fn) + ] + + for inp, out, key, extra_args, expected in baseline: + app = pyotb.ExtractROI({"in": inp, "out": out}) + summary = pyotb.summarize(app, **extra_args) + assert summary["parameters"][key] == expected, \ + f"Failed for input {inp}, output {out}, args {extra_args}" + + def test_pipeline_simple(): # BandMath -> OrthoRectification -> ManageNoData app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"}) @@ -245,3 +265,5 @@ def test_pipeline_diamond(): app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"}) summary = pyotb.summarize(app4) assert summary == COMPLEX_SERIALIZATION + +test_summarize_strip_output() \ No newline at end of file -- GitLab From e24baa1f5d42532dbe4ffde8f982748f9d2263fa Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 14:47:22 +0200 Subject: [PATCH 171/399] ENH: apply missing patch from previous MR --- tests/test_core.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 7b2d53d..2752865 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -265,5 +265,3 @@ def test_pipeline_diamond(): app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"}) summary = pyotb.summarize(app4) assert summary == COMPLEX_SERIALIZATION - -test_summarize_strip_output() \ No newline at end of file -- GitLab From 3231055755ba7fb00b34ca2259ac97b685ab1ec6 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 15:54:37 +0200 Subject: [PATCH 172/399] ENH: new function add_vsi_prefix --- pyotb/core.py | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index b3f2278..d1cd680 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -580,8 +580,13 @@ class App(OTBObject): obj = [obj] logger.info('%s: argument for parameter "%s" was converted to list', self.name, key) try: - # This is when we actually call self.app.SetParameter* - self.__set_param(key, obj) + if self.is_input(key): + if is_key_images_list(self, key): + self.__set_param(key, [add_vsi_prefix(p) for p in obj]) + else: + self.__set_param(key, add_vsi_prefix(obj)) + else: + self.__set_param(key, obj) except (RuntimeError, TypeError, ValueError, KeyError) as e: raise RuntimeError( f"{self.name}: error before execution, while setting parameter '{key}' to '{obj}': {e})" @@ -1243,6 +1248,34 @@ class Output(OTBObject): return str(self.filepath) +def add_vsi_prefix(filepath: str | Path) -> str: + """Append vsi prefixes to file URL or path if needed. + + Args: + filepath: file path or URL + + Returns: + string with new /vsi prefix(es) + + """ + if isinstance(filepath, Path): + filepath = str(filepath) + if isinstance(filepath, str) and not filepath.startswith("/vsi"): + # Remote file + if filepath.startswith("https://") or filepath.startswith("http://"): + filepath = "/vsicurl/" + filepath + # Compressed file + if filepath.endswith(".tar") or filepath.endswith(".tar.gz") or filepath.endswith(".tgz"): + filepath = "/vsitar/" + filepath + elif filepath.endswith(".gz"): + filepath = "/vsigzip/" + filepath + elif filepath.endswith(".7z"): + filepath = "/vsi7z/" + filepath + elif filepath.endswith(".zip"): + filepath = "/vsizip/" + filepath + return filepath + + def get_nbchannels(inp: str | Path | OTBObject) -> int: """Get the nb of bands of input image. -- GitLab From 50a6020ce13ae052dff9af40e332aec32a131f91 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 15:54:40 +0200 Subject: [PATCH 173/399] ENH: update test, now vsicurl is not required --- tests/serialized_apps.json | 8 ++++---- tests/tests_data.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/serialized_apps.json b/tests/serialized_apps.json index c413645..424afba 100644 --- a/tests/serialized_apps.json +++ b/tests/serialized_apps.json @@ -25,7 +25,7 @@ "name": "BandMath", "parameters": { "il": [ - "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" + "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" ], "exp": "im1b1" } @@ -60,7 +60,7 @@ "name": "BandMath", "parameters": { "il": [ - "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" + "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" ], "exp": "im1b1" } @@ -93,7 +93,7 @@ "name": "BandMath", "parameters": { "il": [ - "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" + "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" ], "exp": "im1b1" } @@ -106,4 +106,4 @@ "exp": "im1+im2" } } -} \ No newline at end of file +} diff --git a/tests/tests_data.py b/tests/tests_data.py index bfb774b..36c3592 100644 --- a/tests/tests_data.py +++ b/tests/tests_data.py @@ -2,7 +2,7 @@ import json from pathlib import Path import pyotb -FILEPATH = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" +FILEPATH = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" INPUT = pyotb.Input(FILEPATH) TEST_IMAGE_STATS = { 'out.mean': [79.5505, 109.225, 115.456, 249.349], -- GitLab From cab52f92c31ad806e7218defd347a0c5c9a4d988 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 16:00:11 +0200 Subject: [PATCH 174/399] CI: docstring indent --- pyotb/core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index d1cd680..b52dbfe 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1251,11 +1251,11 @@ class Output(OTBObject): def add_vsi_prefix(filepath: str | Path) -> str: """Append vsi prefixes to file URL or path if needed. - Args: - filepath: file path or URL + Args: + filepath: file path or URL - Returns: - string with new /vsi prefix(es) + Returns: + string with new /vsi prefix(es) """ if isinstance(filepath, Path): -- GitLab From b7077db05ff38b924310fb579ae162c44750837a Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 16:38:25 +0200 Subject: [PATCH 175/399] ENH: TODO comment about S3 and GCP --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index b52dbfe..7be171d 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1261,7 +1261,7 @@ def add_vsi_prefix(filepath: str | Path) -> str: if isinstance(filepath, Path): filepath = str(filepath) if isinstance(filepath, str) and not filepath.startswith("/vsi"): - # Remote file + # Remote file. TODO: add support for S3 and GCP URLs if filepath.startswith("https://") or filepath.startswith("http://"): filepath = "/vsicurl/" + filepath # Compressed file -- GitLab From 74447f1f286a08c2d48ce62a6e228618aaca1340 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 16:49:52 +0200 Subject: [PATCH 176/399] ENH: move is_key_list and is_key_images_list back to App --- pyotb/core.py | 54 +++++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 7be171d..8c9cce4 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -394,7 +394,6 @@ class App(OTBObject): otb.ParameterType_InputImage, otb.ParameterType_InputImageList ] - INPUT_PARAM_TYPES = INPUT_IMAGE_TYPES + [ # Vectors otb.ParameterType_InputVectorData, @@ -404,18 +403,29 @@ class App(OTBObject): otb.ParameterType_InputFilenameList, ] - OUTPUT_IMAGES_TYPES = [ + OUTPUT_IMAGE_TYPES = [ # Images only otb.ParameterType_OutputImage ] - - OUTPUT_PARAM_TYPES = OUTPUT_IMAGES_TYPES + [ + OUTPUT_PARAM_TYPES = OUTPUT_IMAGE_TYPES + [ # Vectors otb.ParameterType_OutputVectorData, # Filenames otb.ParameterType_OutputFilename, ] + INPUT_LIST_TYPES = [ + otb.ParameterType_InputImageList, + otb.ParameterType_StringList, + otb.ParameterType_InputFilenameList, + otb.ParameterType_ListView, + otb.ParameterType_InputVectorDataList, + ] + INPUT_IMAGES_LIST_TYPES = [ + otb.ParameterType_InputImageList, + otb.ParameterType_InputFilenameList, + ] + def __init__(self, appname: str, *args, frozen: bool = False, quiet: bool = False, name: str = "", **kwargs): """Common constructor for OTB applications. Handles in-memory connection between apps. @@ -515,6 +525,14 @@ class App(OTBObject): key=key, param_types=self.OUTPUT_PARAM_TYPES ) + def is_key_list(self, key: str) -> bool: + """Check if a parameter key is an input parameter list.""" + return self.app.GetParameterType(key) in self.INPUT_LIST_TYPES + + def is_key_images_list(self, key: str) -> bool: + """Check if a parameter key is an input parameter image list.""" + return self.app.GetParameterType(key) in self.INPUT_IMAGES_LIST_TYPES + def get_first_key(self, param_types: list[int]) -> str: """Get the first param key for specific file types, try each list in args.""" for param_type in param_types: @@ -543,7 +561,7 @@ class App(OTBObject): @property def output_image_key(self) -> str: """Get the name of first output image parameter.""" - return self.get_first_key(self.OUTPUT_IMAGES_TYPES) + return self.get_first_key(self.OUTPUT_IMAGE_TYPES) @property def elapsed_time(self) -> float: @@ -576,12 +594,12 @@ class App(OTBObject): f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}" ) # When the parameter expects a list, if needed, change the value to list - if is_key_list(self, key) and not isinstance(obj, (list, tuple)): + if self.is_key_list(key) and not isinstance(obj, (list, tuple)): obj = [obj] logger.info('%s: argument for parameter "%s" was converted to list', self.name, key) try: if self.is_input(key): - if is_key_images_list(self, key): + if self.is_key_images_list(key): self.__set_param(key, [add_vsi_prefix(p) for p in obj]) else: self.__set_param(key, add_vsi_prefix(obj)) @@ -749,7 +767,7 @@ class App(OTBObject): for arg in args: if isinstance(arg, dict): kwargs.update(arg) - elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.input_key): + elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and self.is_key_list(self.input_key): kwargs.update({self.input_key: arg}) return kwargs @@ -768,7 +786,7 @@ class App(OTBObject): elif not isinstance(obj, list): # any other parameters (str, int...) self.app.SetParameterValue(key, obj) # Images list - elif is_key_images_list(self, key): + elif self.is_key_images_list(key): # To enable possible in-memory connections, we go through the list and set the parameters one by one for inp in obj: if isinstance(inp, OTBObject): @@ -1354,24 +1372,6 @@ def parse_pixel_type(pixel_type: str | int) -> int: raise TypeError(f"Bad pixel type specification ({pixel_type} of type {type(pixel_type)})") -def is_key_list(pyotb_app: OTBObject, key: str) -> bool: - """Check if a key of the OTBObject is an input parameter list.""" - types = ( - otb.ParameterType_InputImageList, - otb.ParameterType_StringList, - otb.ParameterType_InputFilenameList, - otb.ParameterType_ListView, - otb.ParameterType_InputVectorDataList, - ) - return pyotb_app.app.GetParameterType(key) in types - - -def is_key_images_list(pyotb_app: OTBObject, key: str) -> bool: - """Check if a key of the OTBObject is an input parameter image list.""" - types = (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList) - return pyotb_app.app.GetParameterType(key) in types - - def get_out_images_param_keys(app: OTBObject) -> list[str]: """Return every output parameter keys of an OTB app.""" return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage] -- GitLab From c1488e9909e43d219f57cc1968ca777e3ffe2114 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 17:07:50 +0200 Subject: [PATCH 177/399] ENH: add test case for input file with existing vsi prefix --- tests/test_core.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index 2752865..3afffd1 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -6,9 +6,16 @@ from tests_data import * # Input settings def test_parameters(): + assert INPUT.parameters + assert INPUT.parameters["in"] == FILEPATH assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304) +def test_input_with_vsi(): + # Ensure old way is still working - this should raise RuntimeError when the path is malformed + pyotb.Input("/vsicurl/" + FILEPATH) + + def test_wrong_key(): with pytest.raises(KeyError): pyotb.BandMath(INPUT, expression="im1b1") -- GitLab From 90f5845afbcc2e897a4d6ffead84d0645e573713 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 17:18:50 +0200 Subject: [PATCH 178/399] ENH: enhance add_vsi_prefix function, add ftp:// and multi prefix checks, works with extfn --- pyotb/core.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 8c9cce4..5cd566a 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1239,7 +1239,7 @@ class Output(OTBObject): @filepath.setter def filepath(self, path: str): if isinstance(path, str): - if path and not path.startswith("/vsi"): + if path and not path.startswith(["/vsi", "http", "ftp"]): path = Path(path.split("?")[0]) self._filepath = path @@ -1279,17 +1279,18 @@ def add_vsi_prefix(filepath: str | Path) -> str: if isinstance(filepath, Path): filepath = str(filepath) if isinstance(filepath, str) and not filepath.startswith("/vsi"): - # Remote file. TODO: add support for S3 and GCP URLs - if filepath.startswith("https://") or filepath.startswith("http://"): + # Remote file. TODO: add support for S3 / GS / AZ + if filepath.startswith(["https://", "http://", "ftp://"]): filepath = "/vsicurl/" + filepath # Compressed file - if filepath.endswith(".tar") or filepath.endswith(".tar.gz") or filepath.endswith(".tgz"): + basename = filepath.split("?")[0] + if basename.endswith([".tar", ".tar.gz", ".tgz"]): filepath = "/vsitar/" + filepath - elif filepath.endswith(".gz"): + elif basename.endswith(".gz"): filepath = "/vsigzip/" + filepath - elif filepath.endswith(".7z"): + elif basename.endswith(".7z"): filepath = "/vsi7z/" + filepath - elif filepath.endswith(".zip"): + elif basename.endswith(".zip"): filepath = "/vsizip/" + filepath return filepath -- GitLab From 148b0cb06e494949543c1eb35e0f24eea88c7c53 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 17:24:54 +0200 Subject: [PATCH 179/399] FIX: startswith and endswith multi check requires tuple argument --- pyotb/core.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 5cd566a..d7de84a 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1180,7 +1180,9 @@ class Input(App): """ super().__init__("ExtractROI", {"in": filepath}, frozen=True) self._name = f"Input from {filepath}" - self.filepath = Path(filepath) if not filepath.startswith("/vsi") else filepath + if not filepath.startswith(("/vsi", "http", "ftp")): + filepath = Path(filepath) + self.filepath = filepath self.propagate_dtype() self.execute() @@ -1239,7 +1241,7 @@ class Output(OTBObject): @filepath.setter def filepath(self, path: str): if isinstance(path, str): - if path and not path.startswith(["/vsi", "http", "ftp"]): + if path and not path.startswith(("/vsi", "http", "ftp")): path = Path(path.split("?")[0]) self._filepath = path @@ -1280,11 +1282,11 @@ def add_vsi_prefix(filepath: str | Path) -> str: filepath = str(filepath) if isinstance(filepath, str) and not filepath.startswith("/vsi"): # Remote file. TODO: add support for S3 / GS / AZ - if filepath.startswith(["https://", "http://", "ftp://"]): + if filepath.startswith(("https://", "http://", "ftp://")): filepath = "/vsicurl/" + filepath # Compressed file basename = filepath.split("?")[0] - if basename.endswith([".tar", ".tar.gz", ".tgz"]): + if basename.endswith((".tar", ".tar.gz", ".tgz")): filepath = "/vsitar/" + filepath elif basename.endswith(".gz"): filepath = "/vsigzip/" + filepath -- GitLab From 3f5fc1454b859b952d13e090958e803cbc2f7d82 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 17:37:26 +0200 Subject: [PATCH 180/399] FIX: add test error message if image could not be loaded --- tests/tests_data.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/tests_data.py b/tests/tests_data.py index 36c3592..b54c6f2 100644 --- a/tests/tests_data.py +++ b/tests/tests_data.py @@ -2,8 +2,14 @@ import json from pathlib import Path import pyotb -FILEPATH = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" -INPUT = pyotb.Input(FILEPATH) +FILEPATH = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tiff" +try: + INPUT = pyotb.Input(FILEPATH) +except RuntimeError as err: + if str(err.__cause__).startswith("Cannot open image "): + raise Exception("Unable to access the remote image, GitLab could be offline.") from err + raise Exception("Unexpected error while fetching test data.") from err + TEST_IMAGE_STATS = { 'out.mean': [79.5505, 109.225, 115.456, 249.349], 'out.min': [33, 64, 91, 47], -- GitLab From 853a9f49ac3b25d7c5cbea958c32150697e0406f Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 17:37:49 +0200 Subject: [PATCH 181/399] FIX: typo --- tests/tests_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_data.py b/tests/tests_data.py index b54c6f2..ac789ad 100644 --- a/tests/tests_data.py +++ b/tests/tests_data.py @@ -2,7 +2,7 @@ import json from pathlib import Path import pyotb -FILEPATH = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tiff" +FILEPATH = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" try: INPUT = pyotb.Input(FILEPATH) except RuntimeError as err: -- GitLab From 781f10a5cdda63268a15b8be7342a01b1fe14ba5 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 17:51:14 +0200 Subject: [PATCH 182/399] ENH: error message in test_data --- tests/tests_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_data.py b/tests/tests_data.py index ac789ad..3aefc9c 100644 --- a/tests/tests_data.py +++ b/tests/tests_data.py @@ -7,7 +7,7 @@ try: INPUT = pyotb.Input(FILEPATH) except RuntimeError as err: if str(err.__cause__).startswith("Cannot open image "): - raise Exception("Unable to access the remote image, GitLab could be offline.") from err + raise Exception("Unable to access the remote image, GitLab might be offline.") from err raise Exception("Unexpected error while fetching test data.") from err TEST_IMAGE_STATS = { -- GitLab From 3a476b69b371c42db44ed718bd0ea9891e1d4e5b Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 19:10:08 +0200 Subject: [PATCH 183/399] ENH: add test for new vsi behaviour with case remote compressed file --- tests/test_core.py | 15 +++++++++++++-- tests/tests_data.py | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 3afffd1..5a313e0 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -11,8 +11,19 @@ def test_parameters(): assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304) -def test_input_with_vsi(): - # Ensure old way is still working - this should raise RuntimeError when the path is malformed +def test_input_vsi(): + # Simple remote file + info = pyotb.ReadImageInfo("https://fake.com/image.tif", frozen=True) + assert info.app.GetParameterValue("in") == "/vsicurl/https://fake.com/image.tif" + assert info.parameters["in"] == "https://fake.com/image.tif" + # Compressed remote file + info = pyotb.ReadImageInfo("https://fake.com/image.tif.zip", frozen=True) + assert info.app.GetParameterValue("in") == "/vsizip//vsicurl/https://fake.com/image.tif.zip" + assert info.parameters["in"] == "https://fake.com/image.tif.zip" + + +def test_input_vsi_from_user(): + # Ensure old way is still working: ExtractROI will raise RuntimeError if a path is malformed pyotb.Input("/vsicurl/" + FILEPATH) diff --git a/tests/tests_data.py b/tests/tests_data.py index 3aefc9c..0f10b18 100644 --- a/tests/tests_data.py +++ b/tests/tests_data.py @@ -7,7 +7,7 @@ try: INPUT = pyotb.Input(FILEPATH) except RuntimeError as err: if str(err.__cause__).startswith("Cannot open image "): - raise Exception("Unable to access the remote image, GitLab might be offline.") from err + raise Exception("Unable to access remote image, GitLab might be offline.") from err raise Exception("Unexpected error while fetching test data.") from err TEST_IMAGE_STATS = { -- GitLab From 22a968cbab7f49ebdb48ef530c21b1e89b65b5da Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Mon, 22 May 2023 20:47:51 +0200 Subject: [PATCH 184/399] TEST: piped curl -> zip -> tiff --- tests/test_core.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index 5a313e0..228ed07 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -20,6 +20,16 @@ def test_input_vsi(): info = pyotb.ReadImageInfo("https://fake.com/image.tif.zip", frozen=True) assert info.app.GetParameterValue("in") == "/vsizip//vsicurl/https://fake.com/image.tif.zip" assert info.parameters["in"] == "https://fake.com/image.tif.zip" + # Piped curl --> zip --> tiff + ziped_tif_urls = ( + "https://github.com/OSGeo/gdal/raw/master" + "/autotest/gcore/data/byte.tif.zip", # without /vsi + "/vsizip/vsicurl/https://github.com/OSGeo/gdal/raw/master" + "/autotest/gcore/data/byte.tif.zip", # with /vsi + ) + for ziped_tif_url in ziped_tif_urls: + info = pyotb.ReadImageInfo(ziped_tif_url) + assert info["sizex"] == 20 def test_input_vsi_from_user(): -- GitLab From 59b7d0339350592ab478e6c54a785644aa585b9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr> Date: Mon, 22 May 2023 20:42:18 +0000 Subject: [PATCH 185/399] ENH : Check explicit http:// and https:// to avoid any problems --- pyotb/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index d7de84a..f380964 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1180,7 +1180,7 @@ class Input(App): """ super().__init__("ExtractROI", {"in": filepath}, frozen=True) self._name = f"Input from {filepath}" - if not filepath.startswith(("/vsi", "http", "ftp")): + if not filepath.startswith(("/vsi", "http://", "https://", "ftp://")): filepath = Path(filepath) self.filepath = filepath self.propagate_dtype() @@ -1241,7 +1241,7 @@ class Output(OTBObject): @filepath.setter def filepath(self, path: str): if isinstance(path, str): - if path and not path.startswith(("/vsi", "http", "ftp")): + if path and not path.startswith(("/vsi", "http://", "https://", "ftp://")): path = Path(path.split("?")[0]) self._filepath = path -- GitLab From bd093487f06dd02c8ae2d975f1600bfe08117c12 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 20:52:14 +0000 Subject: [PATCH 186/399] ENH: use dict to search for vsi suffixes --- pyotb/core.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index f380964..6debae4 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1285,15 +1285,18 @@ def add_vsi_prefix(filepath: str | Path) -> str: if filepath.startswith(("https://", "http://", "ftp://")): filepath = "/vsicurl/" + filepath # Compressed file + prefixes = { + ".tar": "vsitar", + ".tgz": "vsitar", + ".gz": "vsigzip", + ".7z": "vsi7z", + ".zip": "vsizip", + ".rar": "vsirar" + } basename = filepath.split("?")[0] - if basename.endswith((".tar", ".tar.gz", ".tgz")): - filepath = "/vsitar/" + filepath - elif basename.endswith(".gz"): - filepath = "/vsigzip/" + filepath - elif basename.endswith(".7z"): - filepath = "/vsi7z/" + filepath - elif basename.endswith(".zip"): - filepath = "/vsizip/" + filepath + ext = Path(basename).suffix + if ext in prefixes: + filepath = f"/{prefixes[ext]}/{filepath}" return filepath -- GitLab From 20a9225f85adc6d89c3abe7288c176e36da559cf Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 22:56:04 +0200 Subject: [PATCH 187/399] FIX: bad ident in MR patch --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 6debae4..03b1203 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1296,7 +1296,7 @@ def add_vsi_prefix(filepath: str | Path) -> str: basename = filepath.split("?")[0] ext = Path(basename).suffix if ext in prefixes: - filepath = f"/{prefixes[ext]}/{filepath}" + filepath = f"/{prefixes[ext]}/{filepath}" return filepath -- GitLab From 984691e45ccdc132174853c43debd3dbc8c6d1e1 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 22 May 2023 23:47:46 +0200 Subject: [PATCH 188/399] ENH: make recursion in summarize function more readable --- pyotb/core.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 03b1203..74e426f 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1404,22 +1404,20 @@ def summarize( parameters of an app and its parents """ - - def strip_path(key: str, param: str | Any): - """Truncate text after the first "?" character in filepath.""" - param_summary = summarize(param) - if strip_input_paths and obj.is_input(key) or strip_output_paths and obj.is_output(key): - if isinstance(param, str) and "?" in param_summary: - param_summary = param_summary.split("?")[0] - return param_summary + def strip_path(param: str | Any): + if not isinstance(param, str): + return summarize(param) + return param.split("?")[0] if isinstance(obj, Output): return summarize(obj.parent_pyotb_app) if not isinstance(obj, App): return obj # If we are here, "obj" is an App - parameters = { - key: [strip_path(key, p) for p in param] if isinstance(param, list) else strip_path(key, param) - for key, param in obj.parameters.items() - } + parameters = {} + for key, param in obj.parameters.items(): + if strip_input_paths and obj.is_input(key) or strip_output_paths and obj.is_output(key): + parameters[key] = [strip_path(p) for p in param] if isinstance(param, list) else strip_path(param) + else: + parameters[key] = [summarize(p) for p in param] if isinstance(param, list) else summarize(param) return {"name": obj.app.GetName(), "parameters": parameters} -- GitLab From bbecbd2b2e05a39fa1b97cd5e35491bea397aacb Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 23 May 2023 00:02:10 +0200 Subject: [PATCH 189/399] ENH: add list param case in summarize recursion --- pyotb/core.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 74e426f..28e7e0d 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1409,15 +1409,17 @@ def summarize( return summarize(param) return param.split("?")[0] + if isinstance(obj, list): + return [summarize(o) for o in obj] if isinstance(obj, Output): return summarize(obj.parent_pyotb_app) if not isinstance(obj, App): return obj - # If we are here, "obj" is an App + parameters = {} for key, param in obj.parameters.items(): if strip_input_paths and obj.is_input(key) or strip_output_paths and obj.is_output(key): parameters[key] = [strip_path(p) for p in param] if isinstance(param, list) else strip_path(param) else: - parameters[key] = [summarize(p) for p in param] if isinstance(param, list) else summarize(param) + parameters[key] = summarize(param) return {"name": obj.app.GetName(), "parameters": parameters} -- GitLab From 78d771abcfab878711721e61752847f7801350bf Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 23 May 2023 11:08:52 +0200 Subject: [PATCH 190/399] TEST: check if remote tif file is accessible before trying to use it --- .gitlab-ci.yml | 2 +- pyproject.toml | 2 +- tests/tests_data.py | 15 ++++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a60518f..f5a291e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -86,7 +86,7 @@ pages_test: junit: test-*.xml before_script: - wget $IMAGE_URL -O $TEST_INPUT_IMAGE - - pip install pytest + - pip install pytest requests test_core: extends: .tests diff --git a/pyproject.toml b/pyproject.toml index 845e218..ac4d3df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers=[ ] [project.optional-dependencies] -dev = ["pytest", "pylint", "codespell", "pydocstyle", "tomli"] +dev = ["pytest", "pylint", "codespell", "pydocstyle", "tomli", "requests"] [project.urls] documentation = "https://pyotb.readthedocs.io" diff --git a/tests/tests_data.py b/tests/tests_data.py index 0f10b18..359081d 100644 --- a/tests/tests_data.py +++ b/tests/tests_data.py @@ -1,15 +1,16 @@ import json from pathlib import Path +import requests import pyotb + FILEPATH = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" -try: - INPUT = pyotb.Input(FILEPATH) -except RuntimeError as err: - if str(err.__cause__).startswith("Cannot open image "): - raise Exception("Unable to access remote image, GitLab might be offline.") from err - raise Exception("Unexpected error while fetching test data.") from err +response = requests.get(FILEPATH, timeout=5) +code = response.status_code +if code != 200: + raise requests.HTTPError(f"Unable to fetch remote image, GitLab might be offline (HTTP {code}).") +INPUT = pyotb.Input(FILEPATH) TEST_IMAGE_STATS = { 'out.mean': [79.5505, 109.225, 115.456, 249.349], 'out.min': [33, 64, 91, 47], @@ -18,7 +19,7 @@ TEST_IMAGE_STATS = { } json_file = Path(__file__).parent / "serialized_apps.json" -with json_file.open("r") as js: +with json_file.open("r", encoding="utf-8") as js: data = json.load(js) SIMPLE_SERIALIZATION = data["SIMPLE"] COMPLEX_SERIALIZATION = data["COMPLEX"] -- GitLab From 681222b71adafe60be25b6111a5c87eddfb5c321 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 24 May 2023 10:55:39 +0200 Subject: [PATCH 191/399] FEAT: accept dotted parameters using underscore --- pyotb/core.py | 2 ++ tests/test_core.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/pyotb/core.py b/pyotb/core.py index 28e7e0d..903f71f 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -589,6 +589,8 @@ class App(OTBObject): parameters.update(self.__parse_args(args)) # Going through all arguments for key, obj in parameters.items(): + if "_" in key: + key = key.replace("_", ".") if key not in self.parameters_keys: raise KeyError( f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}" diff --git a/tests/test_core.py b/tests/test_core.py index 228ed07..f766c58 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -37,6 +37,11 @@ def test_input_vsi_from_user(): pyotb.Input("/vsicurl/" + FILEPATH) +def test_param_with_underscore(): + app = pyotb.OrthoRectification(io_in=INPUT, map_epsg_code=2154) + assert app.parameters["map.epsg.code"] == 2154 + + def test_wrong_key(): with pytest.raises(KeyError): pyotb.BandMath(INPUT, expression="im1b1") -- GitLab From 08de3dd8b83cc925ae7c3e146a927b7ae3025dd5 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 31 May 2023 10:04:14 +0000 Subject: [PATCH 192/399] Update README.md --- .gitlab-ci.yml | 20 ++- AUTHORS.md | 10 ++ README.md | 337 +++++----------------------------------- doc/MISC.md | 5 +- doc/comparison_otb.md | 133 +++++++++++----- doc/features.md | 7 +- doc/installation.md | 30 +++- doc/interaction.md | 17 +- doc/managing_loggers.md | 5 +- doc/otb_versions.md | 13 +- doc/troubleshooting.md | 9 +- 11 files changed, 230 insertions(+), 356 deletions(-) create mode 100644 AUTHORS.md diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f5a291e..be29725 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,13 +20,21 @@ stages: allow_failure: true codespell: + rules: + - changes: + - pyotb/*.py + - tests/*.py + - README.md extends: .static_analysis before_script: - pip install codespell script: - - codespell {pyotb,tests} + - codespell {pyotb,tests,doc,README.md} flake8: + rules: + - changes: + - pyotb/*.py extends: .static_analysis before_script: - pip install flake8 @@ -34,6 +42,9 @@ flake8: - flake8 --max-line-length=120 $PWD/pyotb --ignore=F403,E402,F401,W503,W504 pydocstyle: + rules: + - changes: + - pyotb/*.py extends: .static_analysis before_script: - pip install pydocstyle tomli @@ -41,6 +52,9 @@ pydocstyle: - pydocstyle $PWD/pyotb pylint: + rules: + - changes: + - pyotb/*.py extends: .static_analysis before_script: - pip install pylint @@ -73,6 +87,10 @@ pages_test: .tests: stage: Tests + rules: + - changes: + - pyotb/*.py + - tests/*.py allow_failure: false variables: OTB_ROOT: /opt/otb diff --git a/AUTHORS.md b/AUTHORS.md new file mode 100644 index 0000000..7ac46df --- /dev/null +++ b/AUTHORS.md @@ -0,0 +1,10 @@ +# Project authors + +## Initial codebase + +* Nicolas NARÇON - IT engineer @ ESA - Rome (Italy) + +## Current maintainers + +* Rémi CRESSON - RS research engineer @ INRAe - Montpellier (France) +* Vincent DELBAR - GIS engineer @ La TeleScop - Montpellier (France) diff --git a/README.md b/README.md index 60659a8..d72b294 100644 --- a/README.md +++ b/README.md @@ -1,318 +1,67 @@ -# pyotb: a pythonic extension of OTB +# pyotb: a pythonic extension of Orfeo Toolbox -Full documentation is available at [pyotb.readthedocs.io](https://pyotb.readthedocs.io/) - -[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/releases) +[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/releases) [](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/master) +[](https://pyotb.readthedocs.io/en/master/) +**pyotb** wraps the [Orfeo Toolbox](https://www.orfeo-toolbox.org/) (OTB) +python bindings to make it more developer friendly. -## Installation -Requirements: -- python>=3.5 and numpy -- OrfeoToolBox python API - -```bash -pip install pyotb --upgrade -``` - -For Python>=3.6, latest version available is pyotb 1.5.1 For Python 3.5, latest version available is pyotb 1.2.2 - -## Quickstart: running an OTB application as a oneliner -pyotb has been written so that it is more convenient to run an application in Python. - -You can pass the parameters of an application as a dictionary : -```python -import pyotb -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) -``` -Note that pyotb has a 'lazy' evaluation: it only performs operation when it is needed, i.e. results are written to disk. -Thus, the previous line doesn't trigger the application. - -To actually trigger the application execution, you need to write the result to disk: - -```python -resampled.write('output.tif') # this is when the application actually runs -``` - -## Using Python keyword arguments -It is also possible to use the Python keyword arguments notation for passing the parameters: -```python -output = pyotb.SuperImpose(inr='reference_image.tif', inm='image.tif') -``` -is equivalent to: -```python -output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'}) -``` - -Limitations : for this notation, python doesn't accept the parameter `in` or any parameter that contains a `.`. E.g., it is not possible to use `pyotb.RigidTransformResample(in=input_path...)` or `pyotb.VectorDataExtractROI(io.vd=vector_path...)`. - - - - -## In-memory connections -The big asset of pyotb is the ease of in-memory connections between apps. - -Let's start from our previous example. Consider the case where one wants to apply optical calibration and binary morphological dilatation -following the undersampling. - -Using pyotb, you can pass the output of an app as input of another app : -```python -import pyotb - -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) - -calibrated = pyotb.OpticalCalibration({'in': resampled, 'level': 'toa'}) - -dilated = pyotb.BinaryMorphologicalOperation({'in': calibrated, 'out': 'output.tif', 'filter': 'dilate', - 'structype': 'ball', 'xradius': 3, 'yradius': 3}) -dilated.write('result.tif') -``` - -## Writing the result of an app -Any pyotb object can be written to disk using the `write` method, e.g. : - -```python -import pyotb - -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) -# Here you can set optionally pixel type and extended filename variables -resampled.write({'out': 'output.tif'}, pixel_type='uint16', filename_extension='?nodata=65535') -``` - -Another possibility for writing results is to set the output parameter when initializing the application: -```python -import pyotb - -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', 'out': 'output.tif', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) -# Here you can set optionally pixel type and extended filename variables -resampled.write(pixel_type='uint16', filename_extension='?nodata=65535') -``` - -## Arithmetic operations -Every pyotb object supports arithmetic operations, such as addition, subtraction, comparison... -Consider an example where we want to compute a vegeteation mask from NDVI, i.e. the arithmetic operation `(nir - red) / (nir + red) > 0.3` - -With pyotb, one can simply do : -```python -import pyotb - -# transforming filepaths to pyotb objects -nir, red = pyotb.Input('nir.tif'), pyotb.Input('red.tif') - -res = (nir - red) / (nir + red) > 0.3 -print(res.exp) # prints the BandMath expression: "((im1b1 - im2b1) / (im1b1 + im2b1)) > 0.3 ? 1 : 0" -res.write('vegetation_mask.tif', pixel_type='uint8') -``` - -## Slicing -pyotb objects support slicing in a Python fashion : - -```python -import pyotb - -# transforming filepath to pyotb object -inp = pyotb.Input('my_image.tif') - -inp[:, :, :3] # selecting first 3 bands -inp[:, :, [0, 1, 4]] # selecting bands 1, 2 & 5 -inp[:1000, :1000] # selecting 1000x1000 subset, same as inp[:1000, :1000, :] -inp[:100, :100].write('my_image_roi.tif') # write cropped image to disk -``` - -## Numpy-inspired functions -Some functions have been written, entirely based on OTB, to mimic the behavior of some well-known numpy functions. -### pyotb.where -Equivalent of `numpy.where`. -It is the equivalent of the muparser syntax `condition ? x : y` that can be used in OTB's BandMath. - -```python -import pyotb - -# transforming filepaths to pyotb objects -labels, image1, image2 = pyotb.Input('labels.tif'), pyotb.Input('image1.tif') , pyotb.Input('image2.tif') - -# If labels = 1, returns image1. Else, returns image2 -res = pyotb.where(labels == 1, image1, image2) # this would also work: pyotb.where(labels == 1, 'image1.tif', 'image2.tif') - -# A more complex example -# If labels = 1, returns image1. If labels = 2, returns image2. If labels = 3, returns 3. Else 0 -res = pyotb.where(labels == 1, image1, - pyotb.where(labels == 2, image2, - pyotb.where(labels == 3, 3, 0))) - -``` - -### pyotb.clip -Equivalent of `numpy.clip`. Clip (limit) the values in a raster to a range. - -```python -import pyotb - -res = pyotb.clip('my_image.tif', 0, 255) # clips the values between 0 and 255 -``` - -### pyotb.all -Equivalent of `numpy.all`. - -For only one image, this function checks that all bands of the image are True (i.e. !=0) and outputs -a singleband boolean raster. -For several images, this function checks that all images are True (i.e. !=0) and outputs -a boolean raster, with as many bands as the inputs. - - -### pyotb.any -Equivalent of `numpy.any`. - -For only one image, this function checks that at least one band of the image is True (i.e. !=0) and outputs -a singleband boolean raster. -For several images, this function checks that at least one of the images is True (i.e. !=0) and outputs -a boolean raster, with as many bands as the inputs. - - -## Interaction with Numpy - -pyotb objects can be transparently used in numpy functions. - -For example: - -```python -import pyotb -import numpy as np - -inp = pyotb.Input('image.tif') # this is a pyotb object - -# Creating a numpy array of noise -white_noise = np.random.normal(0, 50, size=inp.shape) # this is a numpy object - -# Adding the noise to the image -noisy_image = inp + white_noise # magic: this is a pyotb object that has the same georeference as input. - # `np.add(inp, white_noise)` would have worked the same -noisy_image.write('image_plus_noise.tif') -``` -Limitations : -- The whole image is loaded into memory -- The georeference can not be modified. Thus, numpy operations can not change the image or pixel size - - -## Export to rasterio -pyotb objects can also be exported in a format that is usable by rasterio. - -For example: +## Key features -```python -import pyotb -import rasterio -from scipy import ndimage - -# Pansharpening + NDVI + creating bare soils mask -pxs = pyotb.BundleToPerfectSensor(inp='panchromatic.tif', inxs='multispectral.tif') -ndvi = pyotb.RadiometricIndices({'in': pxs, 'channels.red': 3, 'channels.nir': 4, 'list': 'Vegetation:NDVI'}) -bare_soil_mask = (ndvi < 0.3) - -# Exporting the result as array & profile usable by rasterio -mask_array, profile = bare_soil_mask.to_rasterio() - -# Doing something in Python that is not possible with OTB, e.g. gathering the contiguous groups of pixels -# with an integer index -labeled_mask_array, nb_groups = ndimage.label(mask_array) - -# Writing the result to disk -with rasterio.open('labeled_bare_soil.tif', 'w', **profile) as f: - f.write(labeled_mask_array) - -``` -This way of exporting pyotb objects is more flexible that exporting to numpy, as the user gets the `profile` dictionary. -If the georeference or pixel size is modified, the user can update the `profile` accordingly. +- Easy use of OTB applications from python +- Simplify common sophisticated I/O features of OTB +- Lazy execution of in-memory pipelines with OTB streaming mechanism +- Interoperable with popular python libraries (numpy, rasterio) +- Extensible +Documentation hosted at [pyotb.readthedocs.io](https://pyotb.readthedocs.io/). -## Interaction with Tensorflow +## Example -We saw that numpy operations had some limitations. To bypass those limitations, it is possible to use some Tensorflow operations on pyotb objects. +Building a simple pipeline with OTB applications - -You need a working installation of OTBTF >=3.0 for this and then the code is like this: - -```python +```py import pyotb -def scalar_product(x1, x2): - """This is a function composed of tensorflow operations.""" - import tensorflow as tf - return tf.reduce_sum(tf.multiply(x1, x2), axis=-1) +# RigidTransformResample application, with input parameters as dict +resampled = pyotb.RigidTransformResample({ + "in": "https://some.remote.data/input.tif", # Note: no /vsicurl/... + "interpolator": "linear", + "transform.type.id.scaley": 0.5, + "transform.type.id.scalex": 0.5 +}) -# Compute the scalar product -res = pyotb.run_tf_function(scalar_product)('image1.tif', 'image2.tif') # magic: this is a pyotb object -res.write('scalar_product.tif') -``` +# OpticalCalibration, with automatic input parameters resolution +calib = pyotb.OpticalCalibration(resampled) -For some easy syntax, one can use `pyotb.run_tf_function` as a function decorator, such as: -```python -import pyotb +# BandMath, with input parameters passed as kwargs +ndvi = pyotb.BandMath(calib, exp="ndvi(im1b1, im1b4)") -@pyotb.run_tf_function # The decorator enables the use of pyotb objects as inputs/output of the function -def scalar_product(x1, x2): - import tensorflow as tf - return tf.reduce_sum(tf.multiply(x1, x2), axis=-1) +# Pythonic slicing using lazy computation (no memory used) +roi = ndvi[20:586, 9:572] -res = scalar_product('image1.tif', 'image2.tif') # magic: this is a pyotb object +# Pipeline execution +# The actual computation happens here ! +roi.write("output.tif", pixel_type="float") ``` -Advantages : -- The process supports streaming, hence the whole image is **not** loaded into memory -- Can be integrated in OTB pipelines - -Limitations : -- It is not possible to use the tensorflow python API inside a script where OTBTF is used because of compilation issues -between Tensorflow and OTBTF, i.e. `import tensorflow` doesn't work in a script where OTBTF apps have been initialized - - -## Some examples -### Compute the mean of several rasters, taking into account NoData -Let's consider we have at disposal 73 NDVI rasters for a year, where clouds have been masked with NoData (nodata value of -10 000 for example). - -Goal: compute the mean across time (keeping the spatial dimension) of the NDVIs, excluding cloudy pixels. Piece of code to achieve that: +pyotb's objects also enable easy interoperability with +[numpy](https://numpy.org/) and [rasterio](https://rasterio.readthedocs.io/): ```python -import pyotb +# Numpy and RasterIO style attributes +print(roi.shape, roi.dtype, roi.transform) +print(roi.metadata) -nodata = -10000 -ndvis = [pyotb.Input(path) for path in ndvi_paths] +# Other useful information +print(roi.get_infos()) +print(roi.get_statistics()) -# For each pixel location, summing all valid NDVI values -summed = sum([pyotb.where(ndvi != nodata, ndvi, 0) for ndvi in ndvis]) - -# Printing the generated BandMath expression -print(summed.exp) # this returns a very long exp: "0 + ((im1b1 != -10000) ? im1b1 : 0) + ((im2b1 != -10000) ? im2b1 : 0) + ... + ((im73b1 != -10000) ? im73b1 : 0)" - -# For each pixel location, getting the count of valid pixels -count = sum([pyotb.where(ndvi == nodata, 0, 1) for ndvi in ndvis]) - -mean = summed / count # BandMath exp of this is very long: "(0 + ((im1b1 != -10000) ? im1b1 : 0) + ... + ((im73b1 != -10000) ? im73b1 : 0)) / (0 + ((im1b1 == -10000) ? 0 : 1) + ... + ((im73b1 == -10000) ? 0 : 1))" -mean.write('ndvi_annual_mean.tif') +array = roi.to_numpy() +array, profile = roi.to_rasterio() ``` -Note that no actual computation is executed before the last line where the result is written to disk. - -### Process raw Pleiades data -This is a common case of Pleiades data preprocessing : optical calibration -> orthorectification -> pansharpening +## Contributing -```python -import pyotb -srtm = '/media/data/raster/nasa/srtm_30m' -geoid = '/media/data/geoid/egm96.grd' - -pan = pyotb.OpticalCalibration('IMG_PHR1A_P_001/DIM_PHR1A_P_201509011347379_SEN_1791374101-001.XML', level='toa') -ms = pyotb.OpticalCalibration('IMG_PHR1A_MS_002/DIM_PHR1A_MS_201509011347379_SEN_1791374101-002.XML', level='toa') - -pan_ortho = pyotb.OrthoRectification({'io.in': pan, 'elev.dem': srtm, 'elev.geoid': geoid}) -ms_ortho = pyotb.OrthoRectification({'io.in': ms, 'elev.dem': srtm, 'elev.geoid': geoid}) - -pxs = pyotb.BundleToPerfectSensor(inp=pan_ortho, inxs=ms_ortho, method='bayes', mode="default") - -# Here we trigger every app in the pipeline and the process is blocked until result is written to disk -pxs.write('pxs_image.tif', pixel_type='uint16', filename_extension='?gdal:co:COMPRESS=DEFLATE&gdal:co:PREDICTOR=2') -``` +Contributions are welcome on [Github](https://github.com/orfeotoolbox/pyotb) or the source repository hosted on the Orfeo ToolBox [GitLab](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb). diff --git a/doc/MISC.md b/doc/MISC.md index 8da0eea..2a92b03 100644 --- a/doc/MISC.md +++ b/doc/MISC.md @@ -1,4 +1,5 @@ ## Miscellaneous: Work with images with different footprints / resolutions + OrfeoToolBox provides a handy `Superimpose` application that enables the projection of an image into the geometry of another one. In pyotb, a function has been created to handle more than 2 images. @@ -17,6 +18,7 @@ print(s2_image.shape) # (286, 195, 4) print(vhr_image.shape) # (2048, 2048, 3) print(labels.shape) # (1528, 1360, 1) ``` + Our goal is to obtain all images at the same footprint, same resolution and same shape. Let's consider we want the intersection of all footprints and the same resolution as `labels` image. @@ -25,7 +27,8 @@ Let's consider we want the intersection of all footprints and the same resolutio Here is the final result :  -The piece of code to achieve this : +The piece of code to achieve this : + ```python s2_image, vhr_image, labels = pyotb.define_processing_area(s2_image, vhr_image, labels, window_rule='intersection', pixel_size_rule='same_as_input', diff --git a/doc/comparison_otb.md b/doc/comparison_otb.md index 612eab6..a733678 100644 --- a/doc/comparison_otb.md +++ b/doc/comparison_otb.md @@ -1,7 +1,9 @@ ## Comparison between otbApplication and pyotb ### Single application execution + Using OTB, the code would be like : + ```python import otbApplication @@ -12,66 +14,123 @@ resampled.SetParameterString('interpolator', 'linear') resampled.SetParameterFloat('transform.type.id.scalex', 0.5) resampled.SetParameterFloat('transform.type.id.scaley', 0.5) resampled.SetParameterString('out', 'output.tif') -resampled.SetParameterOutputImagePixelType('out', otbApplication.ImagePixelType_uint16) +resampled.SetParameterOutputImagePixelType( + 'out', otbApplication.ImagePixelType_uint16 +) + resampled.ExecuteAndWriteOutput() ``` Using pyotb: + ```python import pyotb -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) + +resampled = pyotb.RigidTransformResample({ + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 +}) + resampled.write('output.tif', pixel_type='uint16') ``` ### In-memory connections -Using OTB : +<table> +<tr> +<th> OTB </th> +<th> pyotb </th> +</tr> +<tr> +<td> + ```python import otbApplication -resampled = otbApplication.Registry.CreateApplication('RigidTransformResample') -resampled.SetParameterString('in', 'my_image.tif') -resampled.SetParameterString('interpolator', 'linear') -resampled.SetParameterFloat('transform.type.id.scalex', 0.5) -resampled.SetParameterFloat('transform.type.id.scaley', 0.5) -resampled.Execute() - -calibrated = otbApplication.Registry.CreateApplication('OpticalCalibration') -calibrated.ConnectImage('in', resampled, 'out') -calibrated.SetParameterString('level', 'toa') -calibrated.Execute() - -dilated = otbApplication.Registry.CreateApplication('BinaryMorphologicalOperation') -dilated.ConnectImage('in', calibrated, 'out') -dilated.SetParameterString("filter", 'dilate') -dilated.SetParameterString("structype", 'ball') -dilated.SetParameterInt("xradius", 3) -dilated.SetParameterInt("yradius", 3) -dilated.SetParameterString('out', 'output.tif') -dilated.SetParameterOutputImagePixelType('out', otbApplication.ImagePixelType_uint16) -dilated.ExecuteAndWriteOutput() +app1 = otbApplication.Registry.CreateApplication( + 'RigidTransformResample' +) +app1.SetParameterString('in', 'my_image.tif') +app1.SetParameterString('interpolator', 'linear') +app1.SetParameterFloat( + 'transform.type.id.scalex', + 0.5 +) +app1.SetParameterFloat( + 'transform.type.id.scaley', + 0.5 +) +app1.Execute() + +app2 = otbApplication.Registry.CreateApplication( + 'OpticalCalibration' +) +app2.ConnectImage('in', app1, 'out') +app2.SetParameterString('level', 'toa') +app2.Execute() + +app3 = otbApplication.Registry.CreateApplication( + 'BinaryMorphologicalOperation' +) +app3.ConnectImage('in', app2, 'out') +app3.SetParameterString('filter', 'dilate') +app3.SetParameterString('structype', 'ball') +app3.SetParameterInt('xradius', 3) +app3.SetParameterInt('yradius', 3) +app3.SetParameterString('out', 'output.tif') +app3.SetParameterOutputImagePixelType( + 'out', + otbApplication.ImagePixelType_uint16 +) +app3.ExecuteAndWriteOutput() ``` -Using pyotb: +</td> +<td> + ```python import pyotb -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) - -calibrated = pyotb.OpticalCalibration({'in': resampled, 'level': 'toa'}) - -dilated = pyotb.BinaryMorphologicalOperation({'in': calibrated, 'out': 'output.tif', 'filter': 'dilate', - 'structype': 'ball', 'xradius': 3, 'yradius': 3}) -dilated.write('result.tif', pixel_type='uint16') +app1 = pyotb.RigidTransformResample({ + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 +}) + +app2 = pyotb.OpticalCalibration({ + 'in': app1, + 'level': 'toa' +}) + +app3 = pyotb.BinaryMorphologicalOperation({ + 'in': app2, + 'out': 'output.tif', + 'filter': 'dilate', + 'structype': 'ball', + 'xradius': 3, + 'yradius': 3 +}) + +app3.write( + 'result.tif', + pixel_type='uint16' +) ``` +</td> +</tr> +</table> + ### Arithmetic operations + Every pyotb object supports arithmetic operations, such as addition, subtraction, comparison... Consider an example where we want to perform the arithmetic operation `image1 * image2 - 2*image3` Using OTB, the following code works for 3-bands images : + ```python import otbApplication @@ -85,6 +144,7 @@ bmx.ExecuteAndWriteOutput() ``` With pyotb, the following works with images of any number of bands : + ```python import pyotb @@ -98,6 +158,7 @@ res.write('output.tif', pixel_type='uint8') ### Slicing Using OTB, for selection bands or ROI, the code looks like: + ```python import otbApplication @@ -119,7 +180,7 @@ extracted.SetParameterFloat('mode.extent.lry', 999) extracted.Execute() ``` -Instead, using pyotb: +Instead, using pyotb: ```python import pyotb @@ -129,4 +190,4 @@ inp = pyotb.Input('my_image.tif') extracted = inp[:, :, :3] # selecting first 3 bands extracted = inp[:1000, :1000] # selecting 1000x1000 subset -``` \ No newline at end of file +``` diff --git a/doc/features.md b/doc/features.md index 7bc78ce..cc2c39e 100644 --- a/doc/features.md +++ b/doc/features.md @@ -1,8 +1,10 @@ ## Arithmetic operations + Every pyotb object supports arithmetic operations, such as addition, subtraction, comparison... Consider an example where we want to compute a vegeteation mask from NDVI, i.e. the arithmetic operation `(nir - red) / (nir + red) > 0.3` With pyotb, one can simply do : + ```python import pyotb @@ -15,6 +17,7 @@ res.write('vegetation_mask.tif', pixel_type='uint8') ``` ## Slicing + pyotb objects support slicing in a Python fashion : ```python @@ -30,6 +33,7 @@ inp[:100, :100].write('my_image_roi.tif') # write cropped image to disk ``` ## Shape attributes + You can access the shape of any in-memory pyotb object. ```python @@ -39,6 +43,3 @@ import pyotb inp = pyotb.Input('my_image.tif') print(inp[:1000, :500].shape) # (1000, 500, 4) ``` - - - diff --git a/doc/installation.md b/doc/installation.md index dd7b729..de5c0ae 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -1,11 +1,31 @@ +## Prerequisites -## Requirements -- Python>=3.5 and NumPy -- Orfeo ToolBox python API (instructions available [on the official website](https://www.orfeo-toolbox.org/CookBook/Installation.html)) +Requirements: + +- Python >= 3.7 and NumPy +- Orfeo ToolBox binaries (follow these + [instructions](https://www.orfeo-toolbox.org/CookBook/Installation.html)) +- Orfeo ToolBox python binding (follow these + [instructions](https://www.orfeo-toolbox.org/CookBook/Installation.html)) + +## Install with pip -## Installation ```bash pip install pyotb --upgrade ``` -For Python>=3.6, latest version available is pyotb 1.5.0. For Python 3.5, latest version available is pyotb 1.2.2 +For development, use the following: + +```bash +git clone https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb +cd pyotb +pip install -e ".[dev]" +``` + +## Old versions + +If you need compatibility with python3.6, install `pyotb<2.0` and for + python3.5 use `pyotb==1.2.2`. + +For Python>=3.6, latest version available is pyotb 1.5.0. +For Python 3.5, latest version available is pyotb 1.2.2 diff --git a/doc/interaction.md b/doc/interaction.md index d44ca68..217dbaa 100644 --- a/doc/interaction.md +++ b/doc/interaction.md @@ -1,16 +1,15 @@ ## Export to Numpy pyotb objects can be exported to numpy array. + ```python import pyotb import numpy as np calibrated = pyotb.OpticalCalibration('image.tif', level='toa') # this is a pyotb object arr = np.asarray(calibrated) # same as calibrated.to_numpy() - ``` - ## Interaction with Numpy pyotb objects can be transparently used in numpy functions. @@ -31,13 +30,14 @@ noisy_image = inp + white_noise # magic: this is a pyotb object that has the sa # `np.add(inp, white_noise)` would have worked the same noisy_image.write('image_plus_noise.tif') ``` -Limitations : + +Limitations : - The whole image is loaded into memory - The georeference can not be modified. Thus, numpy operations can not change the image or pixel size - ## Export to rasterio + pyotb objects can also be exported in a format that is usable by rasterio. For example: @@ -62,17 +62,15 @@ labeled_mask_array, nb_groups = ndimage.label(mask_array) # Writing the result to disk with rasterio.open('labeled_bare_soil.tif', 'w', **profile) as f: f.write(labeled_mask_array) - ``` + This way of exporting pyotb objects is more flexible that exporting to numpy, as the user gets the `profile` dictionary. If the georeference or pixel size is modified, the user can update the `profile` accordingly. - ## Interaction with Tensorflow We saw that numpy operations had some limitations. To bypass those limitations, it is possible to use some Tensorflow operations on pyotb objects. - You need a working installation of OTBTF >=3.0 for this and then the code is like this: ```python @@ -89,6 +87,7 @@ res.write('scalar_product.tif') ``` For some easy syntax, one can use `pyotb.run_tf_function` as a function decorator, such as: + ```python import pyotb @@ -107,5 +106,5 @@ Advantages : Limitations : -- It is not possible to use the tensorflow python API inside a script where OTBTF is used because of compilation issues -between Tensorflow and OTBTF, i.e. `import tensorflow` doesn't work in a script where OTBTF apps have been initialized +- It is not possible to use the tensorflow python API inside a script where OTBTF is used because of compilation issues + between Tensorflow and OTBTF, i.e. `import tensorflow` doesn't work in a script where OTBTF apps have been initialized diff --git a/doc/managing_loggers.md b/doc/managing_loggers.md index f329d5e..31c960b 100644 --- a/doc/managing_loggers.md +++ b/doc/managing_loggers.md @@ -9,6 +9,7 @@ If none of those two variables is set, the logger level will be set to 'INFO'. Available levels are : DEBUG, INFO, WARNING, ERROR, CRITICAL You may also change the logger level after import (for pyotb only) with the function `set_logger_level`. + ```python import pyotb pyotb.set_logger_level('DEBUG') @@ -20,6 +21,7 @@ One useful trick is to redirect these logs to a file. This can be done using the ## Named applications in logs It is possible to change an app name in order to track it easily in the logs : + ```python import os os.environ['PYOTB_LOGGER_LEVEL'] = 'DEBUG' @@ -29,8 +31,9 @@ bm = pyotb.BandMath(['image.tif'], exp='im1b1 * 100') bm.name = 'CustomBandMathApp' bm.execute() ``` + ```text 2022-06-14 14:22:38 (DEBUG) [pyOTB] CustomBandMathApp: run execute() with parameters={'exp': 'im1b1 * 100', 'il': ['/home/vidlb/Téléchargements/test_4b.tif']} 2022-06-14 14:22:38 (INFO) BandMath: Image #1 has 4 components 2022-06-14 14:22:38 (DEBUG) [pyOTB] CustomBandMathApp: execution succeeded -``` \ No newline at end of file +``` diff --git a/doc/otb_versions.md b/doc/otb_versions.md index 6b06c28..7eeb5cf 100644 --- a/doc/otb_versions.md +++ b/doc/otb_versions.md @@ -1,21 +1,25 @@ ## System with multiple OTB versions -If you want to quickly switch between OTB versions, or override the default system version, you may use the `OTB_ROOT` env variable : +If you want to quickly switch between OTB versions, or override the default system version, you may use the `OTB_ROOT` env variable : + ```python import os # This is equivalent to "[set/export] OTB_ROOT=/opt/otb" before launching python os.environ['OTB_ROOT'] = '/opt/otb' import pyotb ``` + ```text 2022-06-14 13:59:03 (INFO) [pyOTB] Preparing environment for OTB in /opt/otb 2022-06-14 13:59:04 (INFO) [pyOTB] Successfully loaded 126 OTB applications ``` -If you try to import pyotb without having set environment, it will try to find any OTB version installed on your system: +If you try to import pyotb without having set environment, it will try to find any OTB version installed on your system: + ```python import pyotb ``` + ```text 2022-06-14 13:55:41 (INFO) [pyOTB] Failed to import OTB. Searching for it... 2022-06-14 13:55:41 (INFO) [pyOTB] Found /opt/otb/lib/otb/ @@ -24,21 +28,26 @@ import pyotb 2022-06-14 13:55:43 (INFO) [pyOTB] Preparing environment for OTB in /home/otbuser/Applications/OTB-8.0.1-Linux64 2022-06-14 13:55:44 (INFO) [pyOTB] Successfully loaded 117 OTB applications ``` + Here is the path precedence for this automatic env configuration : + ```text OTB_ROOT env variable > python bindings directory OR search for releases installations : HOME OR (for linux) : /opt/otbtf > /opt/otb > /usr/local > /usr OR (for windows) : C:/Program Files ``` + N.B. : in case `otbApplication` is found in `PYTHONPATH` (and if `OTB_ROOT` was not set), the OTB which the python API is linked to will be used. ## Fresh OTB installation If you've just installed OTB binaries in a Linux environment, you may encounter an error at first import, pyotb will help you fix it : + ```python import pyotb ``` + ```text 2022-06-14 14:00:34 (INFO) [pyOTB] Preparing environment for OTB in /home/otbuser/Applications/OTB-8.0.1-Linux64 2022-07-07 16:56:04 (CRITICAL) [pyOTB] An error occurred while importing OTB Python API diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md index e6b0b4c..e96e3e7 100644 --- a/doc/troubleshooting.md +++ b/doc/troubleshooting.md @@ -1,4 +1,4 @@ -## Troubleshooting: Known limitations +## Troubleshooting: known limitations ### Failure of intermediate writing @@ -7,8 +7,9 @@ the writings are requested. Some examples can be found below: #### Example of failures involving slicing -For some applications (non-exhaustive know list: OpticalCalibration, DynamicConvert, BandMath), we can face unexpected -failures when using channels slicing +For some applications (non-exhaustive know list: OpticalCalibration, DynamicConvert, BandMath), we can face unexpected + failures when using channels slicing + ```python import pyotb @@ -28,6 +29,7 @@ one_band.write('one_band.tif') # Failure here ``` When writing is triggered right after the application declaration, no problem occurs: + ```python import pyotb @@ -51,7 +53,6 @@ inp.write('stretched.tif') one_band.write('one_band.tif') ``` - #### Example of failures involving arithmetic operation One can meet errors when using arithmetic operations at the end of a pipeline when DynamicConvert, BandMath or -- GitLab From ea02e5331c7dc0b379a527afefa37c3d773fa6b4 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 1 Jun 2023 17:55:11 +0000 Subject: [PATCH 193/399] Format code with black --- pyotb/__init__.py | 13 +- pyotb/apps.py | 49 ++++-- pyotb/core.py | 372 ++++++++++++++++++++++++++++++++++----------- pyotb/functions.py | 266 ++++++++++++++++++++++---------- pyotb/helpers.py | 77 +++++++--- pyproject.toml | 17 ++- 6 files changed, 576 insertions(+), 218 deletions(-) diff --git a/pyotb/__init__.py b/pyotb/__init__.py index d4efd39..c9fe258 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -4,19 +4,12 @@ __version__ = "2.0.0" from .helpers import logger, set_logger_level from .apps import * -from .core import ( - App, - Input, - Output, - get_nbchannels, - get_pixel_type, - summarize -) +from .core import App, Input, Output, get_nbchannels, get_pixel_type, summarize from .functions import ( # pylint: disable=redefined-builtin all, any, - where, clip, + define_processing_area, run_tf_function, - define_processing_area + where, ) diff --git a/pyotb/apps.py b/pyotb/apps.py index bad5089..6f54b27 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -1,12 +1,14 @@ # -*- coding: utf-8 -*- """Search for OTB (set env if necessary), subclass core.App for each available application.""" from __future__ import annotations + import os -import sys import subprocess +import sys from pathlib import Path import otbApplication as otb # pylint: disable=import-error + from .core import App from .helpers import logger @@ -31,36 +33,48 @@ def get_available_applications(as_subprocess: bool = False) -> list[str]: if "PYTHONPATH" not in env: env["PYTHONPATH"] = "" env["PYTHONPATH"] += ":" + str(Path(otb.__file__).parent) - env["OTB_LOGGER_LEVEL"] = "CRITICAL" # in order to suppress warnings while listing applications + env[ + "OTB_LOGGER_LEVEL" + ] = "CRITICAL" # in order to suppress warnings while listing applications pycmd = "import otbApplication; print(otbApplication.Registry.GetAvailableApplications())" cmd_args = [sys.executable, "-c", pycmd] try: params = {"env": env, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE} with subprocess.Popen(cmd_args, **params) as process: - logger.debug('Exec "%s \'%s\'"', ' '.join(cmd_args[:-1]), pycmd) + logger.debug("Exec \"%s '%s'\"", " ".join(cmd_args[:-1]), pycmd) stdout, stderr = process.communicate() stdout, stderr = stdout.decode(), stderr.decode() # ast.literal_eval is secure and will raise more handy Exceptions than eval from ast import literal_eval # pylint: disable=import-outside-toplevel + app_list = literal_eval(stdout.strip()) assert isinstance(app_list, (tuple, list)) except subprocess.SubprocessError: logger.debug("Failed to call subprocess") except (ValueError, SyntaxError, AssertionError): - logger.debug("Failed to decode output or convert to tuple:\nstdout=%s\nstderr=%s", stdout, stderr) + logger.debug( + "Failed to decode output or convert to tuple:\nstdout=%s\nstderr=%s", + stdout, + stderr, + ) if not app_list: - logger.debug("Failed to list applications in an independent process. Falling back to local python import") + logger.debug( + "Failed to list applications in an independent process. Falling back to local python import" + ) # Find applications using the normal way if not app_list: app_list = otb.Registry.GetAvailableApplications() if not app_list: - raise SystemExit("Unable to load applications. Set env variable OTB_APPLICATION_PATH and try again.") + raise SystemExit( + "Unable to load applications. Set env variable OTB_APPLICATION_PATH and try again." + ) logger.info("Successfully loaded %s OTB applications", len(app_list)) return app_list class OTBTFApp(App): """Helper for OTBTF.""" + @staticmethod def set_nb_sources(*args, n_sources: int = None): """Set the number of sources of TensorflowModelServe. Can be either user-defined or deduced from the args. @@ -72,13 +86,17 @@ class OTBTFApp(App): """ if n_sources: - os.environ['OTB_TF_NSOURCES'] = str(int(n_sources)) + os.environ["OTB_TF_NSOURCES"] = str(int(n_sources)) else: # Retrieving the number of `source#.il` parameters - params_dic = {k: v for arg in args if isinstance(arg, dict) for k, v in arg.items()} - n_sources = len([k for k in params_dic if 'source' in k and k.endswith('.il')]) + params_dic = { + k: v for arg in args if isinstance(arg, dict) for k, v in arg.items() + } + n_sources = len( + [k for k in params_dic if "source" in k and k.endswith(".il")] + ) if n_sources >= 1: - os.environ['OTB_TF_NSOURCES'] = str(n_sources) + os.environ["OTB_TF_NSOURCES"] = str(n_sources) def __init__(self, name: str, *args, n_sources: int = None, **kwargs): """Constructor for an OTBTFApp object. @@ -98,17 +116,22 @@ class OTBTFApp(App): AVAILABLE_APPLICATIONS = get_available_applications(as_subprocess=True) # This is to enable aliases of Apps, i.e. using apps like `pyotb.AppName(...)` instead of `pyotb.App("AppName", ...)` -_CODE_TEMPLATE = """ +_CODE_TEMPLATE = ( + """ class {name}(App): - """ """ + """ + """ def __init__(self, *args, **kwargs): super().__init__('{name}', *args, **kwargs) """ +) for _app in AVAILABLE_APPLICATIONS: # Customize the behavior for some OTBTF applications. `OTB_TF_NSOURCES` is now handled by pyotb if _app in ("PatchesExtraction", "TensorflowModelTrain", "TensorflowModelServe"): - exec(_CODE_TEMPLATE.format(name=_app).replace("(App)", "(OTBTFApp)")) # pylint: disable=exec-used + exec( # pylint: disable=exec-used + _CODE_TEMPLATE.format(name=_app).replace("(App)", "(OTBTFApp)") + ) # Default behavior for any OTB application else: exec(_CODE_TEMPLATE.format(name=_app)) # pylint: disable=exec-used diff --git a/pyotb/core.py b/pyotb/core.py index 903f71f..dfe2598 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -2,11 +2,11 @@ """This module is the core of pyotb.""" from __future__ import annotations +from abc import ABC, abstractmethod from ast import literal_eval from pathlib import Path from time import perf_counter from typing import Any -from abc import ABC, abstractmethod import numpy as np import otbApplication as otb # pylint: disable=import-error @@ -16,6 +16,7 @@ from .helpers import logger class OTBObject(ABC): """Abstraction of an image object.""" + @property @abstractmethod def name(self) -> str: @@ -85,7 +86,9 @@ class OTBObject(ABC): """Return a dict output of ComputeImagesStatistics for the first image output.""" return App("ComputeImagesStatistics", self, quiet=True).data - def get_values_at_coords(self, row: int, col: int, bands: int = None) -> list[int | float] | int | float: + def get_values_at_coords( + self, row: int, col: int, bands: int = None + ) -> list[int | float] | int | float: """Get pixel value(s) at a given YX coordinates. Args: @@ -107,7 +110,9 @@ class OTBObject(ABC): elif isinstance(bands, slice): channels = self.channels_list_from_slice(bands) elif not isinstance(bands, list): - raise TypeError(f"{self.name}: type '{type(bands)}' cannot be interpreted as a valid slicing") + raise TypeError( + f"{self.name}: type '{type(bands)}' cannot be interpreted as a valid slicing" + ) if channels: app.app.Execute() app.set_parameters({"cl": [f"Channel{n + 1}" for n in channels]}) @@ -130,9 +135,13 @@ class OTBObject(ABC): return list(range(0, stop, step)) if start is None and stop is None: return list(range(0, nb_channels, step)) - raise ValueError(f"{self.name}: '{bands}' cannot be interpreted as valid slicing.") + raise ValueError( + f"{self.name}: '{bands}' cannot be interpreted as valid slicing." + ) - def export(self, key: str = None, preserve_dtype: bool = True) -> dict[str, dict[str, np.ndarray]]: + def export( + self, key: str = None, preserve_dtype: bool = True + ) -> dict[str, dict[str, np.ndarray]]: """Export a specific output image as numpy array and store it in object exports_dic. Args: @@ -149,10 +158,14 @@ class OTBObject(ABC): if key not in self.exports_dic: self.exports_dic[key] = self.app.ExportImage(key) if preserve_dtype: - self.exports_dic[key]["array"] = self.exports_dic[key]["array"].astype(self.dtype) + self.exports_dic[key]["array"] = self.exports_dic[key]["array"].astype( + self.dtype + ) return self.exports_dic[key] - def to_numpy(self, key: str = None, preserve_dtype: bool = True, copy: bool = False) -> np.ndarray: + def to_numpy( + self, key: str = None, preserve_dtype: bool = True, copy: bool = False + ) -> np.ndarray: """Export a pyotb object to numpy array. Args: @@ -370,12 +383,18 @@ class OTBObject(ABC): row, col = key[0], key[1] if isinstance(row, int) and isinstance(col, int): if row < 0 or col < 0: - raise ValueError(f"{self.name} cannot read pixel value at negative coordinates ({row}, {col})") + raise ValueError( + f"{self.name} cannot read pixel value at negative coordinates ({row}, {col})" + ) channels = key[2] if len(key) == 3 else None return self.get_values_at_coords(row, col, channels) # Slicing - if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)): - raise ValueError(f'"{key}" cannot be interpreted as valid slicing. Slicing should be 2D or 3D.') + if not isinstance(key, tuple) or ( + isinstance(key, tuple) and (len(key) < 2 or len(key) > 3) + ): + raise ValueError( + f'"{key}" cannot be interpreted as valid slicing. Slicing should be 2D or 3D.' + ) if isinstance(key, tuple) and len(key) == 2: # Adding a 3rd dimension key = key + (slice(None, None, None),) @@ -392,7 +411,7 @@ class App(OTBObject): INPUT_IMAGE_TYPES = [ # Images only otb.ParameterType_InputImage, - otb.ParameterType_InputImageList + otb.ParameterType_InputImageList, ] INPUT_PARAM_TYPES = INPUT_IMAGE_TYPES + [ # Vectors @@ -426,7 +445,15 @@ class App(OTBObject): otb.ParameterType_InputFilenameList, ] - def __init__(self, appname: str, *args, frozen: bool = False, quiet: bool = False, name: str = "", **kwargs): + def __init__( + self, + appname: str, + *args, + frozen: bool = False, + quiet: bool = False, + name: str = "", + **kwargs, + ): """Common constructor for OTB applications. Handles in-memory connection between apps. Args: @@ -445,7 +472,11 @@ class App(OTBObject): """ # Attributes and data structures used by properties - create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication + create = ( + otb.Registry.CreateApplicationWithoutLogger + if quiet + else otb.Registry.CreateApplication + ) self._app = create(appname) self._name = name or appname self._exports_dic = {} @@ -455,14 +486,25 @@ class App(OTBObject): self.quiet, self.frozen = quiet, frozen # Param keys and types self.parameters_keys = tuple(self.app.GetParametersKeys()) - self._all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys} - types = (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename) - self._out_param_types = {k: v for k, v in self._all_param_types.items() if v in types} + self._all_param_types = { + k: self.app.GetParameterType(k) for k in self.parameters_keys + } + types = ( + otb.ParameterType_OutputImage, + otb.ParameterType_OutputVectorData, + otb.ParameterType_OutputFilename, + ) + self._out_param_types = { + k: v for k, v in self._all_param_types.items() if v in types + } # Init, execute and write (auto flush only when output param was provided) if args or kwargs: self.set_parameters(*args, **kwargs) # Create Output image objects - for key in filter(lambda k: self._out_param_types[k] == otb.ParameterType_OutputImage, self._out_param_types): + for key in filter( + lambda k: self._out_param_types[k] == otb.ParameterType_OutputImage, + self._out_param_types, + ): self.outputs[key] = Output(self, key, self._settings.get(key)) if not self.frozen: self.execute() @@ -492,9 +534,7 @@ class App(OTBObject): def __is_one_of_types(self, key: str, param_types: list[int]) -> bool: """Helper to factor is_input and is_output.""" if key not in self._all_param_types: - raise KeyError( - f"key {key} not found in the application parameters types" - ) + raise KeyError(f"key {key} not found in the application parameters types") return self._all_param_types[key] in param_types def is_input(self, key: str) -> bool: @@ -507,9 +547,7 @@ class App(OTBObject): True if the parameter is an input, else False """ - return self.__is_one_of_types( - key=key, param_types=self.INPUT_PARAM_TYPES - ) + return self.__is_one_of_types(key=key, param_types=self.INPUT_PARAM_TYPES) def is_output(self, key: str) -> bool: """Returns True if the key is an output. @@ -521,9 +559,7 @@ class App(OTBObject): True if the parameter is an output, else False """ - return self.__is_one_of_types( - key=key, param_types=self.OUTPUT_PARAM_TYPES - ) + return self.__is_one_of_types(key=key, param_types=self.OUTPUT_PARAM_TYPES) def is_key_list(self, key: str) -> bool: """Check if a parameter key is an input parameter list.""" @@ -541,7 +577,9 @@ class App(OTBObject): for key, value in sorted(self._all_param_types.items()): if value == param_type: return key - raise TypeError(f"{self.name}: could not find any parameter key matching the provided types") + raise TypeError( + f"{self.name}: could not find any parameter key matching the provided types" + ) @property def input_key(self) -> str: @@ -598,7 +636,11 @@ class App(OTBObject): # When the parameter expects a list, if needed, change the value to list if self.is_key_list(key) and not isinstance(obj, (list, tuple)): obj = [obj] - logger.info('%s: argument for parameter "%s" was converted to list', self.name, key) + logger.info( + '%s: argument for parameter "%s" was converted to list', + self.name, + key, + ) try: if self.is_input(key): if self.is_key_images_list(key): @@ -630,19 +672,28 @@ class App(OTBObject): if not dtype: param = self._settings.get(self.input_image_key) if not param: - logger.warning("%s: could not propagate pixel type from inputs to output", self.name) + logger.warning( + "%s: could not propagate pixel type from inputs to output", + self.name, + ) return if isinstance(param, (list, tuple)): param = param[0] # first image in "il" try: dtype = get_pixel_type(param) except (TypeError, RuntimeError): - logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param) + logger.warning( + '%s: unable to identify pixel type of key "%s"', self.name, param + ) return if target_key: keys = [target_key] else: - keys = [k for k, v in self._out_param_types.items() if v == otb.ParameterType_OutputImage] + keys = [ + k + for k, v in self._out_param_types.items() + if v == otb.ParameterType_OutputImage + ] for key in keys: self.app.SetParameterOutputImagePixelType(key, dtype) @@ -653,7 +704,9 @@ class App(OTBObject): try: self.app.Execute() except (RuntimeError, FileNotFoundError) as e: - raise RuntimeError(f"{self.name}: error during during app execution ({e}") from e + raise RuntimeError( + f"{self.name}: error during during app execution ({e}" + ) from e self.frozen = False self._time_end = perf_counter() logger.debug("%s: execution ended", self.name) @@ -665,13 +718,22 @@ class App(OTBObject): logger.debug("%s: flushing data to disk", self.name) self.app.WriteOutput() except RuntimeError: - logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name) + logger.debug( + "%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", + self.name, + ) self._time_start = perf_counter() self.app.ExecuteAndWriteOutput() self._time_end = perf_counter() - def write(self, path: str | Path | dict[str, str] = None, ext_fname: str = "", - pixel_type: dict[str, str] | str = None, preserve_dtype: bool = False, **kwargs) -> bool: + def write( + self, + path: str | Path | dict[str, str] = None, + ext_fname: str = "", + pixel_type: dict[str, str] | str = None, + preserve_dtype: bool = False, + **kwargs, + ) -> bool: """Set output pixel type and write the output raster files. Args: @@ -697,24 +759,35 @@ class App(OTBObject): if isinstance(path, dict): kwargs.update(path) elif isinstance(path, str) and kwargs: - logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, path) + logger.warning( + '%s: keyword arguments specified, ignoring argument "%s"', + self.name, + path, + ) elif isinstance(path, (str, Path)) and self.output_key: kwargs.update({self.output_key: str(path)}) elif path is not None: raise TypeError(f"{self.name}: unsupported filepath type ({type(path)})") if not (kwargs or any(k in self._settings for k in self._out_param_types)): - raise KeyError(f"{self.name}: at least one filepath is required, if not provided during App init") + raise KeyError( + f"{self.name}: at least one filepath is required, if not provided during App init" + ) parameters = kwargs.copy() # Append filename extension to filenames if ext_fname: - logger.debug("%s: using extended filename for outputs: %s", self.name, ext_fname) + logger.debug( + "%s: using extended filename for outputs: %s", self.name, ext_fname + ) if not ext_fname.startswith("?"): ext_fname = "?&" + ext_fname elif not ext_fname.startswith("?&"): ext_fname = "?&" + ext_fname[1:] for key, value in kwargs.items(): - if self._out_param_types[key] == otb.ParameterType_OutputImage and "?" not in value: + if ( + self._out_param_types[key] == otb.ParameterType_OutputImage + and "?" not in value + ): parameters[key] = value + ext_fname # Manage output pixel types data_types = {} @@ -722,12 +795,16 @@ class App(OTBObject): if isinstance(pixel_type, str): dtype = parse_pixel_type(pixel_type) type_name = self.app.ConvertPixelTypeToNumpy(dtype) - logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name) + logger.debug( + '%s: output(s) will be written with type "%s"', self.name, type_name + ) for key in parameters: if self._out_param_types[key] == otb.ParameterType_OutputImage: data_types[key] = dtype elif isinstance(pixel_type, dict): - data_types = {key: parse_pixel_type(dtype) for key, dtype in pixel_type.items()} + data_types = { + key: parse_pixel_type(dtype) for key, dtype in pixel_type.items() + } elif preserve_dtype: self.propagate_dtype() # all outputs will have the same type as the main input raster @@ -751,7 +828,11 @@ class App(OTBObject): dest = files if filepath.exists() else missing dest.append(str(filepath.absolute())) for filename in missing: - logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename) + logger.error( + "%s: execution seems to have failed, %s does not exist", + self.name, + filename, + ) return bool(files) and not missing # Private functions @@ -769,11 +850,17 @@ class App(OTBObject): for arg in args: if isinstance(arg, dict): kwargs.update(arg) - elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and self.is_key_list(self.input_key): + elif ( + isinstance(arg, (str, OTBObject)) + or isinstance(arg, list) + and self.is_key_list(self.input_key) + ): kwargs.update({self.input_key: arg}) return kwargs - def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]): + def __set_param( + self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any] + ): """Set one parameter, decide which otb.Application method to use depending on target object.""" if obj is None or (isinstance(obj, (list, tuple)) and not obj): self.app.ClearValue(key) @@ -781,9 +868,13 @@ class App(OTBObject): # Single-parameter cases if isinstance(obj, OTBObject): self.app.ConnectImage(key, obj.app, obj.output_image_key) - elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB + elif isinstance( + obj, otb.Application + ): # this is for backward comp with plain OTB self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0]) - elif key == "ram": # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200 + elif ( + key == "ram" + ): # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200 self.app.SetParameterInt("ram", int(obj)) elif not isinstance(obj, list): # any other parameters (str, int...) self.app.SetParameterValue(key, obj) @@ -793,7 +884,9 @@ class App(OTBObject): for inp in obj: if isinstance(inp, OTBObject): self.app.ConnectImage(key, inp.app, inp.output_image_key) - elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB + elif isinstance( + inp, otb.Application + ): # this is for backward comp with plain OTB self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0]) else: # here `input` should be an image filepath # Append `input` to the list, do not overwrite any previously set element of the image list @@ -809,9 +902,13 @@ class App(OTBObject): continue value = self.app.GetParameterValue(key) # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken - if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(key): + if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue( + key + ): try: - value = str(value) # some default str values like "mode" or "interpolator" + value = str( + value + ) # some default str values like "mode" or "interpolator" self._auto_parameters[key] = value continue except RuntimeError: @@ -841,13 +938,21 @@ class App(OTBObject): if key in self.parameters: return self.parameters[key] raise KeyError(f"{self.name}: unknown or undefined parameter '{key}'") - raise TypeError(f"{self.name}: cannot access object item or slice using {type(key)} object") + raise TypeError( + f"{self.name}: cannot access object item or slice using {type(key)} object" + ) class Slicer(App): """Slicer objects i.e. when we call something like raster[:, :, 2] from Python.""" - def __init__(self, obj: OTBObject, rows: slice, cols: slice, channels: slice | list[int] | int): + def __init__( + self, + obj: OTBObject, + rows: slice, + cols: slice, + channels: slice | list[int] | int, + ): """Create a slicer object, that can be used directly for writing or inside a BandMath. It contains : @@ -861,7 +966,14 @@ class Slicer(App): channels: channels, can be slicing, list or int """ - super().__init__("ExtractROI", obj, mode="extent", quiet=True, frozen=True, name=f"Slicer from {obj.name}") + super().__init__( + "ExtractROI", + obj, + mode="extent", + quiet=True, + frozen=True, + name=f"Slicer from {obj.name}", + ) self.rows, self.cols = rows, cols parameters = {} @@ -879,7 +991,9 @@ class Slicer(App): elif isinstance(channels, tuple): channels = list(channels) elif not isinstance(channels, list): - raise ValueError(f"Invalid type for channels, should be int, slice or list of bands. : {channels}") + raise ValueError( + f"Invalid type for channels, should be int, slice or list of bands. : {channels}" + ) # Change the potential negative index values to reverse index channels = [c if c >= 0 else nb_channels + c for c in channels] parameters.update({"cl": [f"Channel{i + 1}" for i in channels]}) @@ -891,17 +1005,23 @@ class Slicer(App): parameters.update({"mode.extent.uly": rows.start}) spatial_slicing = True if rows.stop is not None and rows.stop != -1: - parameters.update({"mode.extent.lry": rows.stop - 1}) # subtract 1 to respect python convention + parameters.update( + {"mode.extent.lry": rows.stop - 1} + ) # subtract 1 to respect python convention spatial_slicing = True if cols.start is not None: parameters.update({"mode.extent.ulx": cols.start}) spatial_slicing = True if cols.stop is not None and cols.stop != -1: - parameters.update({"mode.extent.lrx": cols.stop - 1}) # subtract 1 to respect python convention + parameters.update( + {"mode.extent.lrx": cols.stop - 1} + ) # subtract 1 to respect python convention spatial_slicing = True # These are some attributes when the user simply wants to extract *one* band to be used in an Operation if not spatial_slicing and isinstance(channels, list) and len(channels) == 1: - self.one_band_sliced = channels[0] + 1 # OTB convention: channels start at 1 + self.one_band_sliced = ( + channels[0] + 1 + ) # OTB convention: channels start at 1 self.input = obj # Execute app @@ -956,7 +1076,9 @@ class Operation(App): # NB: the keys of the dictionary are strings-only, instead of 'complex' objects, to enable easy serialization self.im_dic = {} self.im_count = 1 - map_repr_to_input = {} # to be able to retrieve the real python object from its string representation + map_repr_to_input = ( + {} + ) # to be able to retrieve the real python object from its string representation for inp in self.inputs: if not isinstance(inp, (int, float)): if str(inp) not in self.im_dic: @@ -964,13 +1086,23 @@ class Operation(App): map_repr_to_input[repr(inp)] = inp self.im_count += 1 # Getting unique image inputs, in the order im1, im2, im3 ... - self.unique_inputs = [map_repr_to_input[id_str] for id_str in sorted(self.im_dic, key=self.im_dic.get)] + self.unique_inputs = [ + map_repr_to_input[id_str] + for id_str in sorted(self.im_dic, key=self.im_dic.get) + ] self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands) appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX" name = f'Operation exp="{self.exp}"' - super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True, name=name) + super().__init__( + appname, il=self.unique_inputs, exp=self.exp, quiet=True, name=name + ) - def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None): + def build_fake_expressions( + self, + operator: str, + inputs: list[OTBObject | str | int | float], + nb_bands: int = None, + ): """Create a list of 'fake' expressions, one for each band. E.g for the operation input1 + input2, we create a fake expression that is like "str(input1) + str(input2)" @@ -989,12 +1121,21 @@ class Operation(App): pass # For any other operations, the output number of bands is the same as inputs else: - if any(isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced") for inp in inputs): + if any( + isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced") + for inp in inputs + ): nb_bands = 1 else: - nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))] + nb_bands_list = [ + get_nbchannels(inp) + for inp in inputs + if not isinstance(inp, (float, int)) + ] # check that all inputs have the same nb of bands - if len(nb_bands_list) > 1 and not all(x == nb_bands_list[0] for x in nb_bands_list): + if len(nb_bands_list) > 1 and not all( + x == nb_bands_list[0] for x in nb_bands_list + ): raise ValueError("All images do not have the same number of bands") nb_bands = nb_bands_list[0] @@ -1008,10 +1149,14 @@ class Operation(App): if len(inputs) == 3 and k == 0: # When cond is monoband whereas the result is multiband, we expand the cond to multiband cond_band = 1 if nb_bands != inp.shape[2] else band - fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp(inp, cond_band, keep_logical=True) + fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp( + inp, cond_band, keep_logical=True + ) else: # Any other input - fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp(inp, band, keep_logical=False) + fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp( + inp, band, keep_logical=False + ) expressions.append(fake_exp) # Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates) if i == 0 and corresponding_inputs and nb_channels: @@ -1025,7 +1170,9 @@ class Operation(App): # We create here the "fake" expression. For example, for a BandMathX expression such as '2 * im1 + im2', # the false expression stores the expression 2 * str(input1) + str(input2) fake_exp = f"({expressions[0]} {operator} {expressions[1]})" - elif len(inputs) == 3 and operator == "?": # this is only for ternary expression + elif ( + len(inputs) == 3 and operator == "?" + ): # this is only for ternary expression fake_exp = f"({expressions[0]} ? {expressions[1]} : {expressions[2]})" self.fake_exp_bands.append(fake_exp) @@ -1052,7 +1199,9 @@ class Operation(App): return exp_bands, ";".join(exp_bands) @staticmethod - def make_fake_exp(x: OTBObject | str, band: int, keep_logical: bool = False) -> tuple[str, list[OTBObject], int]: + def make_fake_exp( + x: OTBObject | str, band: int, keep_logical: bool = False + ) -> tuple[str, list[OTBObject], int]: """This an internal function, only to be used by `build_fake_expressions`. Enable to create a fake expression just for one input and one band. @@ -1127,9 +1276,16 @@ class LogicalOperation(Operation): """ self.logical_fake_exp_bands = [] super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation") - self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands) + self.logical_exp_bands, self.logical_exp = self.get_real_exp( + self.logical_fake_exp_bands + ) - def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None): + def build_fake_expressions( + self, + operator: str, + inputs: list[OTBObject | str | int | float], + nb_bands: int = None, + ): """Create a list of 'fake' expressions, one for each band. e.g for the operation input1 > input2, we create a fake expression that is like @@ -1142,19 +1298,30 @@ class LogicalOperation(Operation): """ # For any other operations, the output number of bands is the same as inputs - if any(isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced") for inp in inputs): + if any( + isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced") + for inp in inputs + ): nb_bands = 1 else: - nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))] + nb_bands_list = [ + get_nbchannels(inp) + for inp in inputs + if not isinstance(inp, (float, int)) + ] # check that all inputs have the same nb of bands - if len(nb_bands_list) > 1 and not all(x == nb_bands_list[0] for x in nb_bands_list): + if len(nb_bands_list) > 1 and not all( + x == nb_bands_list[0] for x in nb_bands_list + ): raise ValueError("All images do not have the same number of bands") nb_bands = nb_bands_list[0] # Create a list of fake exp, each item of the list corresponding to one band for i, band in enumerate(range(1, nb_bands + 1)): expressions = [] for inp in inputs: - fake_exp, corresp_inputs, nb_channels = super().make_fake_exp(inp, band, keep_logical=True) + fake_exp, corresp_inputs, nb_channels = super().make_fake_exp( + inp, band, keep_logical=True + ) expressions.append(fake_exp) # Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates) if i == 0 and corresp_inputs and nb_channels: @@ -1195,9 +1362,16 @@ class Input(App): class Output(OTBObject): """Object that behave like a pointer to a specific application output file.""" + _filepath: str | Path = None - def __init__(self, pyotb_app: App, param_key: str = None, filepath: str = None, mkdir: bool = True): + def __init__( + self, + pyotb_app: App, + param_key: str = None, + filepath: str = None, + mkdir: bool = True, + ): """Constructor for an Output object. Args: @@ -1262,7 +1436,9 @@ class Output(OTBObject): def write(self, filepath: None | str | Path = None, **kwargs) -> bool: """Write output to disk, filepath is not required if it was provided to parent App during init.""" if filepath is None: - return self.parent_pyotb_app.write({self.output_image_key: self.filepath}, **kwargs) + return self.parent_pyotb_app.write( + {self.output_image_key: self.filepath}, **kwargs + ) return self.parent_pyotb_app.write({self.output_image_key: filepath}, **kwargs) def __str__(self) -> str: @@ -1293,7 +1469,7 @@ def add_vsi_prefix(filepath: str | Path) -> str: ".gz": "vsigzip", ".7z": "vsi7z", ".zip": "vsizip", - ".rar": "vsirar" + ".rar": "vsirar", } basename = filepath.split("?")[0] ext = Path(basename).suffix @@ -1319,8 +1495,12 @@ def get_nbchannels(inp: str | Path | OTBObject) -> int: try: info = App("ReadImageInfo", inp, quiet=True) return info["numberbands"] - except RuntimeError as info_err: # this happens when we pass a str that is not a filepath - raise TypeError(f"Could not get the number of channels file '{inp}' ({info_err})") from info_err + except ( + RuntimeError + ) as info_err: # this happens when we pass a str that is not a filepath + raise TypeError( + f"Could not get the number of channels file '{inp}' ({info_err})" + ) from info_err raise TypeError(f"Can't read number of channels of type '{type(inp)}' object {inp}") @@ -1341,8 +1521,12 @@ def get_pixel_type(inp: str | Path | OTBObject) -> str: try: info = App("ReadImageInfo", inp, quiet=True) datatype = info["datatype"] # which is such as short, float... - except RuntimeError as info_err: # this happens when we pass a str that is not a filepath - raise TypeError(f"Could not get the pixel type of `{inp}` ({info_err})") from info_err + except ( + RuntimeError + ) as info_err: # this happens when we pass a str that is not a filepath + raise TypeError( + f"Could not get the pixel type of `{inp}` ({info_err})" + ) from info_err if datatype: return parse_pixel_type(datatype) raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}") @@ -1376,13 +1560,21 @@ def parse_pixel_type(pixel_type: str | int) -> int: return getattr(otb, f"ImagePixelType_{pixel_type}") if pixel_type in datatype_to_pixeltype: return getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[pixel_type]}") - raise KeyError(f"Unknown data type `{pixel_type}`. Available ones: {datatype_to_pixeltype}") - raise TypeError(f"Bad pixel type specification ({pixel_type} of type {type(pixel_type)})") + raise KeyError( + f"Unknown data type `{pixel_type}`. Available ones: {datatype_to_pixeltype}" + ) + raise TypeError( + f"Bad pixel type specification ({pixel_type} of type {type(pixel_type)})" + ) def get_out_images_param_keys(app: OTBObject) -> list[str]: """Return every output parameter keys of an OTB app.""" - return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage] + return [ + key + for key in app.GetParametersKeys() + if app.GetParameterType(key) == otb.ParameterType_OutputImage + ] def summarize( @@ -1406,6 +1598,7 @@ def summarize( parameters of an app and its parents """ + def strip_path(param: str | Any): if not isinstance(param, str): return summarize(param) @@ -1420,8 +1613,17 @@ def summarize( parameters = {} for key, param in obj.parameters.items(): - if strip_input_paths and obj.is_input(key) or strip_output_paths and obj.is_output(key): - parameters[key] = [strip_path(p) for p in param] if isinstance(param, list) else strip_path(param) + if ( + strip_input_paths + and obj.is_input(key) + or strip_output_paths + and obj.is_output(key) + ): + parameters[key] = ( + [strip_path(p) for p in param] + if isinstance(param, list) + else strip_path(param) + ) else: parameters[key] = summarize(param) return {"name": obj.app.GetName(), "parameters": parameters} diff --git a/pyotb/functions.py b/pyotb/functions.py index 6cc7889..96431cd 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -1,19 +1,22 @@ # -*- coding: utf-8 -*- """This module provides a set of functions for pyotb.""" from __future__ import annotations + import inspect import os +import subprocess import sys -import uuid import textwrap -import subprocess +import uuid from collections import Counter -from .core import App, Operation, LogicalOperation, Input, get_nbchannels +from .core import App, Input, LogicalOperation, Operation, get_nbchannels from .helpers import logger -def where(cond: App | str, x: App | str | int | float, y: App | str | int | float) -> Operation: +def where( + cond: App | str, x: App | str | int | float, y: App | str | int | float +) -> Operation: """Functionally similar to numpy.where. Where cond is True (!=0), returns x. Else returns y. Args: @@ -36,18 +39,29 @@ def where(cond: App | str, x: App | str | int | float, y: App | str | int | floa y_nb_channels = get_nbchannels(y) if x_nb_channels and y_nb_channels: if x_nb_channels != y_nb_channels: - raise ValueError('X and Y images do not have the same number of bands. ' - f'X has {x_nb_channels} bands whereas Y has {y_nb_channels} bands') + raise ValueError( + "X and Y images do not have the same number of bands. " + f"X has {x_nb_channels} bands whereas Y has {y_nb_channels} bands" + ) x_or_y_nb_channels = x_nb_channels if x_nb_channels else y_nb_channels cond_nb_channels = get_nbchannels(cond) - if cond_nb_channels != 1 and x_or_y_nb_channels and cond_nb_channels != x_or_y_nb_channels: - raise ValueError('Condition and X&Y do not have the same number of bands. Condition has ' - f'{cond_nb_channels} bands whereas X&Y have {x_or_y_nb_channels} bands') + if ( + cond_nb_channels != 1 + and x_or_y_nb_channels + and cond_nb_channels != x_or_y_nb_channels + ): + raise ValueError( + "Condition and X&Y do not have the same number of bands. Condition has " + f"{cond_nb_channels} bands whereas X&Y have {x_or_y_nb_channels} bands" + ) # If needed, duplicate the single band binary mask to multiband to match the dimensions of x & y if cond_nb_channels == 1 and x_or_y_nb_channels and x_or_y_nb_channels != 1: - logger.info('The condition has one channel whereas X/Y has/have %s channels. Expanding number' - ' of channels of condition to match the number of channels of X/Y', x_or_y_nb_channels) + logger.info( + "The condition has one channel whereas X/Y has/have %s channels. Expanding number" + " of channels of condition to match the number of channels of X/Y", + x_or_y_nb_channels, + ) # Get the number of bands of the result if x_or_y_nb_channels: # if X or Y is a raster @@ -55,10 +69,12 @@ def where(cond: App | str, x: App | str | int | float, y: App | str | int | floa else: # if only cond is a raster out_nb_channels = cond_nb_channels - return Operation('?', cond, x, y, nb_bands=out_nb_channels) + return Operation("?", cond, x, y, nb_bands=out_nb_channels) -def clip(image: App | str, v_min: App | str | int | float, v_max: App | str | int | float): +def clip( + image: App | str, v_min: App | str | int | float, v_max: App | str | int | float +): """Clip values of image in a range of values. Args: @@ -96,7 +112,11 @@ def all(*inputs): # pylint: disable=redefined-builtin if len(inputs) == 1 and isinstance(inputs[0], (list, tuple)): inputs = inputs[0] # Add support for generator inputs (to have the same behavior as built-in `all` function) - if isinstance(inputs, tuple) and len(inputs) == 1 and inspect.isgenerator(inputs[0]): + if ( + isinstance(inputs, tuple) + and len(inputs) == 1 + and inspect.isgenerator(inputs[0]) + ): inputs = list(inputs[0]) # Transforming potential filepaths to pyotb objects inputs = [Input(inp) if isinstance(inp, str) else inp for inp in inputs] @@ -107,7 +127,7 @@ def all(*inputs): # pylint: disable=redefined-builtin if isinstance(inp, LogicalOperation): res = inp[:, :, 0] else: - res = (inp[:, :, 0] != 0) + res = inp[:, :, 0] != 0 for band in range(1, inp.shape[-1]): if isinstance(inp, LogicalOperation): res = res & inp[:, :, band] @@ -147,7 +167,11 @@ def any(*inputs): # pylint: disable=redefined-builtin if len(inputs) == 1 and isinstance(inputs[0], (list, tuple)): inputs = inputs[0] # Add support for generator inputs (to have the same behavior as built-in `any` function) - if isinstance(inputs, tuple) and len(inputs) == 1 and inspect.isgenerator(inputs[0]): + if ( + isinstance(inputs, tuple) + and len(inputs) == 1 + and inspect.isgenerator(inputs[0]) + ): inputs = list(inputs[0]) # Transforming potential filepaths to pyotb objects inputs = [Input(inp) if isinstance(inp, str) else inp for inp in inputs] @@ -158,7 +182,7 @@ def any(*inputs): # pylint: disable=redefined-builtin if isinstance(inp, LogicalOperation): res = inp[:, :, 0] else: - res = (inp[:, :, 0] != 0) + res = inp[:, :, 0] != 0 for band in range(1, inp.shape[-1]): if isinstance(inp, LogicalOperation): @@ -203,10 +227,14 @@ def run_tf_function(func): """ try: - from .apps import TensorflowModelServe # pylint: disable=import-outside-toplevel + from .apps import ( # pylint: disable=import-outside-toplevel + TensorflowModelServe, + ) except ImportError: - logger.error('Could not run Tensorflow function: failed to import TensorflowModelServe.' - 'Check that you have OTBTF configured (https://github.com/remicres/otbtf#how-to-install)') + logger.error( + "Could not run Tensorflow function: failed to import TensorflowModelServe." + "Check that you have OTBTF configured (https://github.com/remicres/otbtf#how-to-install)" + ) raise def get_tf_pycmd(output_dir, channels, scalar_inputs): @@ -228,7 +256,8 @@ def run_tf_function(func): create_and_save_model_str = func_def_str # Adding the instructions to create the model and save it to output dir - create_and_save_model_str += textwrap.dedent(f""" + create_and_save_model_str += textwrap.dedent( + f""" import tensorflow as tf model_inputs = [] @@ -248,11 +277,12 @@ def run_tf_function(func): # Create and save the .pb model model = tf.keras.Model(inputs=model_inputs, outputs=output) model.save("{output_dir}") - """) + """ + ) return create_and_save_model_str - def wrapper(*inputs, tmp_dir='/tmp'): + def wrapper(*inputs, tmp_dir="/tmp"): """For the user point of view, this function simply applies some TensorFlow operations to some rasters. Implicitly, it saves a .pb model that describe the TF operations, then creates an OTB ModelServe application @@ -284,22 +314,35 @@ def run_tf_function(func): # Create and save the model. This is executed **inside an independent process** because (as of 2022-03), # tensorflow python library and OTBTF are incompatible - out_savedmodel = os.path.join(tmp_dir, f'tmp_otbtf_model_{uuid.uuid4()}') + out_savedmodel = os.path.join(tmp_dir, f"tmp_otbtf_model_{uuid.uuid4()}") pycmd = get_tf_pycmd(out_savedmodel, channels, scalar_inputs) cmd_args = [sys.executable, "-c", pycmd] try: - subprocess.run(cmd_args, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + subprocess.run( + cmd_args, + env=os.environ, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) except subprocess.SubprocessError: logger.debug("Failed to call subprocess") if not os.path.isdir(out_savedmodel): logger.info("Failed to save the model") # Initialize the OTBTF model serving application - model_serve = TensorflowModelServe({'model.dir': out_savedmodel, 'optim.disabletiling': 'on', - 'model.fullyconv': 'on'}, n_sources=len(raster_inputs), frozen=True) + model_serve = TensorflowModelServe( + { + "model.dir": out_savedmodel, + "optim.disabletiling": "on", + "model.fullyconv": "on", + }, + n_sources=len(raster_inputs), + frozen=True, + ) # Set parameters and execute for i, inp in enumerate(raster_inputs): - model_serve.set_parameters({f'source{i + 1}.il': [inp]}) + model_serve.set_parameters({f"source{i + 1}.il": [inp]}) model_serve.execute() # TODO: handle the deletion of the temporary model ? @@ -308,9 +351,14 @@ def run_tf_function(func): return wrapper -def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_rule: str = 'minimal', - interpolator: str = 'nn', reference_window_input: dict = None, - reference_pixel_size_input: str = None) -> list[App]: +def define_processing_area( + *args, + window_rule: str = "intersection", + pixel_size_rule: str = "minimal", + interpolator: str = "nn", + reference_window_input: dict = None, + reference_pixel_size_input: str = None, +) -> list[App]: """Given several inputs, this function handles the potential resampling and cropping to same extent. WARNING: Not fully implemented / tested @@ -338,7 +386,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ metadatas = {} for inp in inputs: if isinstance(inp, str): # this is for filepaths - metadata = Input(inp).app.GetImageMetaData('out') + metadata = Input(inp).app.GetImageMetaData("out") elif isinstance(inp, App): metadata = inp.app.GetImageMetaData(inp.output_param) else: @@ -348,100 +396,147 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ # Get a metadata of an arbitrary image. This is just to compare later with other images any_metadata = next(iter(metadatas.values())) # Checking if all images have the same projection - if not all(metadata['ProjectionRef'] == any_metadata['ProjectionRef'] - for metadata in metadatas.values()): - logger.warning('All images may not have the same CRS, which might cause unpredictable results') + if not all( + metadata["ProjectionRef"] == any_metadata["ProjectionRef"] + for metadata in metadatas.values() + ): + logger.warning( + "All images may not have the same CRS, which might cause unpredictable results" + ) # Handling different spatial footprints # TODO: there seems to have a bug, ImageMetaData is not updated when running an app, # cf https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2234. Should we use ImageOrigin instead? - if not all(metadata['UpperLeftCorner'] == any_metadata['UpperLeftCorner'] - and metadata['LowerRightCorner'] == any_metadata['LowerRightCorner'] - for metadata in metadatas.values()): + if not all( + metadata["UpperLeftCorner"] == any_metadata["UpperLeftCorner"] + and metadata["LowerRightCorner"] == any_metadata["LowerRightCorner"] + for metadata in metadatas.values() + ): # Retrieving the bounding box that will be common for all inputs - if window_rule == 'intersection': + if window_rule == "intersection": # The coordinates depend on the orientation of the axis of projection - if any_metadata['GeoTransform'][1] >= 0: - ulx = max(metadata['UpperLeftCorner'][0] for metadata in metadatas.values()) - lrx = min(metadata['LowerRightCorner'][0] for metadata in metadatas.values()) + if any_metadata["GeoTransform"][1] >= 0: + ulx = max( + metadata["UpperLeftCorner"][0] for metadata in metadatas.values() + ) + lrx = min( + metadata["LowerRightCorner"][0] for metadata in metadatas.values() + ) else: - ulx = min(metadata['UpperLeftCorner'][0] for metadata in metadatas.values()) - lrx = max(metadata['LowerRightCorner'][0] for metadata in metadatas.values()) - if any_metadata['GeoTransform'][-1] >= 0: - lry = min(metadata['LowerRightCorner'][1] for metadata in metadatas.values()) - uly = max(metadata['UpperLeftCorner'][1] for metadata in metadatas.values()) + ulx = min( + metadata["UpperLeftCorner"][0] for metadata in metadatas.values() + ) + lrx = max( + metadata["LowerRightCorner"][0] for metadata in metadatas.values() + ) + if any_metadata["GeoTransform"][-1] >= 0: + lry = min( + metadata["LowerRightCorner"][1] for metadata in metadatas.values() + ) + uly = max( + metadata["UpperLeftCorner"][1] for metadata in metadatas.values() + ) else: - lry = max(metadata['LowerRightCorner'][1] for metadata in metadatas.values()) - uly = min(metadata['UpperLeftCorner'][1] for metadata in metadatas.values()) - - elif window_rule == 'same_as_input': - ulx = metadatas[reference_window_input]['UpperLeftCorner'][0] - lrx = metadatas[reference_window_input]['LowerRightCorner'][0] - lry = metadatas[reference_window_input]['LowerRightCorner'][1] - uly = metadatas[reference_window_input]['UpperLeftCorner'][1] - elif window_rule == 'specify': + lry = max( + metadata["LowerRightCorner"][1] for metadata in metadatas.values() + ) + uly = min( + metadata["UpperLeftCorner"][1] for metadata in metadatas.values() + ) + + elif window_rule == "same_as_input": + ulx = metadatas[reference_window_input]["UpperLeftCorner"][0] + lrx = metadatas[reference_window_input]["LowerRightCorner"][0] + lry = metadatas[reference_window_input]["LowerRightCorner"][1] + uly = metadatas[reference_window_input]["UpperLeftCorner"][1] + elif window_rule == "specify": pass # TODO : it is when the user explicitly specifies the bounding box -> add some arguments in the function - elif window_rule == 'union': + elif window_rule == "union": pass # TODO : it is when the user wants the final bounding box to be the union of all bounding box # It should replace any 'outside' pixel by some NoData -> add `fillvalue` argument in the function # Applying this bounding box to all inputs - logger.info('Cropping all images to extent Upper Left (%s, %s), Lower Right (%s, %s)', ulx, uly, lrx, lry) + logger.info( + "Cropping all images to extent Upper Left (%s, %s), Lower Right (%s, %s)", + ulx, + uly, + lrx, + lry, + ) new_inputs = [] for inp in inputs: try: params = { - 'in': inp, 'mode': 'extent', 'mode.extent.unit': 'phy', - 'mode.extent.ulx': ulx, 'mode.extent.uly': lry, # bug in OTB <= 7.3 : - 'mode.extent.lrx': lrx, 'mode.extent.lry': uly, # ULY/LRY are inverted + "in": inp, + "mode": "extent", + "mode.extent.unit": "phy", + "mode.extent.ulx": ulx, + "mode.extent.uly": lry, # bug in OTB <= 7.3 : + "mode.extent.lrx": lrx, + "mode.extent.lry": uly, # ULY/LRY are inverted } - new_input = App('ExtractROI', params) + new_input = App("ExtractROI", params) # TODO: OTB 7.4 fixes this bug, how to handle different versions of OTB? new_inputs.append(new_input) # Potentially update the reference inputs for later resampling - if str(inp) == str(reference_pixel_size_input): # we use comparison of string because calling '==' + if str(inp) == str( + reference_pixel_size_input + ): # we use comparison of string because calling '==' # on pyotb objects implicitly calls BandMathX application, which is not desirable reference_pixel_size_input = new_input except RuntimeError as e: - logger.error('Cannot define the processing area for input %s: %s', inp, e) + logger.error( + "Cannot define the processing area for input %s: %s", inp, e + ) raise inputs = new_inputs # Update metadatas - metadatas = {input: input.app.GetImageMetaData('out') for input in inputs} + metadatas = {input: input.app.GetImageMetaData("out") for input in inputs} # Get a metadata of an arbitrary image. This is just to compare later with other images any_metadata = next(iter(metadatas.values())) # Handling different pixel sizes - if not all(metadata['GeoTransform'][1] == any_metadata['GeoTransform'][1] - and metadata['GeoTransform'][5] == any_metadata['GeoTransform'][5] - for metadata in metadatas.values()): + if not all( + metadata["GeoTransform"][1] == any_metadata["GeoTransform"][1] + and metadata["GeoTransform"][5] == any_metadata["GeoTransform"][5] + for metadata in metadatas.values() + ): # Retrieving the pixel size that will be common for all inputs - if pixel_size_rule == 'minimal': + if pixel_size_rule == "minimal": # selecting the input with the smallest x pixel size - reference_input = min(metadatas, key=lambda x: metadatas[x]['GeoTransform'][1]) - if pixel_size_rule == 'maximal': + reference_input = min( + metadatas, key=lambda x: metadatas[x]["GeoTransform"][1] + ) + if pixel_size_rule == "maximal": # selecting the input with the highest x pixel size - reference_input = max(metadatas, key=lambda x: metadatas[x]['GeoTransform'][1]) - elif pixel_size_rule == 'same_as_input': + reference_input = max( + metadatas, key=lambda x: metadatas[x]["GeoTransform"][1] + ) + elif pixel_size_rule == "same_as_input": reference_input = reference_pixel_size_input - elif pixel_size_rule == 'specify': + elif pixel_size_rule == "specify": pass # TODO : when the user explicitly specify the pixel size -> add argument inside the function - pixel_size = metadatas[reference_input]['GeoTransform'][1] + pixel_size = metadatas[reference_input]["GeoTransform"][1] # Perform resampling on inputs that do not comply with the target pixel size - logger.info('Resampling all inputs to resolution: %s', pixel_size) + logger.info("Resampling all inputs to resolution: %s", pixel_size) new_inputs = [] for inp in inputs: - if metadatas[inp]['GeoTransform'][1] != pixel_size: - superimposed = App('Superimpose', inr=reference_input, inm=inp, interpolator=interpolator) + if metadatas[inp]["GeoTransform"][1] != pixel_size: + superimposed = App( + "Superimpose", + inr=reference_input, + inm=inp, + interpolator=interpolator, + ) new_inputs.append(superimposed) else: new_inputs.append(inp) inputs = new_inputs - metadatas = {inp: inp.app.GetImageMetaData('out') for inp in inputs} + metadatas = {inp: inp.app.GetImageMetaData("out") for inp in inputs} # Final superimposition to be sure to have the exact same image sizes image_sizes = {} @@ -451,13 +546,22 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_ image_sizes[inp] = inp.shape[:2] # Selecting the most frequent image size. It will be used as reference. most_common_image_size, _ = Counter(image_sizes.values()).most_common(1)[0] - same_size_images = [inp for inp, image_size in image_sizes.items() if image_size == most_common_image_size] + same_size_images = [ + inp + for inp, image_size in image_sizes.items() + if image_size == most_common_image_size + ] # Superimposition for images that do not have the same size as the others new_inputs = [] for inp in inputs: if image_sizes[inp] != most_common_image_size: - superimposed = App('Superimpose', inr=same_size_images[0], inm=inp, interpolator=interpolator) + superimposed = App( + "Superimpose", + inr=same_size_images[0], + inm=inp, + interpolator=interpolator, + ) new_inputs.append(superimposed) else: new_inputs.append(inp) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 03200a2..5363937 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -1,12 +1,11 @@ # -*- coding: utf-8 -*- """This module helps to ensure we properly initialize pyotb: only in case OTB is found and apps are available.""" +import logging import os import sys -import logging from pathlib import Path from shutil import which - # Allow user to switch between OTB directories without setting every env variable OTB_ROOT = os.environ.get("OTB_ROOT") @@ -15,10 +14,14 @@ OTB_ROOT = os.environ.get("OTB_ROOT") # then use pyotb.set_logger_level() to adjust logger verbosity logger = logging.getLogger("pyOTB") logger_handler = logging.StreamHandler(sys.stdout) -formatter = logging.Formatter(fmt="%(asctime)s (%(levelname)-4s) [pyOTB] %(message)s", datefmt="%Y-%m-%d %H:%M:%S") +formatter = logging.Formatter( + fmt="%(asctime)s (%(levelname)-4s) [pyOTB] %(message)s", datefmt="%Y-%m-%d %H:%M:%S" +) logger_handler.setFormatter(formatter) # Search for PYOTB_LOGGER_LEVEL, else use OTB_LOGGER_LEVEL as pyOTB level, or fallback to INFO -LOG_LEVEL = os.environ.get("PYOTB_LOGGER_LEVEL") or os.environ.get("OTB_LOGGER_LEVEL") or "INFO" +LOG_LEVEL = ( + os.environ.get("PYOTB_LOGGER_LEVEL") or os.environ.get("OTB_LOGGER_LEVEL") or "INFO" +) logger.setLevel(getattr(logging, LOG_LEVEL)) # Here it would be possible to use a different level for a specific handler # A more verbose one can go to text file while print only errors to stdout @@ -60,6 +63,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru try: set_environment(prefix) import otbApplication as otb # pylint: disable=import-outside-toplevel + return otb except EnvironmentError as e: raise SystemExit(f"Failed to import OTB with prefix={prefix}") from e @@ -71,6 +75,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru # Here, we can't properly set env variables before OTB import. We assume user did this before running python # For LD_LIBRARY_PATH problems, use OTB_ROOT instead of PYTHONPATH import otbApplication as otb # pylint: disable=import-outside-toplevel + if "OTB_APPLICATION_PATH" not in os.environ: lib_dir = __find_lib(otb_module=otb) apps_path = __find_apps_path(lib_dir) @@ -79,7 +84,9 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru except ImportError as e: pythonpath = os.environ.get("PYTHONPATH") if not scan: - raise SystemExit(f"Failed to import OTB with env PYTHONPATH={pythonpath}") from e + raise SystemExit( + f"Failed to import OTB with env PYTHONPATH={pythonpath}" + ) from e # Else search system logger.info("Failed to import OTB. Searching for it...") prefix = __find_otb_root(scan_userdir) @@ -87,6 +94,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru try: set_environment(prefix) import otbApplication as otb # pylint: disable=import-outside-toplevel + return otb except EnvironmentError as e: raise SystemExit("Auto setup for OTB env failed. Exiting.") from e @@ -112,7 +120,7 @@ def set_environment(prefix: str): if not prefix.exists(): raise FileNotFoundError(str(prefix)) built_from_source = False - if not (prefix / 'README').exists(): + if not (prefix / "README").exists(): built_from_source = True # External libraries lib_dir = __find_lib(prefix) @@ -151,7 +159,9 @@ def set_environment(prefix: str): gdal_data = str(prefix / "share/data") proj_lib = str(prefix / "share/proj") else: - raise EnvironmentError(f"Can't find GDAL location with current OTB prefix '{prefix}' or in /usr") + raise EnvironmentError( + f"Can't find GDAL location with current OTB prefix '{prefix}' or in /usr" + ) os.environ["GDAL_DATA"] = gdal_data os.environ["PROJ_LIB"] = proj_lib @@ -168,7 +178,7 @@ def __find_lib(prefix: str = None, otb_module=None): """ if prefix is not None: - lib_dir = prefix / 'lib' + lib_dir = prefix / "lib" if lib_dir.exists(): return lib_dir.absolute() if otb_module is not None: @@ -276,33 +286,54 @@ def __suggest_fix_import(error_message: str, prefix: str): logger.critical("An error occurred while importing OTB Python API") logger.critical("OTB error message was '%s'", error_message) if sys.platform == "linux": - if error_message.startswith('libpython3.'): - logger.critical("It seems like you need to symlink or recompile python bindings") - if sys.executable.startswith('/usr/bin'): - lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so" - if which('ctest'): - logger.critical("To recompile python bindings, use 'cd %s ; source otbenv.profile ; " - "ctest -S share/otb/swig/build_wrapping.cmake -VV'", prefix) + if error_message.startswith("libpython3."): + logger.critical( + "It seems like you need to symlink or recompile python bindings" + ) + if sys.executable.startswith("/usr/bin"): + lib = ( + f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so" + ) + if which("ctest"): + logger.critical( + "To recompile python bindings, use 'cd %s ; source otbenv.profile ; " + "ctest -S share/otb/swig/build_wrapping.cmake -VV'", + prefix, + ) elif Path(lib).exists(): expect_minor = int(error_message[11]) if expect_minor != sys.version_info.minor: - logger.critical("Python library version mismatch (OTB was expecting 3.%s) : " - "a simple symlink may not work, depending on your python version", expect_minor) + logger.critical( + "Python library version mismatch (OTB was expecting 3.%s) : " + "a simple symlink may not work, depending on your python version", + expect_minor, + ) target_lib = f"{prefix}/lib/libpython3.{expect_minor}.so.rh-python3{expect_minor}-1.0" logger.critical("Use 'ln -s %s %s'", lib, target_lib) else: - logger.critical("You may need to install cmake in order to recompile python bindings") + logger.critical( + "You may need to install cmake in order to recompile python bindings" + ) else: - logger.critical("Unable to automatically locate python dynamic library of %s", sys.executable) + logger.critical( + "Unable to automatically locate python dynamic library of %s", + sys.executable, + ) elif sys.platform == "win32": if error_message.startswith("DLL load failed"): if sys.version_info.minor != 7: - logger.critical("You need Python 3.5 (OTB releases 6.4 to 7.4) or Python 3.7 (since OTB 8)") + logger.critical( + "You need Python 3.5 (OTB releases 6.4 to 7.4) or Python 3.7 (since OTB 8)" + ) else: - logger.critical("It seems that your env variables aren't properly set," - " first use 'call otbenv.bat' then try to import pyotb once again") + logger.critical( + "It seems that your env variables aren't properly set," + " first use 'call otbenv.bat' then try to import pyotb once again" + ) docs_link = "https://www.orfeo-toolbox.org/CookBook/Installation.html" - logger.critical("You can verify installation requirements for your OS at %s", docs_link) + logger.critical( + "You can verify installation requirements for your OS at %s", docs_link + ) # Since helpers is the first module to be inititialized, this will prevent pyotb to run if OTB is not found diff --git a/pyproject.toml b/pyproject.toml index ac4d3df..6f86ad0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,14 +5,18 @@ build-backend = "setuptools.build_meta" [project] name = "pyotb" description = "Library to enable easy use of the Orfeo ToolBox (OTB) in Python" -authors = [{name = "Rémi Cresson", email = "remi.cresson@inrae.fr"}, {name = "Nicolas Narçon"}, {name = "Vincent Delbar"}] +authors = [ + { name = "Rémi Cresson", email = "remi.cresson@inrae.fr" }, + { name = "Nicolas Narçon" }, + { name = "Vincent Delbar" }, +] requires-python = ">=3.7" keywords = ["gis", "remote sensing", "otb", "orfeotoolbox", "orfeo toolbox"] dependencies = ["numpy>=1.16"] readme = "README.md" license = { file = "LICENSE" } dynamic = ["version"] -classifiers=[ +classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", @@ -37,22 +41,23 @@ repository = "https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb" packages = ["pyotb"] [tool.setuptools.dynamic] -version = {attr = "pyotb.__version__"} +version = { attr = "pyotb.__version__" } [tool.pylint] -max-line-length = 120 +max-line-length = 88 max-module-lines = 2000 good-names = ["x", "y", "i", "j", "k", "e"] disable = [ "fixme", + "line-too-long", "too-many-locals", "too-many-branches", "too-many-statements", - "too-many-instance-attributes" + "too-many-instance-attributes", ] [tool.pydocstyle] convention = "google" [tool.black] -line-length = 120 +line-length = 88 -- GitLab From 5eddf64d71e60b86b6a9895d1f0177cc52e37f66 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 1 Jun 2023 18:36:38 +0000 Subject: [PATCH 194/399] Refac CI --- .gitlab-ci.yml | 125 ++++++++++++++++++++-------------------------- pyotb/__init__.py | 2 +- setup.py | 3 -- 3 files changed, 54 insertions(+), 76 deletions(-) delete mode 100644 setup.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index be29725..4d53e04 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,15 +1,23 @@ -workflow: - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_REF_NAME =~ /master/ - default: image: $CI_REGISTRY/orfeotoolbox/otb-build-env/otb-ubuntu-native-develop-headless:20.04 tags: - light + interruptible: true + +variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + +cache: + key: $CI_COMMIT_REF_SLUG + paths: + - .cache/pip + +workflow: + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" stages: - Static Analysis - - Documentation test - Tests - Ship @@ -18,33 +26,11 @@ stages: .static_analysis: stage: Static Analysis allow_failure: true - -codespell: rules: - changes: - - pyotb/*.py - - tests/*.py - - README.md - extends: .static_analysis - before_script: - - pip install codespell - script: - - codespell {pyotb,tests,doc,README.md} - -flake8: - rules: - - changes: - - pyotb/*.py - extends: .static_analysis - before_script: - - pip install flake8 - script: - - flake8 --max-line-length=120 $PWD/pyotb --ignore=F403,E402,F401,W503,W504 + - pyotb/*.py pydocstyle: - rules: - - changes: - - pyotb/*.py extends: .static_analysis before_script: - pip install pydocstyle tomli @@ -52,83 +38,74 @@ pydocstyle: - pydocstyle $PWD/pyotb pylint: - rules: - - changes: - - pyotb/*.py extends: .static_analysis before_script: - pip install pylint script: - pylint $PWD/pyotb -# ---------------------------------- Documentation ---------------------------------- - -.docs: +codespell: + extends: .static_analysis + rules: + - changes: + - .gitlab-ci.yml + - pyotb/*.py + - tests/*.py + - doc/*.md + - README.md before_script: - - apt-get update && apt-get -y install virtualenv - - virtualenv doc_env - - source doc_env/bin/activate - - pip install -U pip - - pip install -U -r doc/doc_requirements.txt - -pages_test: - stage: Documentation test - extends: .docs - except: - - master - when: manual + - pip install codespell script: - - mkdocs build --site-dir public_test - artifacts: - paths: - - public_test + - codespell {pyotb,tests,doc,README.md} -# -------------------------------------- Tests -------------------------------------- +# -------------------------------------- Tests -------------------------------------- .tests: stage: Tests + allow_failure: false rules: - changes: - - pyotb/*.py - - tests/*.py - allow_failure: false + - .gitlab-ci.yml + - pyotb/*.py + - tests/*.py variables: OTB_ROOT: /opt/otb LD_LIBRARY_PATH: /opt/otb/lib OTB_LOGGER_LEVEL: INFO PYOTB_LOGGER_LEVEL: DEBUG - IMAGE_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false - TEST_INPUT_IMAGE: tests/image.tif artifacts: reports: junit: test-*.xml before_script: - - wget $IMAGE_URL -O $TEST_INPUT_IMAGE - - pip install pytest requests + - pip install pytest + - pip install . test_core: extends: .tests script: - - python3 -m pytest --color=yes --junitxml=test-core.xml tests/test_core.py + - pytest --color=yes --junitxml=test-core.xml tests/test_core.py test_numpy: extends: .tests script: - - python3 -m pytest --color=yes --junitxml=test-numpy.xml tests/test_numpy.py + - pytest --color=yes --junitxml=test-numpy.xml tests/test_numpy.py test_pipeline: - #when: manual extends: .tests script: - - python3 -m pytest --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py + - pytest --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py -# -------------------------------------- Ship --------------------------------------- +# -------------------------------------- Ship --------------------------------------- -pages: +docs: stage: Ship - extends: .docs - only: - - master + when: manual + before_script: + - apt update && apt install -y python3.8-venv + - python3 -m venv docs_venv + - source docs_venv/bin/activate + - python3 -m pip install -U pip + - python3 -m pip install -r doc/doc_requirements.txt script: - mkdocs build --site-dir public artifacts: @@ -137,11 +114,15 @@ pages: pypi: stage: Ship + when: manual only: - - master + - tags + - master + - develop + before_script: - - apt update && apt install -y python3.8-venv - - python3 -m pip install --upgrade build twine + - apt update && apt install -y python3.8-venv + - pip install --upgrade build twine script: - - python3 -m build - - python3 -m twine upload --repository-url https://upload.pypi.org/legacy/ --non-interactive -u __token__ -p $pypi_token dist/* + - python3 -m build + - python3 -m twine upload --non-interactive -u __token__ -p $pypi_token dist/* diff --git a/pyotb/__init__.py b/pyotb/__init__.py index c9fe258..65943d1 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """This module provides convenient python wrapping of otbApplications.""" -__version__ = "2.0.0" +__version__ = "2.0.0.dev1" from .helpers import logger, set_logger_level from .apps import * diff --git a/setup.py b/setup.py deleted file mode 100644 index 6068493..0000000 --- a/setup.py +++ /dev/null @@ -1,3 +0,0 @@ -from setuptools import setup - -setup() -- GitLab From 110c28c6106b3764dc547ac08e9374ab165a04eb Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 1 Jun 2023 19:14:08 +0000 Subject: [PATCH 195/399] Change CI trigger rule with branch name OR merge request event --- .gitlab-ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4d53e04..77cfdca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,7 +14,7 @@ cache: workflow: rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "develop" || $CI_PIPELINE_SOURCE == "merge_request_event" stages: - Static Analysis @@ -99,7 +99,10 @@ test_pipeline: docs: stage: Ship - when: manual + #when: manual + only: + - master + - develop before_script: - apt update && apt install -y python3.8-venv - python3 -m venv docs_venv @@ -114,9 +117,8 @@ docs: pypi: stage: Ship - when: manual + #when: manual only: - - tags - master - develop -- GitLab From afc8290e9eac6e8c0adc3655e5912f1608c79638 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 1 Jun 2023 21:25:39 +0000 Subject: [PATCH 196/399] Resolve "CI trigger is not right" --- .gitlab-ci.yml | 18 ++++++++++-------- pyotb/__init__.py | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 77cfdca..dea3721 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,7 +14,11 @@ cache: workflow: rules: - - if: $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "develop" || $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_TAG + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_REF_PROTECTED && $CI_OPEN_MERGE_REQUESTS + when: never + - if: $CI_COMMIT_REF_PROTECTED stages: - Static Analysis @@ -99,10 +103,9 @@ test_pipeline: docs: stage: Ship - #when: manual + when: manual only: - - master - - develop + - tags before_script: - apt update && apt install -y python3.8-venv - python3 -m venv docs_venv @@ -117,14 +120,13 @@ docs: pypi: stage: Ship - #when: manual + when: manual only: - - master - - develop + - tags before_script: - apt update && apt install -y python3.8-venv - - pip install --upgrade build twine + - pip install build twine script: - python3 -m build - python3 -m twine upload --non-interactive -u __token__ -p $pypi_token dist/* diff --git a/pyotb/__init__.py b/pyotb/__init__.py index 65943d1..5191fd6 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """This module provides convenient python wrapping of otbApplications.""" -__version__ = "2.0.0.dev1" +__version__ = "2.0.0.dev2" from .helpers import logger, set_logger_level from .apps import * -- GitLab From 75385e348d279cd4268052364a8be3cecee21d68 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 1 Jun 2023 23:51:45 +0200 Subject: [PATCH 197/399] CI: test token --- .gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dea3721..22b7127 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,11 +14,9 @@ cache: workflow: rules: - - if: $CI_COMMIT_TAG - - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_REF_PROTECTED|| $CI_COMMIT_TAG || $CI_PIPELINE_SOURCE == "merge_request_event" - if: $CI_COMMIT_REF_PROTECTED && $CI_OPEN_MERGE_REQUESTS when: never - - if: $CI_COMMIT_REF_PROTECTED stages: - Static Analysis -- GitLab From 77f3e7c3b873522dfa53fb30057f8e9bdc711ea7 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 1 Jun 2023 23:54:37 +0200 Subject: [PATCH 198/399] CI: test rules on protected branch --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 22b7127..dea3721 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,9 +14,11 @@ cache: workflow: rules: - - if: $CI_COMMIT_REF_PROTECTED|| $CI_COMMIT_TAG || $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_TAG + - if: $CI_PIPELINE_SOURCE == "merge_request_event" - if: $CI_COMMIT_REF_PROTECTED && $CI_OPEN_MERGE_REQUESTS when: never + - if: $CI_COMMIT_REF_PROTECTED stages: - Static Analysis -- GitLab From f372db40498cbc49409930477ac93bf679ed058c Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 2 Jun 2023 00:00:54 +0200 Subject: [PATCH 199/399] CI: auto ship on tags, build docs only for protected branch --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dea3721..39da3b0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -103,9 +103,9 @@ test_pipeline: docs: stage: Ship - when: manual only: - - tags + - master + - develop before_script: - apt update && apt install -y python3.8-venv - python3 -m venv docs_venv @@ -120,7 +120,7 @@ docs: pypi: stage: Ship - when: manual + # when: manual only: - tags -- GitLab From 8549252ea645e773d2cff2c0d9374208a457a147 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 2 Jun 2023 11:52:38 +0200 Subject: [PATCH 200/399] ENH: pixel_type is 2d argument in write() --- pyotb/core.py | 2 +- tests/test_core.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index dfe2598..06da0c7 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -729,9 +729,9 @@ class App(OTBObject): def write( self, path: str | Path | dict[str, str] = None, - ext_fname: str = "", pixel_type: dict[str, str] | str = None, preserve_dtype: bool = False, + ext_fname: str = "", **kwargs, ) -> bool: """Set output pixel type and write the output raster files. diff --git a/tests/test_core.py b/tests/test_core.py index f766c58..99b21de 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -246,8 +246,8 @@ def test_ndvi_comparison(): ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (INPUT[:, :, -1] + INPUT[:, :, 0]) ndvi_indices = pyotb.RadiometricIndices(INPUT, {"list": ["Vegetation:NDVI"], "channels.red": 1, "channels.nir": 4}) assert ndvi_bandmath.exp == "((im1b4 - im1b1) / (im1b4 + im1b1))" - assert ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", pixel_type="float") - assert ndvi_indices.write("/tmp/ndvi_indices.tif", pixel_type="float") + assert ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", "float") + assert ndvi_indices.write("/tmp/ndvi_indices.tif", "float") compared = pyotb.CompareImages({"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"}) assert (compared["count"], compared["mse"]) == (0, 0) -- GitLab From 814efc6714058e7ad9add98af1df58532062f31f Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 2 Jun 2023 11:54:16 +0200 Subject: [PATCH 201/399] DOC: update README since pixel_type kwarg is not required --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d72b294..5447959 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ roi = ndvi[20:586, 9:572] # Pipeline execution # The actual computation happens here ! -roi.write("output.tif", pixel_type="float") +roi.write("output.tif", "float") ``` pyotb's objects also enable easy interoperability with -- GitLab From 0b5a84e8c5c70798c74b396d1811155d787fb577 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 6 Jun 2023 08:02:23 +0000 Subject: [PATCH 202/399] CI: last changes since workflow update --- .gitlab-ci.yml | 28 ++++++++++++++-------------- doc/examples/nodata_mean.md | 1 - doc/installation.md | 5 +---- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 39da3b0..49211fa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,6 +23,7 @@ workflow: stages: - Static Analysis - Tests + - Documentation - Ship # -------------------------------- Static analysis -------------------------------- @@ -52,11 +53,8 @@ codespell: extends: .static_analysis rules: - changes: - - .gitlab-ci.yml - - pyotb/*.py - - tests/*.py - - doc/*.md - - README.md + - "**/*.py" + - "**/*.md" before_script: - pip install codespell script: @@ -69,9 +67,7 @@ codespell: allow_failure: false rules: - changes: - - .gitlab-ci.yml - - pyotb/*.py - - tests/*.py + - "**/*.py" variables: OTB_ROOT: /opt/otb LD_LIBRARY_PATH: /opt/otb/lib @@ -99,13 +95,16 @@ test_pipeline: script: - pytest --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py -# -------------------------------------- Ship --------------------------------------- +# -------------------------------------- Docs --------------------------------------- docs: - stage: Ship - only: - - master - - develop + stage: Documentation + # when: manual + rules: + - changes: + - mkdocs.yml + - doc/* + - pyotb/*.py before_script: - apt update && apt install -y python3.8-venv - python3 -m venv docs_venv @@ -118,12 +117,13 @@ docs: paths: - public +# -------------------------------------- Ship --------------------------------------- + pypi: stage: Ship # when: manual only: - tags - before_script: - apt update && apt install -y python3.8-venv - pip install build twine diff --git a/doc/examples/nodata_mean.md b/doc/examples/nodata_mean.md index 9ec4900..d3818ce 100644 --- a/doc/examples/nodata_mean.md +++ b/doc/examples/nodata_mean.md @@ -23,4 +23,3 @@ mean.write('ndvi_annual_mean.tif') ``` Note that no actual computation is executed before the last line where the result is written to disk. - diff --git a/doc/installation.md b/doc/installation.md index de5c0ae..404d7bd 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -6,7 +6,7 @@ Requirements: - Orfeo ToolBox binaries (follow these [instructions](https://www.orfeo-toolbox.org/CookBook/Installation.html)) - Orfeo ToolBox python binding (follow these - [instructions](https://www.orfeo-toolbox.org/CookBook/Installation.html)) + [instructions](https://www.orfeo-toolbox.org/CookBook/Installation.html#python-bindings)) ## Install with pip @@ -26,6 +26,3 @@ pip install -e ".[dev]" If you need compatibility with python3.6, install `pyotb<2.0` and for python3.5 use `pyotb==1.2.2`. - -For Python>=3.6, latest version available is pyotb 1.5.0. -For Python 3.5, latest version available is pyotb 1.2.2 -- GitLab From 0b1ae1e0a5d2c1e455a0f235bfe1cbbb18d0f1c1 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Tue, 27 Jun 2023 16:04:58 +0200 Subject: [PATCH 203/399] DOC: Lot of docfixes --- doc/MISC.md | 20 ++++-- doc/comparison_otb.md | 135 +++++++++++++++++++++++++++--------- doc/examples/nodata_mean.md | 20 ++++-- doc/examples/pleiades.md | 31 +++++++-- doc/features.md | 10 ++- doc/functions.md | 57 ++++++++++----- doc/index.md | 9 ++- doc/interaction.md | 79 +++++++++++++++------ doc/managing_loggers.md | 15 ++-- doc/otb_versions.md | 12 ++-- doc/quickstart.md | 91 ++++++++++++++++++------ doc/troubleshooting.md | 18 +++-- 12 files changed, 365 insertions(+), 132 deletions(-) diff --git a/doc/MISC.md b/doc/MISC.md index 2a92b03..6be696f 100644 --- a/doc/MISC.md +++ b/doc/MISC.md @@ -1,10 +1,12 @@ ## Miscellaneous: Work with images with different footprints / resolutions -OrfeoToolBox provides a handy `Superimpose` application that enables the projection of an image into the geometry of another one. +OrfeoToolBox provides a handy `Superimpose` application that enables the +projection of an image into the geometry of another one. In pyotb, a function has been created to handle more than 2 images. -Let's consider the case where we have 3 images with different resolutions and different footprints : +Let's consider the case where we have 3 images with different resolutions and +different footprints :  @@ -12,7 +14,9 @@ Let's consider the case where we have 3 images with different resolutions and di import pyotb # transforming filepaths to pyotb objects -s2_image, vhr_image, labels = pyotb.Input('image_10m.tif'), pyotb.Input('image_60cm.tif'), pyotb.Input('land_cover_2m.tif') +s2_image = pyotb.Input('image_10m.tif') +vhr_image = pyotb.Input('image_60cm.tif') +labels = pyotb.Input('land_cover_2m.tif') print(s2_image.shape) # (286, 195, 4) print(vhr_image.shape) # (2048, 2048, 3) @@ -30,9 +34,13 @@ Here is the final result : The piece of code to achieve this : ```python -s2_image, vhr_image, labels = pyotb.define_processing_area(s2_image, vhr_image, labels, window_rule='intersection', - pixel_size_rule='same_as_input', - reference_pixel_size_input=labels, interpolator='bco') +s2_image, vhr_image, labels = pyotb.define_processing_area( + s2_image, vhr_image, labels, + window_rule='intersection', + pixel_size_rule='same_as_input', + reference_pixel_size_input=labels, + interpolator='bco' +) print(s2_image.shape) # (657, 520, 4) print(vhr_image.shape) # (657, 520, 3) diff --git a/doc/comparison_otb.md b/doc/comparison_otb.md index a733678..29d1e89 100644 --- a/doc/comparison_otb.md +++ b/doc/comparison_otb.md @@ -2,17 +2,31 @@ ### Single application execution -Using OTB, the code would be like : +<table> +<tr> +<th> OTB </th> +<th> pyotb </th> +</tr> +<tr> +<td> ```python import otbApplication input_path = 'my_image.tif' -resampled = otbApplication.Registry.CreateApplication('RigidTransformResample') +resampled = otbApplication.Registry.CreateApplication( + 'RigidTransformResample' +) resampled.SetParameterString('in', input_path) resampled.SetParameterString('interpolator', 'linear') -resampled.SetParameterFloat('transform.type.id.scalex', 0.5) -resampled.SetParameterFloat('transform.type.id.scaley', 0.5) +resampled.SetParameterFloat( + 'transform.type.id.scalex', + 0.5 +) +resampled.SetParameterFloat( + 'transform.type.id.scaley', + 0.5 +) resampled.SetParameterString('out', 'output.tif') resampled.SetParameterOutputImagePixelType( 'out', otbApplication.ImagePixelType_uint16 @@ -21,7 +35,8 @@ resampled.SetParameterOutputImagePixelType( resampled.ExecuteAndWriteOutput() ``` -Using pyotb: +</td> +<td> ```python import pyotb @@ -36,6 +51,10 @@ resampled = pyotb.RigidTransformResample({ resampled.write('output.tif', pixel_type='uint16') ``` +</td> +</tr> +</table> + ### In-memory connections <table> @@ -126,61 +145,105 @@ app3.write( ### Arithmetic operations -Every pyotb object supports arithmetic operations, such as addition, subtraction, comparison... -Consider an example where we want to perform the arithmetic operation `image1 * image2 - 2*image3` +Every pyotb object supports arithmetic operations, such as addition, +subtraction, comparison... +Consider an example where we want to perform the arithmetic operation +`image1 * image2 - 2*image3`. -Using OTB, the following code works for 3-bands images : +<table> +<tr> +<th> OTB </th> +<th> pyotb </th> +</tr> +<tr> +<td> ```python import otbApplication -bmx = otbApplication.Registry.CreateApplication('BandMathX') -bmx.SetParameterStringList('il', ['image1.tif', 'image2.tif', 'image3.tif']) # all images are 3-bands -exp = 'im1b1*im2b1 - 2*im3b1; im1b2*im2b2 - 2*im3b2; im1b3*im2b3 - 2*im3b3' +bmx = otbApplication.Registry.CreateApplication( + 'BandMathX' +) +bmx.SetParameterStringList( + 'il', + ['image1.tif', 'image2.tif', 'image3.tif'] +) # all images are 3-bands +exp = ('im1b1*im2b1-2*im3b1; ' + 'im1b2*im2b2-2*im3b2; ' + 'im1b3*im2b3-2*im3b3') bmx.SetParameterString('exp', exp) bmx.SetParameterString('out', 'output.tif') -bmx.SetParameterOutputImagePixelType('out', otbApplication.ImagePixelType_uint8) +bmx.SetParameterOutputImagePixelType( + 'out', + otbApplication.ImagePixelType_uint8 +) bmx.ExecuteAndWriteOutput() ``` -With pyotb, the following works with images of any number of bands : +In OTB, this code works for 3-bands images. + +</td> +<td> ```python import pyotb # transforming filepaths to pyotb objects -input1, input2, input3 = pyotb.Input('image1.tif'), pyotb.Input('image2.tif') , pyotb.Input('image3.tif') +input1 = pyotb.Input('image1.tif') +input2 = pyotb.Input('image2.tif') +input3 = pyotb.Input('image3.tif') res = input1 * input2 - 2 * input2 res.write('output.tif', pixel_type='uint8') ``` +In pyotb,this code works with images of any number of bands. + +</td> +</tr> +</table> + ### Slicing -Using OTB, for selection bands or ROI, the code looks like: +<table> +<tr> +<th> OTB </th> +<th> pyotb </th> +</tr> +<tr> +<td> + ```python import otbApplication -# selecting first 3 bands -extracted = otbApplication.Registry.CreateApplication('ExtractROI') -extracted.SetParameterString('in', 'my_image.tif') -extracted.SetParameterStringList('cl', ['Channel1', 'Channel2', 'Channel3']) -extracted.Execute() - -# selecting 1000x1000 subset -extracted = otbApplication.Registry.CreateApplication('ExtractROI') -extracted.SetParameterString('in', 'my_image.tif') -extracted.SetParameterString('mode', 'extent') -extracted.SetParameterString('mode.extent.unit', 'pxl') -extracted.SetParameterFloat('mode.extent.ulx', 0) -extracted.SetParameterFloat('mode.extent.uly', 0) -extracted.SetParameterFloat('mode.extent.lrx', 999) -extracted.SetParameterFloat('mode.extent.lry', 999) -extracted.Execute() +# first 3 channels +app = otbApplication.Registry.CreateApplication( + 'ExtractROI' +) +app.SetParameterString('in', 'my_image.tif') +app.SetParameterStringList( + 'cl', + ['Channel1', 'Channel2', 'Channel3'] +) +app.Execute() + +# 1000x1000 roi +app = otbApplication.Registry.CreateApplication( + 'ExtractROI' +) +app.SetParameterString('in', 'my_image.tif') +app.SetParameterString('mode', 'extent') +app.SetParameterString('mode.extent.unit', 'pxl') +app.SetParameterFloat('mode.extent.ulx', 0) +app.SetParameterFloat('mode.extent.uly', 0) +app.SetParameterFloat('mode.extent.lrx', 999) +app.SetParameterFloat('mode.extent.lry', 999) +app.Execute() ``` -Instead, using pyotb: +</td> +<td> ```python import pyotb @@ -188,6 +251,10 @@ import pyotb # transforming filepath to pyotb object inp = pyotb.Input('my_image.tif') -extracted = inp[:, :, :3] # selecting first 3 bands -extracted = inp[:1000, :1000] # selecting 1000x1000 subset +extracted = inp[:, :, :3] # first 3 channels +extracted = inp[:1000, :1000] # 1000x1000 roi ``` + +</td> +</tr> +</table> \ No newline at end of file diff --git a/doc/examples/nodata_mean.md b/doc/examples/nodata_mean.md index d3818ce..288be6e 100644 --- a/doc/examples/nodata_mean.md +++ b/doc/examples/nodata_mean.md @@ -1,7 +1,9 @@ ### Compute the mean of several rasters, taking into account NoData -Let's consider we have at disposal 73 NDVI rasters for a year, where clouds have been masked with NoData (nodata value of -10 000 for example). +Let's consider we have at disposal 73 NDVI rasters for a year, where clouds +have been masked with NoData (nodata value of -10 000 for example). -Goal: compute the mean across time (keeping the spatial dimension) of the NDVIs, excluding cloudy pixels. Piece of code to achieve that: +Goal: compute the mean across time (keeping the spatial dimension) of the +NDVIs, excluding cloudy pixels. Piece of code to achieve that: ```python import pyotb @@ -13,13 +15,21 @@ ndvis = [pyotb.Input(path) for path in ndvi_paths] summed = sum([pyotb.where(ndvi != nodata, ndvi, 0) for ndvi in ndvis]) # Printing the generated BandMath expression -print(summed.exp) # this returns a very long exp: "0 + ((im1b1 != -10000) ? im1b1 : 0) + ((im2b1 != -10000) ? im2b1 : 0) + ... + ((im73b1 != -10000) ? im73b1 : 0)" +print(summed.exp) +# this returns a very long exp: +# "0 + ((im1b1 != -10000) ? im1b1 : 0) + ((im2b1 != -10000) ? im2b1 : 0) + ... +# ... + ((im73b1 != -10000) ? im73b1 : 0)" # For each pixel location, getting the count of valid pixels count = sum([pyotb.where(ndvi == nodata, 0, 1) for ndvi in ndvis]) -mean = summed / count # BandMath exp of this is very long: "(0 + ((im1b1 != -10000) ? im1b1 : 0) + ... + ((im73b1 != -10000) ? im73b1 : 0)) / (0 + ((im1b1 == -10000) ? 0 : 1) + ... + ((im73b1 == -10000) ? 0 : 1))" +mean = summed / count +# BandMath exp of this is very long: +# "(0 + ((im1b1 != -10000) ? im1b1 : 0) + ... +# + ((im73b1 != -10000) ? im73b1 : 0)) / (0 + ((im1b1 == -10000) ? 0 : 1) + ... +# + ((im73b1 == -10000) ? 0 : 1))" mean.write('ndvi_annual_mean.tif') ``` -Note that no actual computation is executed before the last line where the result is written to disk. +Note that no actual computation is executed before the last line where the +result is written to disk. diff --git a/doc/examples/pleiades.md b/doc/examples/pleiades.md index 32d0447..eb3f1f2 100644 --- a/doc/examples/pleiades.md +++ b/doc/examples/pleiades.md @@ -6,15 +6,34 @@ import pyotb srtm = '/media/data/raster/nasa/srtm_30m' geoid = '/media/data/geoid/egm96.grd' -pan = pyotb.OpticalCalibration('IMG_PHR1A_P_001/DIM_PHR1A_P_201509011347379_SEN_1791374101-001.XML', level='toa') -ms = pyotb.OpticalCalibration('IMG_PHR1A_MS_002/DIM_PHR1A_MS_201509011347379_SEN_1791374101-002.XML', level='toa') +pan = pyotb.OpticalCalibration( + 'IMG_PHR1A_P_001/DIM_PHR1A_P_201509011347379_SEN_1791374101-001.XML', + level='toa' +) +ms = pyotb.OpticalCalibration( + 'IMG_PHR1A_MS_002/DIM_PHR1A_MS_201509011347379_SEN_1791374101-002.XML', + level='toa' +) -pan_ortho = pyotb.OrthoRectification({'io.in': pan, 'elev.dem': srtm, 'elev.geoid': geoid}) -ms_ortho = pyotb.OrthoRectification({'io.in': ms, 'elev.dem': srtm, 'elev.geoid': geoid}) +pan_ortho = pyotb.OrthoRectification({ + 'io.in': pan, + 'elev.dem': srtm, + 'elev.geoid': geoid +}) +ms_ortho = pyotb.OrthoRectification({ + 'io.in': ms, + 'elev.dem': srtm, + 'elev.geoid': geoid +}) -pxs = pyotb.BundleToPerfectSensor(inp=pan_ortho, inxs=ms_ortho, method='bayes', mode='default') +pxs = pyotb.BundleToPerfectSensor( + inp=pan_ortho, + inxs=ms_ortho, + method='bayes', + mode='default' +) exfn = '?gdal:co:COMPRESS=DEFLATE&gdal:co:PREDICTOR=2&gdal:co:BIGTIFF=YES' # Here we trigger every app in the pipeline and the process is blocked until result is written to disk -pxs.write('pxs_image.tif', pixel_type='uint16', filename_extension=exfn) +pxs.write('pxs_image.tif', pixel_type='uint16', ext_fname=exfn) ``` diff --git a/doc/features.md b/doc/features.md index cc2c39e..2058af5 100644 --- a/doc/features.md +++ b/doc/features.md @@ -1,7 +1,9 @@ ## Arithmetic operations -Every pyotb object supports arithmetic operations, such as addition, subtraction, comparison... -Consider an example where we want to compute a vegeteation mask from NDVI, i.e. the arithmetic operation `(nir - red) / (nir + red) > 0.3` +Every pyotb object supports arithmetic operations, such as addition, +subtraction, comparison... +Consider an example where we want to compute a vegeteation mask from NDVI, +i.e. the arithmetic operation `(nir - red) / (nir + red) > 0.3` With pyotb, one can simply do : @@ -12,7 +14,9 @@ import pyotb nir, red = pyotb.Input('nir.tif'), pyotb.Input('red.tif') res = (nir - red) / (nir + red) > 0.3 -print(res.exp) # prints the BandMath expression: "((im1b1 - im2b1) / (im1b1 + im2b1)) > 0.3 ? 1 : 0" +# Prints the BandMath expression: +# "((im1b1 - im2b1) / (im1b1 + im2b1)) > 0.3 ? 1 : 0" +print(res.exp) res.write('vegetation_mask.tif', pixel_type='uint8') ``` diff --git a/doc/functions.md b/doc/functions.md index 070ba0b..1e82c58 100644 --- a/doc/functions.md +++ b/doc/functions.md @@ -1,26 +1,47 @@ -Some functions have been written, entirely based on OTB, to mimic the behavior of some well-known numpy functions. +Some functions have been written, entirely based on OTB, to mimic the behavior +of some well-known numpy functions. + ## pyotb.where + Equivalent of `numpy.where`. -It is the equivalent of the muparser syntax `condition ? x : y` that can be used in OTB's BandMath. +It is the equivalent of the muparser syntax `condition ? x : y` that can be +used in OTB's BandMath. ```python import pyotb # transforming filepaths to pyotb objects -labels, image1, image2 = pyotb.Input('labels.tif'), pyotb.Input('image1.tif') , pyotb.Input('image2.tif') +labels = pyotb.Input('labels.tif') +image1 = pyotb.Input('image1.tif') +image2 = pyotb.Input('image2.tif') # If labels = 1, returns image1. Else, returns image2 -res = pyotb.where(labels == 1, image1, image2) # this would also work: pyotb.where(labels == 1, 'image1.tif', 'image2.tif') +res = pyotb.where(labels == 1, image1, image2) +# this would also work: `pyotb.where(labels == 1, 'image1.tif', 'image2.tif')` -# A more complex example -# If labels = 1, returns image1. If labels = 2, returns image2. If labels = 3, returns 3. Else 0 -res = pyotb.where(labels == 1, image1, - pyotb.where(labels == 2, image2, - pyotb.where(labels == 3, 3, 0))) +# A more complex example: +# - If labels = 1 --> returns image1, +# - If labels = 2 --> returns image2, +# - If labels = 3 --> returns 3.0, +# - Else, returns 0.0 +res = pyotb.where( + labels == 1, + image1, + pyotb.where( + labels == 2, + image2, + pyotb.where( + labels == 3, + 3.0, + 0.0 + ) + ) +) ``` ## pyotb.clip + Equivalent of `numpy.clip`. Clip (limit) the values in a raster to a range. ```python @@ -30,18 +51,20 @@ res = pyotb.clip('my_image.tif', 0, 255) # clips the values between 0 and 255 ``` ## pyotb.all + Equivalent of `numpy.all`. -For only one image, this function checks that all bands of the image are True (i.e. !=0) and outputs -a singleband boolean raster. -For several images, this function checks that all images are True (i.e. !=0) and outputs -a boolean raster, with as many bands as the inputs. +For only one image, this function checks that all bands of the image are True +(i.e. !=0) and outputs a single band boolean raster. +For several images, this function checks that all images are True (i.e. !=0) +and outputs a boolean raster, with as many bands as the inputs. ## pyotb.any + Equivalent of `numpy.any`. -For only one image, this function checks that at least one band of the image is True (i.e. !=0) and outputs -a singleband boolean raster. -For several images, this function checks that at least one of the images is True (i.e. !=0) and outputs -a boolean raster, with as many bands as the inputs. \ No newline at end of file +For only one image, this function checks that at least one band of the image +is True (i.e. !=0) and outputs a single band boolean raster. +For several images, this function checks that at least one of the images is +True (i.e. !=0) and outputs a boolean raster, with as many bands as the inputs. \ No newline at end of file diff --git a/doc/index.md b/doc/index.md index fa29918..448d139 100644 --- a/doc/index.md +++ b/doc/index.md @@ -1,11 +1,13 @@ # Pyotb: Orfeo Toolbox for Python -pyotb is a Python extension of Orfeo Toolbox. It has been built on top of the existing Python API of OTB, in order +pyotb is a Python extension of Orfeo Toolbox. It has been built on top of the +existing Python API of OTB, in order to make OTB more Python friendly. # Table of Contents ## Get started + - [Installation](installation.md) - [How to use pyotb](quickstart.md) - [Useful features](features.md) @@ -13,10 +15,12 @@ to make OTB more Python friendly. - [Interaction with Python libraries (numpy, rasterio, tensorflow)](interaction.md) ## Examples + - [Pleiades data processing](examples/pleiades.md) - [Computing the mean of several rasters with NoData](examples/nodata_mean.md) ## Advanced use + - [Comparison between pyotb and OTB native library](comparison_otb.md) - [OTB versions](otb_versions.md) - [Managing loggers](managing_loggers.md) @@ -24,3 +28,6 @@ to make OTB more Python friendly. ## API + +- See the API reference. If you have any doubts or questions, feel free to ask +on github or gitlab! \ No newline at end of file diff --git a/doc/interaction.md b/doc/interaction.md index 217dbaa..45da657 100644 --- a/doc/interaction.md +++ b/doc/interaction.md @@ -6,8 +6,14 @@ pyotb objects can be exported to numpy array. import pyotb import numpy as np -calibrated = pyotb.OpticalCalibration('image.tif', level='toa') # this is a pyotb object -arr = np.asarray(calibrated) # same as calibrated.to_numpy() +# The following is a pyotb object +calibrated = pyotb.OpticalCalibration('image.tif', level='toa') + +# The following is a numpy array +arr = np.asarray(calibrated) + +# Note that the following is equivalent: +arr = calibrated.to_numpy() ``` ## Interaction with Numpy @@ -20,21 +26,26 @@ For example: import pyotb import numpy as np -inp = pyotb.Input('image.tif') # this is a pyotb object +# The following is a pyotb object +inp = pyotb.Input('image.tif') -# Creating a numpy array of noise -white_noise = np.random.normal(0, 50, size=inp.shape) # this is a numpy object +# Creating a numpy array of noise. The following is a numpy object +white_noise = np.random.normal(0, 50, size=inp.shape) # Adding the noise to the image -noisy_image = inp + white_noise # magic: this is a pyotb object that has the same georeference as input. - # `np.add(inp, white_noise)` would have worked the same +noisy_image = inp + white_noise +# Magically, this is a pyotb object that has the same geo-reference as `inp`. +# Note the `np.add(inp, white_noise)` would have worked the same + +# Finally we can write the result like any pyotb object noisy_image.write('image_plus_noise.tif') ``` Limitations : - The whole image is loaded into memory -- The georeference can not be modified. Thus, numpy operations can not change the image or pixel size +- The georeference can not be modified. Thus, numpy operations can not change +the image or pixel size ## Export to rasterio @@ -48,15 +59,23 @@ import rasterio from scipy import ndimage # Pansharpening + NDVI + creating bare soils mask -pxs = pyotb.BundleToPerfectSensor(inp='panchromatic.tif', inxs='multispectral.tif') -ndvi = pyotb.RadiometricIndices({'in': pxs, 'channels.red': 3, 'channels.nir': 4, 'list': 'Vegetation:NDVI'}) +pxs = pyotb.BundleToPerfectSensor( + inp='panchromatic.tif', + inxs='multispectral.tif' +) +ndvi = pyotb.RadiometricIndices({ + 'in': pxs, + 'channels.red': 3, + 'channels.nir': 4, + 'list': 'Vegetation:NDVI' +}) bare_soil_mask = (ndvi < 0.3) # Exporting the result as array & profile usable by rasterio mask_array, profile = bare_soil_mask.to_rasterio() -# Doing something in Python that is not possible with OTB, e.g. gathering the contiguous groups of pixels -# with an integer index +# Doing something in Python that is not possible with OTB, e.g. gathering +# the contiguous groups of pixels with an integer index labeled_mask_array, nb_groups = ndimage.label(mask_array) # Writing the result to disk @@ -64,14 +83,18 @@ with rasterio.open('labeled_bare_soil.tif', 'w', **profile) as f: f.write(labeled_mask_array) ``` -This way of exporting pyotb objects is more flexible that exporting to numpy, as the user gets the `profile` dictionary. -If the georeference or pixel size is modified, the user can update the `profile` accordingly. +This way of exporting pyotb objects is more flexible that exporting to numpy, +as the user gets the `profile` dictionary. +If the georeference or pixel size is modified, the user can update the +`profile` accordingly. ## Interaction with Tensorflow -We saw that numpy operations had some limitations. To bypass those limitations, it is possible to use some Tensorflow operations on pyotb objects. +We saw that numpy operations had some limitations. To bypass those +limitations, it is possible to use some Tensorflow operations on pyotb objects. -You need a working installation of OTBTF >=3.0 for this and then the code is like this: +You need a working installation of OTBTF >=3.0 for this and then the code is +like this: ```python import pyotb @@ -82,29 +105,39 @@ def scalar_product(x1, x2): return tf.reduce_sum(tf.multiply(x1, x2), axis=-1) # Compute the scalar product -res = pyotb.run_tf_function(scalar_product)('image1.tif', 'image2.tif') # magic: this is a pyotb object +res = pyotb.run_tf_function(scalar_product)('image1.tif', 'image2.tif') + +# Magically, `res` is a pyotb object res.write('scalar_product.tif') ``` -For some easy syntax, one can use `pyotb.run_tf_function` as a function decorator, such as: +For some easy syntax, one can use `pyotb.run_tf_function` as a function +decorator, such as: ```python import pyotb -@pyotb.run_tf_function # The decorator enables the use of pyotb objects as inputs/output of the function +# The `pyotb.run_tf_function` decorator enables the use of pyotb objects as +# inputs/output of the function +@pyotb.run_tf_function def scalar_product(x1, x2): import tensorflow as tf return tf.reduce_sum(tf.multiply(x1, x2), axis=-1) -res = scalar_product('image1.tif', 'image2.tif') # magic: this is a pyotb object +res = scalar_product('image1.tif', 'image2.tif') +# Magically, `res` is a pyotb object ``` Advantages : -- The process supports streaming, hence the whole image is **not** loaded into memory +- The process supports streaming, hence the whole image is **not** loaded into +memory - Can be integrated in OTB pipelines Limitations : -- It is not possible to use the tensorflow python API inside a script where OTBTF is used because of compilation issues - between Tensorflow and OTBTF, i.e. `import tensorflow` doesn't work in a script where OTBTF apps have been initialized +- For OTBTF versions < 4.0.0, it is not possible to use the tensorflow python +API inside a script where OTBTF is used because of libraries clashing between +Tensorflow and OTBTF, i.e. `import tensorflow` doesn't work in a script where +OTBTF apps have been initialized. This is why we recommend to use latest OTBTF +versions diff --git a/doc/managing_loggers.md b/doc/managing_loggers.md index 31c960b..5c2d80b 100644 --- a/doc/managing_loggers.md +++ b/doc/managing_loggers.md @@ -1,22 +1,27 @@ ## Managing loggers -Several environment variables are used in order to adjust logger level and behaviour. It should be set before importing pyotb. +Several environment variables are used in order to adjust logger level and +behaviour. It should be set before importing pyotb. - `OTB_LOGGER_LEVEL` : used to set the default OTB logger level. -- `PYOTB_LOGGER_LEVEL` : used to set the pyotb logger level. if not set, `OTB_LOGGER_LEVEL` will be used. +- `PYOTB_LOGGER_LEVEL` : used to set the pyotb logger level. if not set, +- `OTB_LOGGER_LEVEL` will be used. If none of those two variables is set, the logger level will be set to 'INFO'. Available levels are : DEBUG, INFO, WARNING, ERROR, CRITICAL -You may also change the logger level after import (for pyotb only) with the function `set_logger_level`. +You may also change the logger level after import (for pyotb only) with the +function `set_logger_level`. ```python import pyotb pyotb.set_logger_level('DEBUG') ``` -Bonus : in some cases, yo may want to silence the GDAL driver logger (for example you will see a lot of errors when reading GML files with OGR). -One useful trick is to redirect these logs to a file. This can be done using the variable `CPL_LOG`. +Bonus : in some cases, yo may want to silence the GDAL driver logger (for +example you will see a lot of errors when reading GML files with OGR). +One useful trick is to redirect these logs to a file. This can be done using +the variable `CPL_LOG`. ## Named applications in logs diff --git a/doc/otb_versions.md b/doc/otb_versions.md index 7eeb5cf..50f30ee 100644 --- a/doc/otb_versions.md +++ b/doc/otb_versions.md @@ -1,6 +1,7 @@ ## System with multiple OTB versions -If you want to quickly switch between OTB versions, or override the default system version, you may use the `OTB_ROOT` env variable : +If you want to quickly switch between OTB versions, or override the default +system version, you may use the `OTB_ROOT` env variable : ```python import os @@ -14,7 +15,8 @@ import pyotb 2022-06-14 13:59:04 (INFO) [pyOTB] Successfully loaded 126 OTB applications ``` -If you try to import pyotb without having set environment, it will try to find any OTB version installed on your system: +If you try to import pyotb without having set environment, it will try to find +any OTB version installed on your system: ```python import pyotb @@ -38,11 +40,13 @@ Here is the path precedence for this automatic env configuration : OR (for windows) : C:/Program Files ``` -N.B. : in case `otbApplication` is found in `PYTHONPATH` (and if `OTB_ROOT` was not set), the OTB which the python API is linked to will be used. +N.B. : in case `otbApplication` is found in `PYTHONPATH` (and if `OTB_ROOT` +was not set), the OTB which the python API is linked to will be used. ## Fresh OTB installation -If you've just installed OTB binaries in a Linux environment, you may encounter an error at first import, pyotb will help you fix it : +If you've just installed OTB binaries in a Linux environment, you may +encounter an error at first import, pyotb will help you fix it : ```python import pyotb diff --git a/doc/quickstart.md b/doc/quickstart.md index 294d512..915b3f3 100644 --- a/doc/quickstart.md +++ b/doc/quickstart.md @@ -1,73 +1,120 @@ ## Quickstart: running an OTB application with pyotb -pyotb has been written so that it is more convenient to run an application in Python. + +pyotb has been written so that it is more convenient to run an application in +Python. You can pass the parameters of an application as a dictionary : ```python import pyotb -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'transform.type.id.scaley': 0.5, - 'interpolator': 'linear', 'transform.type.id.scalex': 0.5}) +resampled = pyotb.RigidTransformResample({ + 'in': 'my_image.tif', + 'transform.type.id.scaley': 0.5, + 'interpolator': 'linear', + 'transform.type.id.scalex': 0.5 +}) ``` -Note that pyotb has a 'lazy' evaluation: it only performs operation when it is needed, i.e. results are written to disk. +Note that pyotb has a 'lazy' evaluation: it only performs operation when it is +needed, i.e. results are written to disk. Thus, the previous line doesn't trigger the application. -To actually trigger the application execution, you need to write the result to disk: +To actually trigger the application execution, you need to write the result to +disk: ```python resampled.write('output.tif') # this is when the application actually runs ``` ### Using Python keyword arguments -It is also possible to use the Python keyword arguments notation for passing the parameters: + +It is also possible to use the Python keyword arguments notation for passing +the parameters: + ```python output = pyotb.SuperImpose(inr='reference_image.tif', inm='image.tif') ``` + is equivalent to: + ```python output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'}) ``` -Limitations : for this notation, python doesn't accept the parameter `in` or any parameter that contains a `.`. E.g., it is not possible to use `pyotb.RigidTransformResample(in=input_path...)` or `pyotb.VectorDataExtractROI(io.vd=vector_path...)`. +Limitations : for this notation, python doesn't accept the parameter `in` or +any parameter that contains a `.`. E.g., it is not possible to use +`pyotb.RigidTransformResample(in=input_path...)` or +`pyotb.VectorDataExtractROI(io.vd=vector_path...)`. ## In-memory connections + The big asset of pyotb is the ease of in-memory connections between apps. -Let's start from our previous example. Consider the case where one wants to apply optical calibration and binary morphological dilatation +Let's start from our previous example. Consider the case where one wants to +apply optical calibration and binary morphological dilatation following the undersampling. Using pyotb, you can pass the output of an app as input of another app : + ```python import pyotb -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) - -calibrated = pyotb.OpticalCalibration({'in': resampled, 'level': 'toa'}) +resampled = pyotb.RigidTransformResample({ + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 +}) + +calibrated = pyotb.OpticalCalibration({ + 'in': resampled, + 'level': 'toa' +}) + +dilated = pyotb.BinaryMorphologicalOperation({ + 'in': calibrated, + 'out': 'output.tif', + 'filter': 'dilate', + 'structype': 'ball', + 'xradius': 3, 'yradius': 3 +}) -dilated = pyotb.BinaryMorphologicalOperation({'in': calibrated, 'out': 'output.tif', 'filter': 'dilate', - 'structype': 'ball', 'xradius': 3, 'yradius': 3}) dilated.write('result.tif') ``` ## Writing the result of an app + Any pyotb object can be written to disk using the `write` method, e.g. : ```python import pyotb -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) +resampled = pyotb.RigidTransformResample({ + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 +}) + # Here you can set optionally pixel type and extended filename variables -resampled.write({'out': 'output.tif'}, pixel_type='uint16', filename_extension='?nodata=65535') +resampled.write( + {'out': 'output.tif'}, + pixel_type='uint16', + ext_fname='?nodata=65535' +) ``` -Another possibility for writing results is to set the output parameter when initializing the application: +Another possibility for writing results is to set the output parameter when +initializing the application: + ```python import pyotb -resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', 'out': 'output.tif', - 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5}) -# Here you can set optionally pixel type and extended filename variables -resampled.write(pixel_type='uint16', filename_extension='?nodata=65535') +resampled = pyotb.RigidTransformResample({ + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'out': 'output.tif', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 +}) ``` \ No newline at end of file diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md index e96e3e7..62cfafd 100644 --- a/doc/troubleshooting.md +++ b/doc/troubleshooting.md @@ -1,14 +1,20 @@ -## Troubleshooting: known limitations +## Troubleshooting: known limitations with old versions + +!!! Info + + All defects described below have been fixed since OTB 8.1.2 and pyotb 2.0.0 ### Failure of intermediate writing -When chaining applications in-memory, there may be some problems when writing intermediate results, depending on the order +When chaining applications in-memory, there may be some problems when writing +intermediate results, depending on the order the writings are requested. Some examples can be found below: #### Example of failures involving slicing -For some applications (non-exhaustive know list: OpticalCalibration, DynamicConvert, BandMath), we can face unexpected - failures when using channels slicing +For some applications (non-exhaustive know list: OpticalCalibration, +DynamicConvert, BandMath), we can face unexpected failures when using channels +slicing ```python import pyotb @@ -55,8 +61,8 @@ one_band.write('one_band.tif') #### Example of failures involving arithmetic operation -One can meet errors when using arithmetic operations at the end of a pipeline when DynamicConvert, BandMath or -OpticalCalibration is involved: +One can meet errors when using arithmetic operations at the end of a pipeline +when DynamicConvert, BandMath or OpticalCalibration is involved: ```python import pyotb -- GitLab From bd7163b6e7f0a07018cb0b103f501d3995bad57b Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Tue, 27 Jun 2023 16:23:32 +0200 Subject: [PATCH 204/399] DOC: update mkdocs.yml --- mkdocs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mkdocs.yml b/mkdocs.yml index 5654279..29d51d4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -59,6 +59,7 @@ extra_css: use_directory_urls: false # this creates some pyotb/core.html pages instead of pyotb/core/index.html markdown_extensions: + - admonition - toc: permalink: true title: On this page @@ -67,6 +68,8 @@ markdown_extensions: anchor_linenums: true - pymdownx.inlinehilite - pymdownx.snippets + - pymdownx.details + - pymdownx.superfences # Rest of the navigation.. site_name: "pyotb documentation: a Python extension of OTB" -- GitLab From ed3bdf2eabe37d1033f2404c6123b0bd79391b45 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Tue, 27 Jun 2023 16:23:43 +0200 Subject: [PATCH 205/399] DOC: update troubleshooting --- doc/troubleshooting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md index 62cfafd..f8a4739 100644 --- a/doc/troubleshooting.md +++ b/doc/troubleshooting.md @@ -1,6 +1,6 @@ ## Troubleshooting: known limitations with old versions -!!! Info +!!! note All defects described below have been fixed since OTB 8.1.2 and pyotb 2.0.0 -- GitLab From 8c777f26dc63d2451fc9a0ab320d55976ad04332 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Tue, 27 Jun 2023 16:32:56 +0200 Subject: [PATCH 206/399] Doc: shorten authors info --- AUTHORS.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index 7ac46df..6518258 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -2,9 +2,9 @@ ## Initial codebase -* Nicolas NARÇON - IT engineer @ ESA - Rome (Italy) +* Nicolas NARÇON (INRAE, now ESA) ## Current maintainers -* Rémi CRESSON - RS research engineer @ INRAe - Montpellier (France) -* Vincent DELBAR - GIS engineer @ La TeleScop - Montpellier (France) +* Rémi CRESSON (INRAE) +* Vincent DELBAR (La TeleScop) -- GitLab From fe592d755c12554d4f1e096e6b979fdac393e7a4 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Tue, 27 Jun 2023 16:33:13 +0200 Subject: [PATCH 207/399] Doc: add highlights (warning/note) --- doc/examples/pleiades.md | 6 ++++-- doc/interaction.md | 24 ++++++++++++++---------- doc/troubleshooting.md | 2 +- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/doc/examples/pleiades.md b/doc/examples/pleiades.md index eb3f1f2..120a554 100644 --- a/doc/examples/pleiades.md +++ b/doc/examples/pleiades.md @@ -1,5 +1,6 @@ ### Process raw Pleiades data -This is a common case of Pleiades data preprocessing : optical calibration -> orthorectification -> pansharpening +This is a common case of Pleiades data preprocessing : +*optical calibration -> orthorectification -> pansharpening* ```python import pyotb @@ -34,6 +35,7 @@ pxs = pyotb.BundleToPerfectSensor( ) exfn = '?gdal:co:COMPRESS=DEFLATE&gdal:co:PREDICTOR=2&gdal:co:BIGTIFF=YES' -# Here we trigger every app in the pipeline and the process is blocked until result is written to disk +# Here we trigger every app in the pipeline and the process is blocked until +# result is written to disk pxs.write('pxs_image.tif', pixel_type='uint16', ext_fname=exfn) ``` diff --git a/doc/interaction.md b/doc/interaction.md index 45da657..e4c73fb 100644 --- a/doc/interaction.md +++ b/doc/interaction.md @@ -41,11 +41,13 @@ noisy_image = inp + white_noise noisy_image.write('image_plus_noise.tif') ``` -Limitations : +!!! warning -- The whole image is loaded into memory -- The georeference can not be modified. Thus, numpy operations can not change -the image or pixel size + Limitations : + + - The whole image is loaded into memory + - The georeference can not be modified. Thus, numpy operations can not change + the image or pixel size ## Export to rasterio @@ -134,10 +136,12 @@ Advantages : memory - Can be integrated in OTB pipelines -Limitations : +!!! warning -- For OTBTF versions < 4.0.0, it is not possible to use the tensorflow python -API inside a script where OTBTF is used because of libraries clashing between -Tensorflow and OTBTF, i.e. `import tensorflow` doesn't work in a script where -OTBTF apps have been initialized. This is why we recommend to use latest OTBTF -versions + Limitations : + + - For OTBTF versions < 4.0.0, it is not possible to use the tensorflow + python API inside a script where OTBTF is used because of libraries + clashing between Tensorflow and OTBTF, i.e. `import tensorflow` doesn't + work in a script where OTBTF apps have been initialized. This is why we + recommend to use latest OTBTF versions diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md index f8a4739..0025913 100644 --- a/doc/troubleshooting.md +++ b/doc/troubleshooting.md @@ -4,7 +4,7 @@ All defects described below have been fixed since OTB 8.1.2 and pyotb 2.0.0 -### Failure of intermediate writing +### Failure of intermediate writing (otb < 8.1, pyotb < 1.5.4) When chaining applications in-memory, there may be some problems when writing intermediate results, depending on the order -- GitLab From 001a9cae44c81e82d5a3d8d26cfa86eec019f9fc Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 27 Jun 2023 16:19:22 +0000 Subject: [PATCH 208/399] Apply 1 suggestion(s) to 1 file(s) --- doc/managing_loggers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/managing_loggers.md b/doc/managing_loggers.md index 5c2d80b..7220af4 100644 --- a/doc/managing_loggers.md +++ b/doc/managing_loggers.md @@ -18,7 +18,7 @@ import pyotb pyotb.set_logger_level('DEBUG') ``` -Bonus : in some cases, yo may want to silence the GDAL driver logger (for +Bonus : in some cases, you may want to silence the GDAL driver logger (for example you will see a lot of errors when reading GML files with OGR). One useful trick is to redirect these logs to a file. This can be done using the variable `CPL_LOG`. -- GitLab From 7aa5940c69f1a6e1bb054d301d14fbab73b4cdf6 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 27 Jun 2023 16:19:36 +0000 Subject: [PATCH 209/399] Apply 1 suggestion(s) to 1 file(s) --- doc/quickstart.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/doc/quickstart.md b/doc/quickstart.md index 915b3f3..5e07e43 100644 --- a/doc/quickstart.md +++ b/doc/quickstart.md @@ -42,9 +42,18 @@ output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'}) ``` Limitations : for this notation, python doesn't accept the parameter `in` or -any parameter that contains a `.`. E.g., it is not possible to use -`pyotb.RigidTransformResample(in=input_path...)` or -`pyotb.VectorDataExtractROI(io.vd=vector_path...)`. +any parameter that contains a dots (e.g. `io.in)`. +For `in` and other main input parameters of an OTB app, you may simply pass +the value as first argument, pyotb will guess the parameter name. +For parameters that contains dots, you can either use a dictionary, or replace dots (`.`) with underscores (`_`) as follow : + +```python +resampled = pyotb.RigidTransformResample( + 'my_image.tif', + interpolator = 'linear', + transform_type_id_scaley = 0.5, + transform_type_id_scalex = 0.5 +) ## In-memory connections -- GitLab From 3cc6378de5cddfc4b5a7977373a1de6490c4b3c1 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 27 Jun 2023 16:39:36 +0000 Subject: [PATCH 210/399] Fix CI trigger --- .gitlab-ci.yml | 8 ++++---- pyotb/__init__.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49211fa..a5d80fd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,11 +14,11 @@ cache: workflow: rules: - - if: $CI_COMMIT_TAG - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - if: $CI_COMMIT_REF_PROTECTED && $CI_OPEN_MERGE_REQUESTS + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" when: never - - if: $CI_COMMIT_REF_PROTECTED + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_TAG + - if: $CI_COMMIT_REF_PROTECTED == "true" stages: - Static Analysis diff --git a/pyotb/__init__.py b/pyotb/__init__.py index 5191fd6..b3bc21c 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """This module provides convenient python wrapping of otbApplications.""" -__version__ = "2.0.0.dev2" +__version__ = "2.0.0.dev3" from .helpers import logger, set_logger_level from .apps import * -- GitLab From dbeca6c2944fa6821e209c0527f5ac0582d8fec5 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Tue, 27 Jun 2023 21:30:32 +0200 Subject: [PATCH 211/399] CI: add metadata tests --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a5d80fd..bfad764 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -95,6 +95,11 @@ test_pipeline: script: - pytest --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py +test_metadata: + extends: .tests + script: + - pytest --color=yes --junitxml=test-metadata.xml tests/test_metadata.py + # -------------------------------------- Docs --------------------------------------- docs: -- GitLab From b970b1af79588d2be593adb2a72384eb0b53d370 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Tue, 27 Jun 2023 21:30:46 +0200 Subject: [PATCH 212/399] ADD: metadata tests --- tests/test_metadata.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/test_metadata.py diff --git a/tests/test_metadata.py b/tests/test_metadata.py new file mode 100644 index 0000000..5d42a44 --- /dev/null +++ b/tests/test_metadata.py @@ -0,0 +1,11 @@ +from tests_data import INPUT +import pyotb + +INPUT2 = pyotb.Input( + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/" + "47/Q/RU/2021/12/S2B_47QRU_20211227_0_L2A/B04.tif" +) + +def test_metadata(): + assert "ProjectionRef", "TIFFTAG_SOFTWARE" in INPUT.metadata + assert "ProjectionRef", "OVR_RESAMPLING_ALG" in INPUT2.metadata -- GitLab From c5a15613e4709aa0a4da3769a0a1936f6c466368 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Tue, 27 Jun 2023 21:31:20 +0200 Subject: [PATCH 213/399] ENH: metadata property to return IMD and MDD --- pyotb/core.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 06da0c7..4f5a724 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -39,8 +39,36 @@ class OTBObject(ABC): @property def metadata(self) -> dict[str, (str, float, list[float])]: - """Return first output image metadata dictionary.""" - return dict(self.app.GetMetadataDictionary(self.output_image_key)) + """Return the concatenation of the first output image metadata + dictionary and the metadata dictionary.""" + + # Image Metadata only when there are values + otb_imd = self.app.GetImageMetadata(self.output_image_key) + cats = ["Num", "Str", "L1D", "Time"] + imd = { + key: getattr(otb_imd, f"get_{cat.lower()}")(key) + for cat in cats + for key in getattr(otb_imd, f"GetKeyList{cat}")().split(" ") + if getattr(otb_imd, "has")(key) + } + + # Metadata dictionary + # Replace items like {"metadata_1": "TIFFTAG_SOFTWARE=CSinG - 13 + # SEPTEMBRE 2012"} with {"TIFFTAG_SOFTWARE": "CSinG - 13 SEPTEMBRE + # 2012"} + mdd = dict(self.app.GetMetadataDictionary(self.output_image_key)) + new_mdd = { + splits[0].strip() + if (replace := (k.lower().startswith("metadata_") and + len(splits := v.split("=")) == 2)) else k: + splits[1].strip() if replace else v + for k, v in mdd.items() + } + + return { + **new_mdd, + **imd + } @property def dtype(self) -> np.dtype: -- GitLab From 9cd14d7f01366df5e67e3aad0e876329065c4200 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Tue, 27 Jun 2023 21:46:28 +0200 Subject: [PATCH 214/399] CI: move metadata tests inside core tests --- .gitlab-ci.yml | 5 ----- tests/test_metadata.py | 11 ----------- 2 files changed, 16 deletions(-) delete mode 100644 tests/test_metadata.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bfad764..a5d80fd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -95,11 +95,6 @@ test_pipeline: script: - pytest --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py -test_metadata: - extends: .tests - script: - - pytest --color=yes --junitxml=test-metadata.xml tests/test_metadata.py - # -------------------------------------- Docs --------------------------------------- docs: diff --git a/tests/test_metadata.py b/tests/test_metadata.py deleted file mode 100644 index 5d42a44..0000000 --- a/tests/test_metadata.py +++ /dev/null @@ -1,11 +0,0 @@ -from tests_data import INPUT -import pyotb - -INPUT2 = pyotb.Input( - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/" - "47/Q/RU/2021/12/S2B_47QRU_20211227_0_L2A/B04.tif" -) - -def test_metadata(): - assert "ProjectionRef", "TIFFTAG_SOFTWARE" in INPUT.metadata - assert "ProjectionRef", "OVR_RESAMPLING_ALG" in INPUT2.metadata -- GitLab From 69b31abc40d88b7944130d5694b0ce1e1f993e15 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Tue, 27 Jun 2023 21:46:40 +0200 Subject: [PATCH 215/399] ADD: move metadata tests inside core tests --- tests/test_core.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 99b21de..52cdc97 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,6 +1,5 @@ import pytest -import pyotb from tests_data import * @@ -78,7 +77,12 @@ def test_data(): def test_metadata(): - assert INPUT.metadata["Metadata_1"] == "TIFFTAG_SOFTWARE=CSinG - 13 SEPTEMBRE 2012" + INPUT2 = pyotb.Input( + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/" + "47/Q/RU/2021/12/S2B_47QRU_20211227_0_L2A/B04.tif" + ) + assert "ProjectionRef", "TIFFTAG_SOFTWARE" in INPUT.metadata + assert "ProjectionRef", "OVR_RESAMPLING_ALG" in INPUT2.metadata def test_nonraster_property(): -- GitLab From f3bcefd9bfbbb2adc46074a74fd94b3725b1345c Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Tue, 27 Jun 2023 21:53:07 +0200 Subject: [PATCH 216/399] DOC: metadata property doc --- pyotb/core.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 4f5a724..05198ef 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -39,10 +39,13 @@ class OTBObject(ABC): @property def metadata(self) -> dict[str, (str, float, list[float])]: - """Return the concatenation of the first output image metadata - dictionary and the metadata dictionary.""" + """Return metadata. - # Image Metadata only when there are values + The returned dict results from the concatenation of the first output + image metadata dictionary and the metadata dictionary. + + """ + # Image Metadata otb_imd = self.app.GetImageMetadata(self.output_image_key) cats = ["Num", "Str", "L1D", "Time"] imd = { -- GitLab From 82f3ab70f906e3479d0138fb3183da97c349c527 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Tue, 27 Jun 2023 22:02:53 +0200 Subject: [PATCH 217/399] DOC: metadata property doc --- pyotb/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 05198ef..daa6b60 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -41,8 +41,8 @@ class OTBObject(ABC): def metadata(self) -> dict[str, (str, float, list[float])]: """Return metadata. - The returned dict results from the concatenation of the first output - image metadata dictionary and the metadata dictionary. + The returned dict results from the concatenation of the first output + image metadata dictionary and the metadata dictionary. """ # Image Metadata -- GitLab From 35bfb6ab95fdbf7cb48ea4a8e743dd7df8bf6851 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Tue, 27 Jun 2023 22:06:48 +0200 Subject: [PATCH 218/399] DOC: metadata property doc --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index daa6b60..676b7c5 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -44,7 +44,7 @@ class OTBObject(ABC): The returned dict results from the concatenation of the first output image metadata dictionary and the metadata dictionary. - """ + """ # Image Metadata otb_imd = self.app.GetImageMetadata(self.output_image_key) cats = ["Num", "Str", "L1D", "Time"] -- GitLab From e03489e79b820316cff21105108a6af02e73777a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr> Date: Wed, 28 Jun 2023 08:58:13 +0000 Subject: [PATCH 219/399] Apply 1 suggestion(s) to 1 file(s) --- pyotb/core.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 676b7c5..7675858 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -60,13 +60,13 @@ class OTBObject(ABC): # SEPTEMBRE 2012"} with {"TIFFTAG_SOFTWARE": "CSinG - 13 SEPTEMBRE # 2012"} mdd = dict(self.app.GetMetadataDictionary(self.output_image_key)) - new_mdd = { - splits[0].strip() - if (replace := (k.lower().startswith("metadata_") and - len(splits := v.split("=")) == 2)) else k: - splits[1].strip() if replace else v - for k, v in mdd.items() - } + new_mdd = {} + for key, val in mdd.items(): + splits = val.split("=") + if key.lower().startswith("metadata_") and len(splits) == 2: + new_mdd[splits[0].strip()] = splits[1].strip() + else: + new_mdd[key] = val return { **new_mdd, -- GitLab From bb736175a863c44e53994455e84c9991924353ba Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 28 Jun 2023 14:09:42 +0200 Subject: [PATCH 220/399] ADD: process metadata values other that str --- pyotb/core.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 7675858..040968d 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -62,11 +62,14 @@ class OTBObject(ABC): mdd = dict(self.app.GetMetadataDictionary(self.output_image_key)) new_mdd = {} for key, val in mdd.items(): - splits = val.split("=") - if key.lower().startswith("metadata_") and len(splits) == 2: - new_mdd[splits[0].strip()] = splits[1].strip() - else: - new_mdd[key] = val + new_key = key + new_val = val + if isinstance(val, str): + splits = val.split("=") + if key.lower().startswith("metadata_") and len(splits) == 2: + new_key = splits[0].strip() + new_val = splits[1].strip() + new_mdd[new_key] = new_val return { **new_mdd, -- GitLab From 491242716abc3baaf0222f690ee8b43c51758c06 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 28 Jun 2023 14:37:04 +0200 Subject: [PATCH 221/399] ADD: process metadata values other that str --- tests/test_core.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index 52cdc97..eab42d4 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -84,6 +84,12 @@ def test_metadata(): assert "ProjectionRef", "TIFFTAG_SOFTWARE" in INPUT.metadata assert "ProjectionRef", "OVR_RESAMPLING_ALG" in INPUT2.metadata + # Metadata with numeric values (e.g. TileHintX) + fp = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/" \ + "Data/Input/radarsat2/RADARSAT2_ALTONA_300_300_VV.tif?inline=false" + app = pyotb.BandMath({"il": [fp], "exp": "im1b1"}) + assert "TileHintX" in app.metadata + def test_nonraster_property(): with pytest.raises(TypeError): -- GitLab From 8fa6e5e7dfededfff8286b4862db70b647687b12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr> Date: Wed, 28 Jun 2023 13:34:13 +0000 Subject: [PATCH 222/399] Extended filenames as dict --- pyotb/core.py | 44 ++++++++++++++++++++++---------- tests/test_core.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 13 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 040968d..1582079 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -810,19 +810,37 @@ class App(OTBObject): # Append filename extension to filenames if ext_fname: - logger.debug( - "%s: using extended filename for outputs: %s", self.name, ext_fname - ) - if not ext_fname.startswith("?"): - ext_fname = "?&" + ext_fname - elif not ext_fname.startswith("?&"): - ext_fname = "?&" + ext_fname[1:] - for key, value in kwargs.items(): - if ( - self._out_param_types[key] == otb.ParameterType_OutputImage - and "?" not in value - ): - parameters[key] = value + ext_fname + if not isinstance(ext_fname, (dict, str)): + raise ValueError("Extended filename must be a str or a dict") + def _str2dict(ext_str): + """Function that converts str to dict.""" + splits = [pair.split("=") for pair in ext_str.split("&")] + return dict(split for split in splits if len(split) == 2) + + if isinstance(ext_fname, str): + ext_fname = _str2dict(ext_fname) + + logger.debug("%s: extended filename for all outputs:", self.name) + for key, ext in ext_fname.items(): + logger.debug("%s: %s", key, ext) + + for key, filepath in kwargs.items(): + if self._out_param_types[key] == otb.ParameterType_OutputImage: + new_ext_fname = ext_fname.copy() + + # grab already set extended filename key/values + if "?&" in filepath: + filepath, already_set_ext = filepath.split("?&", 1) + # extensions in filepath prevail over `new_ext_fname` + new_ext_fname.update(_str2dict(already_set_ext)) + + # transform dict to str + ext_fname_str = "&".join([ + f"{key}={value}" + for key, value in new_ext_fname.items() + ]) + parameters[key] = f"{filepath}?&{ext_fname_str}" + # Manage output pixel types data_types = {} if pixel_type: diff --git a/tests/test_core.py b/tests/test_core.py index eab42d4..5207b80 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -121,6 +121,68 @@ def test_write(): INPUT["out"].filepath.unlink() +def test_ext_fname(): + def _check(expected: str, key: str = "out", app = INPUT.app): + fn = app.GetParameterString(key) + assert "?&" in fn + assert fn.split("?&", 1)[1] == expected + + assert INPUT.write("/tmp/test_write.tif", ext_fname="nodata=0") + _check("nodata=0") + assert INPUT.write("/tmp/test_write.tif", ext_fname={"nodata": "0"}) + _check("nodata=0") + assert INPUT.write("/tmp/test_write.tif", ext_fname={"nodata": 0}) + _check("nodata=0") + assert INPUT.write( + "/tmp/test_write.tif", + ext_fname={ + "nodata": 0, + "gdal:co:COMPRESS": "DEFLATE" + } + ) + _check("nodata=0&gdal:co:COMPRESS=DEFLATE") + assert INPUT.write( + "/tmp/test_write.tif", + ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE" + ) + _check("nodata=0&gdal:co:COMPRESS=DEFLATE") + assert INPUT.write( + "/tmp/test_write.tif?&box=0:0:10:10", + ext_fname={ + "nodata": "0", + "gdal:co:COMPRESS": "DEFLATE", + "box": "0:0:20:20" + } + ) + # Check that the bbox is the one specified in the filepath, not the one + # specified in `ext_filename` + _check("nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:10:10") + assert INPUT.write( + "/tmp/test_write.tif?&box=0:0:10:10", + ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:20:20" + ) + _check("nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:10:10") + + INPUT["out"].filepath.unlink() + + mss = pyotb.MeanShiftSmoothing(FILEPATH) + mss.write( + { + "fout": "/tmp/test_ext_fn_fout.tif?&nodata=1", + "foutpos": "/tmp/test_ext_fn_foutpos.tif?&nodata=2" + }, + ext_fname={ + "nodata": 0, + "gdal:co:COMPRESS": "DEFLATE" + } + ) + _check("nodata=1&gdal:co:COMPRESS=DEFLATE", key="fout", app=mss.app) + _check("nodata=2&gdal:co:COMPRESS=DEFLATE", key="foutpos", app=mss.app) + mss["fout"].filepath.unlink() + mss["foutpos"].filepath.unlink() + + + def test_frozen_app_write(): app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True) assert app.write("/tmp/test_frozen_app_write.tif") -- GitLab From d811ee0959387080ded681dfd23e4c79626bc2ed Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 28 Jun 2023 15:08:29 +0000 Subject: [PATCH 223/399] Avoid storing full license in package metadata --- pyotb/__init__.py | 2 +- pyproject.toml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyotb/__init__.py b/pyotb/__init__.py index b3bc21c..3e09a63 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """This module provides convenient python wrapping of otbApplications.""" -__version__ = "2.0.0.dev3" +__version__ = "2.0.0.dev4" from .helpers import logger, set_logger_level from .apps import * diff --git a/pyproject.toml b/pyproject.toml index 6f86ad0..bc2c576 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,15 +6,15 @@ build-backend = "setuptools.build_meta" name = "pyotb" description = "Library to enable easy use of the Orfeo ToolBox (OTB) in Python" authors = [ - { name = "Rémi Cresson", email = "remi.cresson@inrae.fr" }, - { name = "Nicolas Narçon" }, - { name = "Vincent Delbar" }, + {name="Rémi Cresson", email="remi.cresson@inrae.fr"}, + {name="Nicolas Narçon"}, + {name="Vincent Delbar"}, ] requires-python = ">=3.7" keywords = ["gis", "remote sensing", "otb", "orfeotoolbox", "orfeo toolbox"] dependencies = ["numpy>=1.16"] readme = "README.md" -license = { file = "LICENSE" } +license = {text="Apache-2.0"} dynamic = ["version"] classifiers = [ "Programming Language :: Python :: 3", @@ -41,7 +41,7 @@ repository = "https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb" packages = ["pyotb"] [tool.setuptools.dynamic] -version = { attr = "pyotb.__version__" } +version = {attr="pyotb.__version__"} [tool.pylint] max-line-length = 88 -- GitLab From ebf7646f4d8d143459c2ab2980676acac4c03857 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 29 Jun 2023 23:27:13 +0200 Subject: [PATCH 224/399] ADD: test for bug #106 --- tests/test_core.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index 5207b80..5f65ca0 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -203,6 +203,10 @@ def test_frozen_output_write(): assert app["out"].write("/tmp/test_frozen_app_write.tif") app["out"].filepath.unlink() +def test_parameters(): + app = pyotb.OrthoRectification(FILEPATH) + assert isinstance(app.parameters["map"], str) + assert app.parameters["map"] == "utm" def test_output_in_arg(): info = pyotb.ReadImageInfo(INPUT["out"]) -- GitLab From ef04e7415115aa32960c454fb335938c5399a57c Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 29 Jun 2023 23:31:31 +0200 Subject: [PATCH 225/399] FIX: try to fix #106 --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 1582079..08877d4 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -558,7 +558,7 @@ class App(OTBObject): @property def parameters(self): """Return used OTB application parameters.""" - return {**self._auto_parameters, **self.app.GetParameters(), **self._settings} + return {**self.app.GetParameters(), **self._auto_parameters, **self._settings} @property def exports_dic(self) -> dict[str, dict]: -- GitLab From 645556c76f95a272c287dfb5007e78cbb96305d1 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 30 Jun 2023 10:29:05 +0200 Subject: [PATCH 226/399] ENH: remove key from App._auto_parameters when set by user --- pyotb/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyotb/core.py b/pyotb/core.py index 08877d4..ff0905b 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -691,6 +691,8 @@ class App(OTBObject): self._settings[key] = obj if key in self.outputs: self.outputs[key].filepath = obj + if key in self._auto_parameters: + del self._auto_parameters[key] def propagate_dtype(self, target_key: str = None, dtype: int = None): """Propagate a pixel type from main input to every outputs, or to a target output key only. -- GitLab From 3ea66cf97cb1b764b2fc2b479dafa1478492eb41 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 30 Jun 2023 10:52:02 +0200 Subject: [PATCH 227/399] FIX: duplicated test_parameters function + try update param --- tests/test_core.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 5f65ca0..0c511e7 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -8,6 +8,19 @@ def test_parameters(): assert INPUT.parameters assert INPUT.parameters["in"] == FILEPATH assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304) + app = pyotb.OrthoRectification(INPUT) + assert isinstance(app.parameters["map"], str) + assert app.parameters["map"] == "utm" + assert "map" in app._auto_parameters + app.set_parameters({"map": "epsg", "map.epsg.code": 2154}) + assert app.parameters["map"] == "epsg" + assert "map" in app._settings and "map" not in app._auto_parameters + assert app.parameters["map.epsg.code"] == app.app.GetParameters()["map.epsg.code"] + + +def test_param_with_underscore(): + app = pyotb.OrthoRectification(io_in=INPUT, map_epsg_code=2154) + assert app.parameters["map.epsg.code"] == 2154 def test_input_vsi(): @@ -36,11 +49,6 @@ def test_input_vsi_from_user(): pyotb.Input("/vsicurl/" + FILEPATH) -def test_param_with_underscore(): - app = pyotb.OrthoRectification(io_in=INPUT, map_epsg_code=2154) - assert app.parameters["map.epsg.code"] == 2154 - - def test_wrong_key(): with pytest.raises(KeyError): pyotb.BandMath(INPUT, expression="im1b1") @@ -203,10 +211,6 @@ def test_frozen_output_write(): assert app["out"].write("/tmp/test_frozen_app_write.tif") app["out"].filepath.unlink() -def test_parameters(): - app = pyotb.OrthoRectification(FILEPATH) - assert isinstance(app.parameters["map"], str) - assert app.parameters["map"] == "utm" def test_output_in_arg(): info = pyotb.ReadImageInfo(INPUT["out"]) -- GitLab From a7197dc945f4c7d26dbd5f67c7eb71f0122f47f8 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 1 Jul 2023 12:00:16 +0200 Subject: [PATCH 228/399] ENH: __sync_parameters skip unused choices param keys --- pyotb/core.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index ff0905b..e8ecfcc 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -521,15 +521,17 @@ class App(OTBObject): # Param keys and types self.parameters_keys = tuple(self.app.GetParametersKeys()) self._all_param_types = { - k: self.app.GetParameterType(k) for k in self.parameters_keys + key: self.app.GetParameterType(key) for key in self.parameters_keys } - types = ( - otb.ParameterType_OutputImage, - otb.ParameterType_OutputVectorData, - otb.ParameterType_OutputFilename, - ) self._out_param_types = { - k: v for k, v in self._all_param_types.items() if v in types + key: val + for key, val in self._all_param_types.items() + if val in self.OUTPUT_PARAM_TYPES + } + self._key_choices = { + key: [f"{key}.{choice}" for choice in self.app.GetChoiceKeys(key)] + for key in self.parameters_keys + if self.app.GetParameterType(key) == otb.ParameterType_Choice } # Init, execute and write (auto flush only when output param was provided) if args or kwargs: @@ -951,8 +953,20 @@ class App(OTBObject): def __sync_parameters(self): """Save OTB parameters in _settings, data and outputs dict, for a list of keys or all parameters.""" + skip = [ + k for k in self.parameters_keys if k.split(".")[-1] in ("ram", "default") + ] + # Prune unused choices child params + for key in self._key_choices: + choices = self._key_choices[key].copy() + choices.remove(f"{key}.{self.app.GetParameterValue(key)}") + skip.extend( + [k for k in self.parameters_keys if k.startswith(tuple(choices))] + ) + + self._auto_parameters.clear() for key in self.parameters_keys: - if not self.app.HasValue(key): + if key in skip or key in self._settings or not self.app.HasValue(key): continue value = self.app.GetParameterValue(key) # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken -- GitLab From 280d06e648af2a2f55ef162d134dcdd2d987258f Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 1 Jul 2023 12:01:09 +0200 Subject: [PATCH 229/399] FIX: __sync_parameters find every auto parameters --- pyotb/core.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index e8ecfcc..97bfca1 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -969,20 +969,18 @@ class App(OTBObject): if key in skip or key in self._settings or not self.app.HasValue(key): continue value = self.app.GetParameterValue(key) - # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken - if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue( - key + # Here we should use AND self.app.IsParameterEnabled(key) but it's broken + if not self.app.GetParameterRole(key) and ( + self.app.HasAutomaticValue(key) or self.app.IsParameterEnabled(key) ): - try: - value = str( - value - ) # some default str values like "mode" or "interpolator" - self._auto_parameters[key] = value - continue - except RuntimeError: - continue # grouped parameters + if isinstance(value, otb.ApplicationProxy): + try: + value = str(value) + except RuntimeError: + continue # root of param group + self._auto_parameters[key] = value # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.) - elif self.app.GetParameterRole(key) == 1 and bool(value) or value == 0: + elif self.app.GetParameterRole(key) == 1 and (bool(value) or value == 0): if isinstance(value, str): try: value = literal_eval(value) -- GitLab From 32f74e618bca40033d39c5ba2f4a1abbde6934f3 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 1 Jul 2023 12:20:54 +0200 Subject: [PATCH 230/399] ENH: sync params, and avoir calling App.app.GetParameters in property --- pyotb/core.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 97bfca1..81b56da 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -546,6 +546,8 @@ class App(OTBObject): self.execute() if any(key in self._settings for key in self._out_param_types): self.flush() + else: + self.__sync_parameters() # since not called during execute() @property def name(self) -> str: @@ -560,7 +562,7 @@ class App(OTBObject): @property def parameters(self): """Return used OTB application parameters.""" - return {**self.app.GetParameters(), **self._auto_parameters, **self._settings} + return {**self._auto_parameters, **self._settings} @property def exports_dic(self) -> dict[str, dict]: @@ -762,6 +764,8 @@ class App(OTBObject): ) self._time_start = perf_counter() self.app.ExecuteAndWriteOutput() + self.__sync_parameters() + self.frozen = False self._time_end = perf_counter() def write( -- GitLab From 1e90bc7f74c4118cb0924eafb729a0569ede0227 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 1 Jul 2023 12:21:28 +0200 Subject: [PATCH 231/399] STYLE: black autoformat + comments about types since we use enums --- pyotb/core.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 81b56da..7f38a5c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -71,10 +71,7 @@ class OTBObject(ABC): new_val = splits[1].strip() new_mdd[new_key] = new_val - return { - **new_mdd, - **imd - } + return {**new_mdd, **imd} @property def dtype(self) -> np.dtype: @@ -443,27 +440,19 @@ class App(OTBObject): """Base class that gathers common operations for any OTB application.""" INPUT_IMAGE_TYPES = [ - # Images only otb.ParameterType_InputImage, otb.ParameterType_InputImageList, ] INPUT_PARAM_TYPES = INPUT_IMAGE_TYPES + [ - # Vectors otb.ParameterType_InputVectorData, otb.ParameterType_InputVectorDataList, - # Filenames otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList, ] - OUTPUT_IMAGE_TYPES = [ - # Images only - otb.ParameterType_OutputImage - ] + OUTPUT_IMAGE_TYPES = [otb.ParameterType_OutputImage] OUTPUT_PARAM_TYPES = OUTPUT_IMAGE_TYPES + [ - # Vectors otb.ParameterType_OutputVectorData, - # Filenames otb.ParameterType_OutputFilename, ] @@ -820,6 +809,7 @@ class App(OTBObject): if ext_fname: if not isinstance(ext_fname, (dict, str)): raise ValueError("Extended filename must be a str or a dict") + def _str2dict(ext_str): """Function that converts str to dict.""" splits = [pair.split("=") for pair in ext_str.split("&")] @@ -843,10 +833,9 @@ class App(OTBObject): new_ext_fname.update(_str2dict(already_set_ext)) # transform dict to str - ext_fname_str = "&".join([ - f"{key}={value}" - for key, value in new_ext_fname.items() - ]) + ext_fname_str = "&".join( + [f"{key}={value}" for key, value in new_ext_fname.items()] + ) parameters[key] = f"{filepath}?&{ext_fname_str}" # Manage output pixel types -- GitLab From 6fd7101ac275cfd4ba4157013c673920ad1cfb9c Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 1 Jul 2023 12:27:28 +0200 Subject: [PATCH 232/399] FIX: always try convert to str if otbApplicationProxy --- pyotb/core.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 7f38a5c..afd5787 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -962,15 +962,15 @@ class App(OTBObject): if key in skip or key in self._settings or not self.app.HasValue(key): continue value = self.app.GetParameterValue(key) + if isinstance(value, otb.ApplicationProxy): + try: + value = str(value) + except RuntimeError: + continue # Here we should use AND self.app.IsParameterEnabled(key) but it's broken if not self.app.GetParameterRole(key) and ( self.app.HasAutomaticValue(key) or self.app.IsParameterEnabled(key) ): - if isinstance(value, otb.ApplicationProxy): - try: - value = str(value) - except RuntimeError: - continue # root of param group self._auto_parameters[key] = value # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.) elif self.app.GetParameterRole(key) == 1 and (bool(value) or value == 0): -- GitLab From c6374d4cd0264a1df9b18d56af0b8173291b0527 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 1 Jul 2023 12:43:54 +0200 Subject: [PATCH 233/399] FIX: skip empty collections or strings for _auto_parameters or data --- pyotb/core.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index afd5787..1738cbf 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -967,13 +967,15 @@ class App(OTBObject): value = str(value) except RuntimeError: continue + if not (bool(value) or value == 0): + continue # Here we should use AND self.app.IsParameterEnabled(key) but it's broken - if not self.app.GetParameterRole(key) and ( + if self.app.GetParameterRole(key) == 0 and ( self.app.HasAutomaticValue(key) or self.app.IsParameterEnabled(key) ): self._auto_parameters[key] = value # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.) - elif self.app.GetParameterRole(key) == 1 and (bool(value) or value == 0): + elif self.app.GetParameterRole(key) == 1: if isinstance(value, str): try: value = literal_eval(value) -- GitLab From c8ab03e3c46aa5aa0c19545daf595b4e48d9bf3e Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 1 Jul 2023 12:44:07 +0200 Subject: [PATCH 234/399] TEST: pytest verbose --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a5d80fd..3e7f69d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -83,17 +83,17 @@ codespell: test_core: extends: .tests script: - - pytest --color=yes --junitxml=test-core.xml tests/test_core.py + - pytest -vv --color=yes --junitxml=test-core.xml tests/test_core.py test_numpy: extends: .tests script: - - pytest --color=yes --junitxml=test-numpy.xml tests/test_numpy.py + - pytest -vv --color=yes --junitxml=test-numpy.xml tests/test_numpy.py test_pipeline: extends: .tests script: - - pytest --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py + - pytest -vv --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py # -------------------------------------- Docs --------------------------------------- -- GitLab From 48ee49e216c58f15fea07984900c321ed5c4088f Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 2 Jul 2023 11:58:23 +0200 Subject: [PATCH 235/399] CI: add badge for develop to README, instead of master --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5447959..d51ba17 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # pyotb: a pythonic extension of Orfeo Toolbox [](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/releases) -[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/master) +[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop) [](https://pyotb.readthedocs.io/en/master/) **pyotb** wraps the [Orfeo Toolbox](https://www.orfeo-toolbox.org/) (OTB) -- GitLab From 0fb7370181ecbe95a08df9fa5f8d007383ecf9be Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 2 Jul 2023 11:59:36 +0200 Subject: [PATCH 236/399] ENH: update comment and docstring for __sync_parameters --- pyotb/core.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 1738cbf..decd8aa 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -550,7 +550,7 @@ class App(OTBObject): @property def parameters(self): - """Return used OTB application parameters.""" + """Return used application parameters: automatic values or set by user.""" return {**self._auto_parameters, **self._settings} @property @@ -739,7 +739,7 @@ class App(OTBObject): self.frozen = False self._time_end = perf_counter() logger.debug("%s: execution ended", self.name) - self.__sync_parameters() # this is required for apps like ReadImageInfo or ComputeImagesStatistics + self.__sync_parameters() def flush(self): """Flush data to disk, this is when WriteOutput is actually called.""" @@ -945,7 +945,11 @@ class App(OTBObject): self.app.SetParameterValue(key, obj) def __sync_parameters(self): - """Save OTB parameters in _settings, data and outputs dict, for a list of keys or all parameters.""" + """Save app parameters in _auto_parameters or data dict. + + This is always called during init or after execution, to ensure the + parameters property of the App is in sync with the otb.Application instance. + """ skip = [ k for k in self.parameters_keys if k.split(".")[-1] in ("ram", "default") ] -- GitLab From 182d05946be0346c4e6493a019f5bbb03c07f41d Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 2 Jul 2023 12:13:23 +0200 Subject: [PATCH 237/399] CI: update pipeline test data with missing parameter values --- tests/serialized_apps.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/serialized_apps.json b/tests/serialized_apps.json index 424afba..938958c 100644 --- a/tests/serialized_apps.json +++ b/tests/serialized_apps.json @@ -3,6 +3,9 @@ "name": "ManageNoData", "parameters": { "mode": "buildmask", + "mode.buildmask.inv": 1.0, + "mode.buildmask.outv": 0.0, + "usenan": false, "in": { "name": "OrthoRectification", "parameters": { @@ -19,8 +22,10 @@ "outputs.lry": 5493909.5, "outputs.isotropic": true, "opt.gridspacing": 4.0, + "opt.rpc": 10, "outputs.mode": "auto", "interpolator": "bco", + "interpolator.bco.radius": 2, "io.in": { "name": "BandMath", "parameters": { @@ -54,8 +59,10 @@ "outputs.lry": 5493909.5, "outputs.isotropic": true, "opt.gridspacing": 4.0, + "opt.rpc": 10, "outputs.mode": "auto", "interpolator": "bco", + "interpolator.bco.radius": 2, "io.in": { "name": "BandMath", "parameters": { @@ -71,6 +78,9 @@ "name": "ManageNoData", "parameters": { "mode": "buildmask", + "mode.buildmask.inv": 1.0, + "mode.buildmask.outv": 0.0, + "usenan": false, "in": { "name": "OrthoRectification", "parameters": { @@ -87,8 +97,10 @@ "outputs.lry": 5493909.5, "outputs.isotropic": true, "opt.gridspacing": 4.0, + "opt.rpc": 10, "outputs.mode": "auto", "interpolator": "bco", + "interpolator.bco.radius": 2, "io.in": { "name": "BandMath", "parameters": { -- GitLab From 576305c8715bb6db69051684e01b8b138bd0731e Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 2 Jul 2023 12:42:22 +0200 Subject: [PATCH 238/399] FIX: sync_parameters make sure to keep False bool param values --- pyotb/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index decd8aa..94da09b 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -971,7 +971,8 @@ class App(OTBObject): value = str(value) except RuntimeError: continue - if not (bool(value) or value == 0): + # Keep False or 0 values, but make sure to skip empty collections or str + if hasattr(value, "__iter__") and not value: continue # Here we should use AND self.app.IsParameterEnabled(key) but it's broken if self.app.GetParameterRole(key) == 0 and ( -- GitLab From c99f0131dcf6d8b5dec3127e968b44e7257ad1ab Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Mon, 3 Jul 2023 09:42:01 +0200 Subject: [PATCH 239/399] ADD: test for autoparameters --- tests/test_core.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index 0c511e7..2e07b96 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -360,6 +360,36 @@ def test_summarize_strip_output(): assert summary["parameters"][key] == expected, \ f"Failed for input {inp}, output {out}, args {extra_args}" +def test_summarize_consistency(): + app_fns = [ + lambda inp: pyotb.ExtractROI( + {"in": inp, "startx": 10, "starty": 10, "sizex": 50, "sizey": 50} + ), + lambda inp: pyotb.ManageNoData({"in": inp, "mode": "changevalue"}), + lambda inp: pyotb.DynamicConvert({"in": inp}), + lambda inp: pyotb.Mosaic({"il": [inp]}), + lambda inp: pyotb.BandMath({"il": [inp], "exp": "im1b1 + 1"}), + lambda inp: pyotb.BandMathX({"il": [inp], "exp": "im1"}), + lambda inp: pyotb.OrthoRectification({"io.in": inp}), + ] + def _test(app_fn): + """ + Here we create 2 summaries: + - summary of the app before write() + - summary of the app after write() + Then we check that both only differ with the output parameter + """ + app = app_fn(inp=FILEPATH) + out_file = "/dev/shm/out.tif" + out_key = app.output_image_key + summary_wo_wrt = pyotb.summarize(app) + app.write(out_file) + summay_w_wrt = pyotb.summarize(app) + app[out_key].filepath.unlink() + summary_wo_wrt["parameters"].update({out_key: out_file}) + assert summary_wo_wrt == summay_w_wrt + for app_fn in app_fns: + _test(app_fn) def test_pipeline_simple(): # BandMath -> OrthoRectification -> ManageNoData -- GitLab From 45467540327f5c4985c02626dd88a3008e4c3a6d Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 5 Jul 2023 22:20:00 +0200 Subject: [PATCH 240/399] WIP: depreciation --- pyotb/__init__.py | 12 ++++- pyotb/core.py | 65 ++++++++++++++++++++++++ pyotb/depreciation.py | 114 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 pyotb/depreciation.py diff --git a/pyotb/__init__.py b/pyotb/__init__.py index 3e09a63..e6d3a97 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -4,7 +4,17 @@ __version__ = "2.0.0.dev4" from .helpers import logger, set_logger_level from .apps import * -from .core import App, Input, Output, get_nbchannels, get_pixel_type, summarize +from .core import ( + App, + Input, + Output, + get_nbchannels, + get_pixel_type, + summarize, + OTBObject, + otbObject, +) + from .functions import ( # pylint: disable=redefined-builtin all, any, diff --git a/pyotb/core.py b/pyotb/core.py index 94da09b..08c9fb5 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -11,7 +11,9 @@ from typing import Any import numpy as np import otbApplication as otb # pylint: disable=import-error +import pyotb from .helpers import logger +from .depreciation import deprecated_alias, warning_msg, deprecated_attr class OTBObject(ABC): @@ -32,6 +34,14 @@ class OTBObject(ABC): def output_image_key(self) -> str: """Return the name of a parameter key associated to the main output image of the object.""" + @property + @deprecated_attr(replacement="output_image_key") + def output_param(self) -> str: + """ + Return the name of a parameter key associated to the main output image + of the object (deprecated). + """ + @property @abstractmethod def exports_dic(self) -> dict[str, dict]: @@ -109,6 +119,21 @@ class OTBObject(ABC): origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2 return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y + + def summarize(self, *args, **kwargs): + """Summarize the app with `pyotb.summarize()`. + + Args: + *args: args + **kwargs: keyword args + + Returns: + app summary + + """ + return pyotb.summarize(self, *args, **kwargs) + + def get_info(self) -> dict[str, (str, float, list[float])]: """Return a dict output of ReadImageInfo for the first image output.""" return App("ReadImageInfo", self, quiet=True).data @@ -393,6 +418,30 @@ class OTBObject(ABC): """ return id(self) + def __getattr__(self, item: str): + """ + Provides depreciation of old methods to access the OTB application + values. + ``` + + Args: + item: attribute name + + Returns: + attribute from self.app + + """ + getattr(self.app, item) + raise DeprecationWarning( + "Since pyotb 2.0.0, OTBObject instances have stopped to " + "forward attributes to their own internal otbApplication " + "instance. `App.app` can be used to call otbApplications " + f"methods. Attribute was: \"{item}\". Hint: maybe try " + f"`pyotb_app.app.{item}` instead of `pyotb_app.{item}`?" + ) + + + def __getitem__(self, key) -> Any | list[float] | float | Slicer: """Override the default __getitem__ behaviour. @@ -436,6 +485,14 @@ class OTBObject(ABC): return f"<pyotb.{self.__class__.__name__} object, id {id(self)}>" +class otbObject(OTBObject): # pylint: noqa + def __init_subclass__(cls): + warning_msg( + "Since pyotb 2.0.0, otbObject has been renamed OTBObject. " + "otbObject will be removed definitively in future releases." + ) + + class App(OTBObject): """Base class that gathers common operations for any OTB application.""" @@ -757,6 +814,7 @@ class App(OTBObject): self.frozen = False self._time_end = perf_counter() + @deprecated_alias(filename_extension="ext_fname") def write( self, path: str | Path | dict[str, str] = None, @@ -1431,6 +1489,8 @@ class Output(OTBObject): _filepath: str | Path = None + + @deprecated_alias(app="pyotb_app", output_parameter_key="param_key") def __init__( self, pyotb_app: App, @@ -1463,6 +1523,11 @@ class Output(OTBObject): """Reference to the parent pyotb otb.Application instance.""" return self.parent_pyotb_app.app + @property + @deprecated_attr(replacement="parent_pyotb_app") + def pyotb_app(self) -> App: + """Reference to the parent pyotb App (deprecated).""" + @property def exports_dic(self) -> dict[str, dict]: """Returns internal _exports_dic object that contains numpy array exports.""" diff --git a/pyotb/depreciation.py b/pyotb/depreciation.py new file mode 100644 index 0000000..87d1d77 --- /dev/null +++ b/pyotb/depreciation.py @@ -0,0 +1,114 @@ +"""Helps with deprecated classes and methods. + +Taken from https://stackoverflow.com/questions/49802412/how-to-implement- +deprecation-in-python-with-argument-alias +""" +from typing import Callable, Dict, Any +import functools +import warnings + + +WARN = '\033[91m' +ENDC = '\033[0m' +OKAY = '\033[92m' + +def warning_msg(message: str): + """ + Shows a warning message. + + Args: + message: message + + """ + warnings.warn( + message=message, + category=DeprecationWarning, + stacklevel=3, + ) +def deprecated_alias(**aliases: str) -> Callable: + """ + Decorator for deprecated function and method arguments. + + Use as follows: + + @deprecated_alias(old_arg='new_arg') + def myfunc(new_arg): + ... + + Args: + **aliases: aliases + + Returns: + wrapped function + + """ + def deco(f: Callable): + @functools.wraps(f) + def wrapper(*args, **kwargs): + rename_kwargs(f.__name__, kwargs, aliases) + return f(*args, **kwargs) + + return wrapper + + return deco + + +def rename_kwargs( + func_name: str, + kwargs: Dict[str, Any], + aliases: Dict[str, str] +): + """ + Helper function for deprecating function arguments. + + Args: + func_name: function + kwargs: keyword args + aliases: aliases + + """ + for alias, new in aliases.items(): + if alias in kwargs: + if new in kwargs: + raise TypeError( + f"{func_name} received both {alias} and {new} as arguments!" + f" {alias} is deprecated, use {new} instead." + ) + message = ( + f"{WARN}`{alias}`{ENDC} is deprecated as an argument to " + f"`{func_name}`; use {OKAY}`{new}`{ENDC} instead." + ) + warning_msg(message) + kwargs[new] = kwargs.pop(alias) + +def deprecated_attr(replacement: str) -> Callable: + """ + Decorator for deprecated attr. + + Use as follows: + + @deprecated_attr(replacement='new_attr') + def old_attr(...): + ... + + Args: + replacement: name of the new attr (method or attribute) + + Returns: + wrapped function + + """ + def deco(attr: Any): + @functools.wraps(attr) + def wrapper(self, *args, **kwargs): + warning_msg( + f"{WARN}`{attr.__name__}`{ENDC} will be removed in future " + f"releases. Please replace {WARN}`{attr.__name__}`{ENDC} with " + f"{OKAY}`{replacement}`{ENDC}." + ) + g = getattr(self, replacement) + return g(*args, **kwargs) if isinstance(g, Callable) else g + + return wrapper + + return deco -- GitLab From 09920054c29588786a7a0b6db0b5bc55b775118f Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 5 Jul 2023 22:31:23 +0200 Subject: [PATCH 241/399] WIP: depreciation --- pyotb/core.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 08c9fb5..298389d 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -11,7 +11,6 @@ from typing import Any import numpy as np import otbApplication as otb # pylint: disable=import-error -import pyotb from .helpers import logger from .depreciation import deprecated_alias, warning_msg, deprecated_attr @@ -124,14 +123,14 @@ class OTBObject(ABC): """Summarize the app with `pyotb.summarize()`. Args: - *args: args - **kwargs: keyword args + *args: args for `pyotb.summarize()` + **kwargs: keyword args for `pyotb.summarize()` Returns: - app summary + app summary, same as `pyotb.summarize()` """ - return pyotb.summarize(self, *args, **kwargs) + return summarize(self, *args, **kwargs) def get_info(self) -> dict[str, (str, float, list[float])]: -- GitLab From 7ef93cd298da8d26280ffaf3ea710641cbeb9c70 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 5 Jul 2023 22:42:13 +0200 Subject: [PATCH 242/399] WIP: depreciation --- pyotb/core.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 298389d..9ceab97 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -431,12 +431,19 @@ class OTBObject(ABC): """ getattr(self.app, item) + note = "" + if item.startswith("GetParameter"): + note = ( + "Note: `pyotb_app.app.GetParameterValue('paramname')` can be " + "shorten with `pyotb_app['paramname']` to access parameters " + "values." + ) raise DeprecationWarning( "Since pyotb 2.0.0, OTBObject instances have stopped to " "forward attributes to their own internal otbApplication " "instance. `App.app` can be used to call otbApplications " f"methods. Attribute was: \"{item}\". Hint: maybe try " - f"`pyotb_app.app.{item}` instead of `pyotb_app.{item}`?" + f"`pyotb_app.app.{item}` instead of `pyotb_app.{item}`? {note}" ) -- GitLab From 64d2dc7d83f7dcdd964ee760b83f629e808f6d0c Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 6 Jul 2023 11:07:10 +0200 Subject: [PATCH 243/399] WIP: depreciation --- pyotb/core.py | 57 ++++++++++++++++++++++++++----------------- pyotb/depreciation.py | 12 +++------ 2 files changed, 39 insertions(+), 30 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 9ceab97..3a284bc 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -36,10 +36,7 @@ class OTBObject(ABC): @property @deprecated_attr(replacement="output_image_key") def output_param(self) -> str: - """ - Return the name of a parameter key associated to the main output image - of the object (deprecated). - """ + """Return the name of a parameter key associated to the main output image of the object (deprecated).""" @property @abstractmethod @@ -120,7 +117,7 @@ class OTBObject(ABC): def summarize(self, *args, **kwargs): - """Summarize the app with `pyotb.summarize()`. + """Call `pyotb.summarize()` on itself. Args: *args: args for `pyotb.summarize()` @@ -418,33 +415,47 @@ class OTBObject(ABC): return id(self) def __getattr__(self, item: str): - """ - Provides depreciation of old methods to access the OTB application - values. - ``` + """Provides depreciation of old methods to access the OTB application values. + + This function will be removed completely in future releases. Args: item: attribute name - Returns: - attribute from self.app """ - getattr(self.app, item) - note = "" + note = ( + "Since pyotb 2.0.0, OTBObject instances have stopped to forward " + "attributes to their own internal otbApplication instance. " + "`App.app` can be used to call otbApplications methods. " + ) + + if item[0].isupper(): + # Because otbApplication instances methods names start with an + # upper case + note += ( + f"Maybe try `pyotb_app.app.{item}` instead of " + f"`pyotb_app.{item}`? " + ) + + if item[0].islower(): + # Because in pyotb 1.5.4, applications outputs were added as + # attributes of the instance + note += ( + "Note: `pyotb_app.paramname` is no longer supported. Starting " + "from pytob 2.0.0, `pyotb_app['paramname']` can be used to " + "access parameters values. " + ) + if item.startswith("GetParameter"): - note = ( + note += ( "Note: `pyotb_app.app.GetParameterValue('paramname')` can be " "shorten with `pyotb_app['paramname']` to access parameters " "values." ) - raise DeprecationWarning( - "Since pyotb 2.0.0, OTBObject instances have stopped to " - "forward attributes to their own internal otbApplication " - "instance. `App.app` can be used to call otbApplications " - f"methods. Attribute was: \"{item}\". Hint: maybe try " - f"`pyotb_app.app.{item}` instead of `pyotb_app.{item}`? {note}" - ) + warning_msg(note) + raise AttributeError + @@ -491,8 +502,10 @@ class OTBObject(ABC): return f"<pyotb.{self.__class__.__name__} object, id {id(self)}>" -class otbObject(OTBObject): # pylint: noqa +class otbObject(OTBObject): # noqa + """Class for depreciation of otbObject since pyotb 2.0.0. Will be removed in future releases.""" def __init_subclass__(cls): + """Show a warning for depreciation.""" warning_msg( "Since pyotb 2.0.0, otbObject has been renamed OTBObject. " "otbObject will be removed definitively in future releases." diff --git a/pyotb/depreciation.py b/pyotb/depreciation.py index 87d1d77..213aa56 100644 --- a/pyotb/depreciation.py +++ b/pyotb/depreciation.py @@ -13,8 +13,7 @@ ENDC = '\033[0m' OKAY = '\033[92m' def warning_msg(message: str): - """ - Shows a warning message. + """Shows a warning message. Args: message: message @@ -26,8 +25,7 @@ def warning_msg(message: str): stacklevel=3, ) def deprecated_alias(**aliases: str) -> Callable: - """ - Decorator for deprecated function and method arguments. + """Decorator for deprecated function and method arguments. Use as follows: @@ -58,8 +56,7 @@ def rename_kwargs( kwargs: Dict[str, Any], aliases: Dict[str, str] ): - """ - Helper function for deprecating function arguments. + """Helper function for deprecating function arguments. Args: func_name: function @@ -82,8 +79,7 @@ def rename_kwargs( kwargs[new] = kwargs.pop(alias) def deprecated_attr(replacement: str) -> Callable: - """ - Decorator for deprecated attr. + """Decorator for deprecated attr. Use as follows: -- GitLab From 240ce351b146b7b7ee0cfd43415a95ccafaff22c Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 6 Jul 2023 11:15:14 +0200 Subject: [PATCH 244/399] WIP: depreciation --- pyotb/core.py | 4 ++-- pyotb/depreciation.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 3a284bc..e35c564 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -117,7 +117,7 @@ class OTBObject(ABC): def summarize(self, *args, **kwargs): - """Call `pyotb.summarize()` on itself. + """Recursively summarize parameters and parents. Args: *args: args for `pyotb.summarize()` @@ -502,7 +502,7 @@ class OTBObject(ABC): return f"<pyotb.{self.__class__.__name__} object, id {id(self)}>" -class otbObject(OTBObject): # noqa +class otbObject(OTBObject): # noqa, pylint: disable=invalid-name """Class for depreciation of otbObject since pyotb 2.0.0. Will be removed in future releases.""" def __init_subclass__(cls): """Show a warning for depreciation.""" diff --git a/pyotb/depreciation.py b/pyotb/depreciation.py index 213aa56..0234249 100644 --- a/pyotb/depreciation.py +++ b/pyotb/depreciation.py @@ -40,11 +40,11 @@ def deprecated_alias(**aliases: str) -> Callable: wrapped function """ - def deco(f: Callable): - @functools.wraps(f) + def deco(func: Callable): + @functools.wraps(func) def wrapper(*args, **kwargs): - rename_kwargs(f.__name__, kwargs, aliases) - return f(*args, **kwargs) + rename_kwargs(func.__name__, kwargs, aliases) + return func(*args, **kwargs) return wrapper @@ -102,8 +102,8 @@ def deprecated_attr(replacement: str) -> Callable: f"releases. Please replace {WARN}`{attr.__name__}`{ENDC} with " f"{OKAY}`{replacement}`{ENDC}." ) - g = getattr(self, replacement) - return g(*args, **kwargs) if isinstance(g, Callable) else g + out = getattr(self, replacement) + return out(*args, **kwargs) if isinstance(out, Callable) else out return wrapper -- GitLab From 82c93bdb6e8ff537460ad123f1feb7338520afff Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 6 Jul 2023 11:58:06 +0200 Subject: [PATCH 245/399] REFAC: rename warning_msg --> depreciation_warning --- pyotb/__init__.py | 11 +---------- pyotb/core.py | 6 +++--- pyotb/depreciation.py | 6 +++--- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/pyotb/__init__.py b/pyotb/__init__.py index e6d3a97..62508f5 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -4,16 +4,7 @@ __version__ = "2.0.0.dev4" from .helpers import logger, set_logger_level from .apps import * -from .core import ( - App, - Input, - Output, - get_nbchannels, - get_pixel_type, - summarize, - OTBObject, - otbObject, -) +from .core import App, Input, Output, get_nbchannels, get_pixel_type, summarize, OTBObject, otbObject from .functions import ( # pylint: disable=redefined-builtin all, diff --git a/pyotb/core.py b/pyotb/core.py index e35c564..b7ee57a 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -12,7 +12,7 @@ import numpy as np import otbApplication as otb # pylint: disable=import-error from .helpers import logger -from .depreciation import deprecated_alias, warning_msg, deprecated_attr +from .depreciation import deprecated_alias, depreciation_warning, deprecated_attr class OTBObject(ABC): @@ -453,7 +453,7 @@ class OTBObject(ABC): "shorten with `pyotb_app['paramname']` to access parameters " "values." ) - warning_msg(note) + depreciation_warning(note) raise AttributeError @@ -506,7 +506,7 @@ class otbObject(OTBObject): # noqa, pylint: disable=invalid-name """Class for depreciation of otbObject since pyotb 2.0.0. Will be removed in future releases.""" def __init_subclass__(cls): """Show a warning for depreciation.""" - warning_msg( + depreciation_warning( "Since pyotb 2.0.0, otbObject has been renamed OTBObject. " "otbObject will be removed definitively in future releases." ) diff --git a/pyotb/depreciation.py b/pyotb/depreciation.py index 0234249..859ba60 100644 --- a/pyotb/depreciation.py +++ b/pyotb/depreciation.py @@ -12,7 +12,7 @@ WARN = '\033[91m' ENDC = '\033[0m' OKAY = '\033[92m' -def warning_msg(message: str): +def depreciation_warning(message: str): """Shows a warning message. Args: @@ -75,7 +75,7 @@ def rename_kwargs( f"{WARN}`{alias}`{ENDC} is deprecated as an argument to " f"`{func_name}`; use {OKAY}`{new}`{ENDC} instead." ) - warning_msg(message) + depreciation_warning(message) kwargs[new] = kwargs.pop(alias) def deprecated_attr(replacement: str) -> Callable: @@ -97,7 +97,7 @@ def deprecated_attr(replacement: str) -> Callable: def deco(attr: Any): @functools.wraps(attr) def wrapper(self, *args, **kwargs): - warning_msg( + depreciation_warning( f"{WARN}`{attr.__name__}`{ENDC} will be removed in future " f"releases. Please replace {WARN}`{attr.__name__}`{ENDC} with " f"{OKAY}`{replacement}`{ENDC}." -- GitLab From 2a8c3414e37512ac0aadd2e82b1ff2c8331787db Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 6 Jul 2023 12:53:49 +0200 Subject: [PATCH 246/399] DOC: correct typing of ext_fname --- pyotb/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 94da09b..405e736 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -762,7 +762,7 @@ class App(OTBObject): path: str | Path | dict[str, str] = None, pixel_type: dict[str, str] | str = None, preserve_dtype: bool = False, - ext_fname: str = "", + ext_fname: dict[str, str] | str = None, **kwargs, ) -> bool: """Set output pixel type and write the output raster files. @@ -773,7 +773,7 @@ class App(OTBObject): non-standard characters such as a point, e.g. {'io.out':'output.tif'} - None if output file was passed during App init ext_fname: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES") - Will be used for all outputs (Default value = "") + Will be used for all outputs (Default value = None) pixel_type: Can be : - dictionary {out_param_key: pixeltype} when specifying for several outputs - str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several outputs, all outputs are written with this unique type. -- GitLab From 13cd7bbc1128907bbd53e10de51fe1e126ccd10e Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 6 Jul 2023 12:53:59 +0200 Subject: [PATCH 247/399] DOC: fix missing link --- doc/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.md b/doc/index.md index 448d139..7861878 100644 --- a/doc/index.md +++ b/doc/index.md @@ -11,7 +11,7 @@ to make OTB more Python friendly. - [Installation](installation.md) - [How to use pyotb](quickstart.md) - [Useful features](features.md) -- [Functions](features.md) +- [Functions](functions.md) - [Interaction with Python libraries (numpy, rasterio, tensorflow)](interaction.md) ## Examples -- GitLab From d21a0d6cd92fc80e9e87b80f205381932abb93b3 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 6 Jul 2023 12:54:14 +0200 Subject: [PATCH 248/399] DOC: enhance quickstart --- doc/quickstart.md | 118 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 89 insertions(+), 29 deletions(-) diff --git a/doc/quickstart.md b/doc/quickstart.md index 5e07e43..fa26d45 100644 --- a/doc/quickstart.md +++ b/doc/quickstart.md @@ -15,12 +15,11 @@ resampled = pyotb.RigidTransformResample({ }) ``` -Note that pyotb has a 'lazy' evaluation: it only performs operation when it is -needed, i.e. results are written to disk. -Thus, the previous line doesn't trigger the application. +For now, `resampled` has not been executed. Indeed, pyotb has a 'lazy' +evaluation: applications are executed only when required. Generally, like in +this example, executions happen to write output images to disk. -To actually trigger the application execution, you need to write the result to -disk: +To actually trigger the application execution, `write()` has to be called: ```python resampled.write('output.tif') # this is when the application actually runs @@ -28,32 +27,39 @@ resampled.write('output.tif') # this is when the application actually runs ### Using Python keyword arguments -It is also possible to use the Python keyword arguments notation for passing -the parameters: +One can use the Python keyword arguments notation for passing that parameters: ```python output = pyotb.SuperImpose(inr='reference_image.tif', inm='image.tif') ``` -is equivalent to: +Which is equivalent to: ```python output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'}) ``` -Limitations : for this notation, python doesn't accept the parameter `in` or -any parameter that contains a dots (e.g. `io.in)`. -For `in` and other main input parameters of an OTB app, you may simply pass -the value as first argument, pyotb will guess the parameter name. -For parameters that contains dots, you can either use a dictionary, or replace dots (`.`) with underscores (`_`) as follow : +!!! warning -```python -resampled = pyotb.RigidTransformResample( - 'my_image.tif', - interpolator = 'linear', - transform_type_id_scaley = 0.5, - transform_type_id_scalex = 0.5 -) + For this notation, python doesn't accept the parameter `in` or any + parameter that contains a dots (e.g. `io.in)`. For `in` or other main + input parameters of an OTB application, you may simply pass the value as + first argument, pyotb will guess the parameter name. For parameters that + contains dots, you can either use a dictionary, or replace dots (`.`) + with underscores (`_`). Let's take the example of the `OrthoRectification` + application of OTB, with the input image parameter named "io.in": + + Option #1, keyword-arg-free: + + ```python + ortho = pyotb.OrthoRectification('my_image.tif') + ``` + + Option #2, replacing dots with underscores in parameter name: + + ```python + ortho = pyotb.OrthoRectification(io_in='my_image.tif') + ``` ## In-memory connections @@ -93,26 +99,38 @@ dilated.write('result.tif') ## Writing the result of an app -Any pyotb object can be written to disk using the `write` method, e.g. : +Any pyotb object can be written to disk using `write()`. + +Let's consider the following pyotb application instance: ```python import pyotb - resampled = pyotb.RigidTransformResample({ 'in': 'my_image.tif', 'interpolator': 'linear', 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5 }) +``` -# Here you can set optionally pixel type and extended filename variables -resampled.write( - {'out': 'output.tif'}, - pixel_type='uint16', - ext_fname='?nodata=65535' -) +We can then write the output of `resampled` as following: + +```python +resampled.write('output.tif') ``` +!!! note + + For applications that have multiple outputs, passing a `dict` of filenames + can be considered. Let's take the example of `MeanShiftSmoothing` which + has 2 output images: + + ```python + import pyotb + meanshift = pyotb.MeanShiftSmoothing('my_image.tif') + meanshift.write({'fout': 'output_1.tif', 'foutpos': 'output_2.tif'}) + ``` + Another possibility for writing results is to set the output parameter when initializing the application: @@ -126,4 +144,46 @@ resampled = pyotb.RigidTransformResample({ 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5 }) -``` \ No newline at end of file +``` + +### Pixel type + +Setting the pixel type is optional, and can be achieved setting the +`pixel_type` argument: + +```python +resampled.write('output.tif', pixel_type='uint16') +``` + +The value of `pixel_type` corresponds to the name of a pixel type from OTB +applications (e.g. `'uint8'`, `'float'`, etc). + +### Extended filenames + +Extended filenames can be passed as `str` or `dict`. + +As `str`: + +```python +resampled.write( + ... + ext_fname='nodata=65535&box=0:0:256:256' +) +``` + +As `dict`: + +```python +resampled.write( + ... + ext_fname={'nodata': '65535', 'box': '0:0:256:256'} +) +``` + +!!! info + + When `ext_fname` is provided and the output filenames contain already some + extended filename pattern, the ones provided in the filenames take + priority over the ones passed in `ext_fname`. This allows to fine-grained + tune extended filenames for each output, with a common extended filenames + keys/values basis. -- GitLab From 9dc224ea83cf79c5a213bc68e60f4ffe8dd0f193 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 6 Jul 2023 12:59:16 +0200 Subject: [PATCH 249/399] DOC: edit limitation description --- doc/interaction.md | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/doc/interaction.md b/doc/interaction.md index e4c73fb..a87da8e 100644 --- a/doc/interaction.md +++ b/doc/interaction.md @@ -138,10 +138,6 @@ memory !!! warning - Limitations : - - - For OTBTF versions < 4.0.0, it is not possible to use the tensorflow - python API inside a script where OTBTF is used because of libraries - clashing between Tensorflow and OTBTF, i.e. `import tensorflow` doesn't - work in a script where OTBTF apps have been initialized. This is why we - recommend to use latest OTBTF versions + Due to compilation issues in OTBTF before version 4.0.0, tensorflow and + pyotb can't be imported in the same python code. This problem has been + fixed in OTBTF 4.0.0. -- GitLab From e8d99fa004ba994daf88c11d5c0ccfab1169b184 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 6 Jul 2023 13:01:16 +0200 Subject: [PATCH 250/399] DOC: edit limitation description --- doc/interaction.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/interaction.md b/doc/interaction.md index a87da8e..0195b70 100644 --- a/doc/interaction.md +++ b/doc/interaction.md @@ -43,11 +43,9 @@ noisy_image.write('image_plus_noise.tif') !!! warning - Limitations : - - The whole image is loaded into memory - - The georeference can not be modified. Thus, numpy operations can not change - the image or pixel size + - The georeference can not be modified. Thus, numpy operations can not + change the image or pixel size ## Export to rasterio -- GitLab From 29792c3bb3daa389091ad739dc2a263d034d36a7 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 6 Jul 2023 14:30:07 +0200 Subject: [PATCH 251/399] DOC: use otb stylesheet.css --- doc/stylesheets/extra.css | 4 ---- mkdocs.yml | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) delete mode 100644 doc/stylesheets/extra.css diff --git a/doc/stylesheets/extra.css b/doc/stylesheets/extra.css deleted file mode 100644 index c67f70f..0000000 --- a/doc/stylesheets/extra.css +++ /dev/null @@ -1,4 +0,0 @@ -/* this is for readthedocs theme */ -.wy-nav-content { - max-width: 1000px; -} \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 29d51d4..d196b27 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -55,7 +55,7 @@ extra: - icon: fontawesome/brands/gitlab link: https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb extra_css: - - stylesheets/extra.css + - https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/8.1.2-rc1/Documentation/Cookbook/_static/css/otb_theme.css use_directory_urls: false # this creates some pyotb/core.html pages instead of pyotb/core/index.html markdown_extensions: -- GitLab From f226f2f232d818326b5200e4f7cca15ab3a3e100 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 6 Jul 2023 14:30:24 +0200 Subject: [PATCH 252/399] DOC: nipticks --- doc/quickstart.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/quickstart.md b/doc/quickstart.md index fa26d45..8343520 100644 --- a/doc/quickstart.md +++ b/doc/quickstart.md @@ -42,12 +42,12 @@ output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'}) !!! warning For this notation, python doesn't accept the parameter `in` or any - parameter that contains a dots (e.g. `io.in)`. For `in` or other main + parameter that contains a dots (e.g. `io.in`). For `in` or other main input parameters of an OTB application, you may simply pass the value as first argument, pyotb will guess the parameter name. For parameters that contains dots, you can either use a dictionary, or replace dots (`.`) with underscores (`_`). Let's take the example of the `OrthoRectification` - application of OTB, with the input image parameter named "io.in": + application of OTB, with the input image parameter named `io.in`: Option #1, keyword-arg-free: -- GitLab From 51cc690ac7b3d62863dbafd51f513be7fbc41553 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 6 Jul 2023 14:55:11 +0200 Subject: [PATCH 253/399] DOC: add codehilite --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index d196b27..273d121 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -59,6 +59,7 @@ extra_css: use_directory_urls: false # this creates some pyotb/core.html pages instead of pyotb/core/index.html markdown_extensions: + - codehilite - admonition - toc: permalink: true -- GitLab From 8dcc384b0ecb008d990233845855ae6eda9a499a Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 6 Jul 2023 14:55:20 +0200 Subject: [PATCH 254/399] DOC: enh --- doc/quickstart.md | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/doc/quickstart.md b/doc/quickstart.md index 8343520..e65c68f 100644 --- a/doc/quickstart.md +++ b/doc/quickstart.md @@ -46,8 +46,10 @@ output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'}) input parameters of an OTB application, you may simply pass the value as first argument, pyotb will guess the parameter name. For parameters that contains dots, you can either use a dictionary, or replace dots (`.`) - with underscores (`_`). Let's take the example of the `OrthoRectification` - application of OTB, with the input image parameter named `io.in`: + with underscores (`_`). + + Let's take the example of the `OrthoRectification` application of OTB, + with the input image parameter named `io.in`: Option #1, keyword-arg-free: @@ -63,13 +65,19 @@ output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'}) ## In-memory connections -The big asset of pyotb is the ease of in-memory connections between apps. +One nice feature of pyotb is in-memory connection between apps. It relies on +the so-called [streaming](https://www.orfeo-toolbox.org/CookBook/C++/StreamingAndThreading.html) +mechanism of OTB, that enables to process huge images with a limited memory +footprint. -Let's start from our previous example. Consider the case where one wants to -apply optical calibration and binary morphological dilatation -following the undersampling. +pyotb allows to pass any application's output to another. This enables to +build pipelines composed of several applications. -Using pyotb, you can pass the output of an app as input of another app : +Let's start from our previous example. Consider the case where one wants to +resample the image, then apply optical calibration and binary morphological +dilatation. We can write the following code to build a pipeline that will +generate the output in an end-to-end fashion, without being limited with the +input image size, without writing temporary files. ```python import pyotb @@ -91,12 +99,21 @@ dilated = pyotb.BinaryMorphologicalOperation({ 'out': 'output.tif', 'filter': 'dilate', 'structype': 'ball', - 'xradius': 3, 'yradius': 3 + 'xradius': 3, + 'yradius': 3 }) +``` -dilated.write('result.tif') +We just have built our first pipeline! At this point, it's all symbolic since +no computation has been performed. To trigger our pipeline, one must call the +`write()` method from the pipeline termination: + +```python +dilated.write('output.tif') ``` +In the next section, we will detail how `write()` works. + ## Writing the result of an app Any pyotb object can be written to disk using `write()`. -- GitLab From 356600626849a294970e52daca988c3a6171fb7a Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 6 Jul 2023 15:03:24 +0200 Subject: [PATCH 255/399] DOC: add codehilite --- mkdocs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mkdocs.yml b/mkdocs.yml index 273d121..e80cfac 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -71,6 +71,8 @@ markdown_extensions: - pymdownx.snippets - pymdownx.details - pymdownx.superfences + - pymdownx.highlight: + use_pygments: true # Rest of the navigation.. site_name: "pyotb documentation: a Python extension of OTB" -- GitLab From 58e1789e2ab613cce9f0099f0cdf829d929b4d7f Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 6 Jul 2023 15:03:33 +0200 Subject: [PATCH 256/399] DOC: add codehilite --- doc/doc_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/doc_requirements.txt b/doc/doc_requirements.txt index 4f59692..931c3df 100644 --- a/doc/doc_requirements.txt +++ b/doc/doc_requirements.txt @@ -5,3 +5,4 @@ mkdocs-gen-files mkdocs-section-index mkdocs-literate-nav mkdocs-mermaid2-plugin +pygments -- GitLab From e4d9a3e42092cd6d004f80ec4412c6037799d6f8 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 7 Jul 2023 07:20:16 +0000 Subject: [PATCH 257/399] Apply 1 suggestion(s) to 1 file(s) --- pyotb/core.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index b7ee57a..ab6b620 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -430,34 +430,30 @@ class OTBObject(ABC): "`App.app` can be used to call otbApplications methods. " ) - if item[0].isupper(): + if item in dir(self.app): # Because otbApplication instances methods names start with an # upper case note += ( f"Maybe try `pyotb_app.app.{item}` instead of " f"`pyotb_app.{item}`? " ) + if item.startswith("GetParameter"): + note += ( + "Note: `pyotb_app.app.GetParameterValue('paramname')` can be " + "shorten with `pyotb_app['paramname']` to access parameters " + "values." + ) - if item[0].islower(): + elif item in self.parameters_keys: # Because in pyotb 1.5.4, applications outputs were added as # attributes of the instance note += ( "Note: `pyotb_app.paramname` is no longer supported. Starting " - "from pytob 2.0.0, `pyotb_app['paramname']` can be used to " + "from pyotb 2.0.0, `pyotb_app['paramname']` can be used to " "access parameters values. " ) - - if item.startswith("GetParameter"): - note += ( - "Note: `pyotb_app.app.GetParameterValue('paramname')` can be " - "shorten with `pyotb_app['paramname']` to access parameters " - "values." - ) depreciation_warning(note) - raise AttributeError - - - + raise AttributeError(f"'{type(self)}' object has no attribute '{item}'") def __getitem__(self, key) -> Any | list[float] | float | Slicer: """Override the default __getitem__ behaviour. -- GitLab From 0e49fec8ce3c6f61c91e4f2713c58d32f793ad43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr> Date: Fri, 7 Jul 2023 07:23:37 +0000 Subject: [PATCH 258/399] Apply 1 suggestion(s) to 1 file(s) --- pyotb/core.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index ab6b620..573929a 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -427,18 +427,19 @@ class OTBObject(ABC): note = ( "Since pyotb 2.0.0, OTBObject instances have stopped to forward " "attributes to their own internal otbApplication instance. " - "`App.app` can be used to call otbApplications methods. " + "`App.app` can be used to call otbApplications methods." ) + hint = None if item in dir(self.app): # Because otbApplication instances methods names start with an # upper case - note += ( + hint = ( f"Maybe try `pyotb_app.app.{item}` instead of " f"`pyotb_app.{item}`? " ) if item.startswith("GetParameter"): - note += ( + hint += ( "Note: `pyotb_app.app.GetParameterValue('paramname')` can be " "shorten with `pyotb_app['paramname']` to access parameters " "values." @@ -447,13 +448,14 @@ class OTBObject(ABC): elif item in self.parameters_keys: # Because in pyotb 1.5.4, applications outputs were added as # attributes of the instance - note += ( + hint = ( "Note: `pyotb_app.paramname` is no longer supported. Starting " "from pyotb 2.0.0, `pyotb_app['paramname']` can be used to " "access parameters values. " ) - depreciation_warning(note) - raise AttributeError(f"'{type(self)}' object has no attribute '{item}'") + if hint: + depreciation_warning(f"{note} {hint}") + raise AttributeError def __getitem__(self, key) -> Any | list[float] | float | Slicer: """Override the default __getitem__ behaviour. -- GitLab From 0956d334b86e93471ec52596b73d03e1e05e815a Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 7 Jul 2023 07:26:15 +0000 Subject: [PATCH 259/399] Apply 1 suggestion(s) to 1 file(s) --- pyotb/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 405e736..f9c8aa6 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -772,14 +772,14 @@ class App(OTBObject): - dictionary containing key-arguments enumeration. Useful when a key contains non-standard characters such as a point, e.g. {'io.out':'output.tif'} - None if output file was passed during App init - ext_fname: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES") - Will be used for all outputs (Default value = None) pixel_type: Can be : - dictionary {out_param_key: pixeltype} when specifying for several outputs - str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several outputs, all outputs are written with this unique type. Valid pixel types are uint8, uint16, uint32, int16, int32, float, double, cint16, cint32, cfloat, cdouble. (Default value = None) preserve_dtype: propagate main input pixel type to outputs, in case pixel_type is None + ext_fname: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES") + Will be used for all outputs (Default value = None) **kwargs: keyword arguments e.g. out='output.tif' Returns: -- GitLab From 167ed2c63725bd060f76f2d88331c0791aa7df85 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 7 Jul 2023 07:27:42 +0000 Subject: [PATCH 260/399] Apply 1 suggestion(s) to 1 file(s) --- doc/quickstart.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/quickstart.md b/doc/quickstart.md index e65c68f..111f283 100644 --- a/doc/quickstart.md +++ b/doc/quickstart.md @@ -45,7 +45,7 @@ output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'}) parameter that contains a dots (e.g. `io.in`). For `in` or other main input parameters of an OTB application, you may simply pass the value as first argument, pyotb will guess the parameter name. For parameters that - contains dots, you can either use a dictionary, or replace dots (`.`) + contains dots, you can either use a dictionary, or replace dots (`.`) with underscores (`_`). Let's take the example of the `OrthoRectification` application of OTB, -- GitLab From ff05e9e5839382ac651141f47e42cd19dd2417f1 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 7 Jul 2023 09:38:41 +0200 Subject: [PATCH 261/399] DOC: wip snippets style --- mkdocs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index e80cfac..b24d48c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -59,7 +59,6 @@ extra_css: use_directory_urls: false # this creates some pyotb/core.html pages instead of pyotb/core/index.html markdown_extensions: - - codehilite - admonition - toc: permalink: true -- GitLab From 70acbb8bd11812d31b01d2b36d5e929059e13931 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 7 Jul 2023 10:14:49 +0200 Subject: [PATCH 262/399] DOC: style, closer to OTB docs --- doc/doc_requirements.txt | 1 - doc/extra.css | 11 +++++++++++ mkdocs.yml | 3 +-- 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 doc/extra.css diff --git a/doc/doc_requirements.txt b/doc/doc_requirements.txt index 931c3df..4f59692 100644 --- a/doc/doc_requirements.txt +++ b/doc/doc_requirements.txt @@ -5,4 +5,3 @@ mkdocs-gen-files mkdocs-section-index mkdocs-literate-nav mkdocs-mermaid2-plugin -pygments diff --git a/doc/extra.css b/doc/extra.css new file mode 100644 index 0000000..ec4d4bb --- /dev/null +++ b/doc/extra.css @@ -0,0 +1,11 @@ +.rst-content div[class^=highlight] { + border: 0px; +} + +.rst-content div[class^=highlight] pre { + padding: 0px; +} + +.rst-content pre code { + background: #eeffcc; +} diff --git a/mkdocs.yml b/mkdocs.yml index b24d48c..18cc088 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -56,6 +56,7 @@ extra: link: https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb extra_css: - https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/8.1.2-rc1/Documentation/Cookbook/_static/css/otb_theme.css + - extra.css use_directory_urls: false # this creates some pyotb/core.html pages instead of pyotb/core/index.html markdown_extensions: @@ -70,8 +71,6 @@ markdown_extensions: - pymdownx.snippets - pymdownx.details - pymdownx.superfences - - pymdownx.highlight: - use_pygments: true # Rest of the navigation.. site_name: "pyotb documentation: a Python extension of OTB" -- GitLab From 3a1588266ae92d9bb9669260799dc808d8dfcccd Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 7 Jul 2023 10:35:03 +0200 Subject: [PATCH 263/399] DOC: more compact comparison OTB vs pyotb --- doc/comparison_otb.md | 168 ++++++++++++++++++++++++++---------------- 1 file changed, 105 insertions(+), 63 deletions(-) diff --git a/doc/comparison_otb.md b/doc/comparison_otb.md index 29d1e89..0129cb2 100644 --- a/doc/comparison_otb.md +++ b/doc/comparison_otb.md @@ -11,28 +11,32 @@ <td> ```python -import otbApplication +import otbApplication as otb input_path = 'my_image.tif' -resampled = otbApplication.Registry.CreateApplication( +app = otb.Registry.CreateApplication( 'RigidTransformResample' ) -resampled.SetParameterString('in', input_path) -resampled.SetParameterString('interpolator', 'linear') -resampled.SetParameterFloat( - 'transform.type.id.scalex', - 0.5 +app.SetParameterString( + 'in', input_path ) -resampled.SetParameterFloat( - 'transform.type.id.scaley', - 0.5 +app.SetParameterString( + 'interpolator', 'linear' ) -resampled.SetParameterString('out', 'output.tif') -resampled.SetParameterOutputImagePixelType( - 'out', otbApplication.ImagePixelType_uint16 +app.SetParameterFloat( + 'transform.type.id.scalex', 0.5 +) +app.SetParameterFloat( + 'transform.type.id.scaley', 0.5 +) +app.SetParameterString( + 'out', 'output.tif' +) +app.SetParameterOutputImagePixelType( + 'out', otb.ImagePixelType_uint16 ) -resampled.ExecuteAndWriteOutput() +app.ExecuteAndWriteOutput() ``` </td> @@ -41,14 +45,17 @@ resampled.ExecuteAndWriteOutput() ```python import pyotb -resampled = pyotb.RigidTransformResample({ +app = pyotb.RigidTransformResample({ 'in': 'my_image.tif', 'interpolator': 'linear', 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5 }) -resampled.write('output.tif', pixel_type='uint16') +app.write( + 'output.tif', + pixel_type='uint16' +) ``` </td> @@ -66,42 +73,55 @@ resampled.write('output.tif', pixel_type='uint16') <td> ```python -import otbApplication +import otbApplication as otb -app1 = otbApplication.Registry.CreateApplication( +app1 = otb.Registry.CreateApplication( 'RigidTransformResample' ) -app1.SetParameterString('in', 'my_image.tif') -app1.SetParameterString('interpolator', 'linear') +app1.SetParameterString( + 'in', 'my_image.tif' +) +app1.SetParameterString( + 'interpolator', 'linear' +) app1.SetParameterFloat( - 'transform.type.id.scalex', - 0.5 + 'transform.type.id.scalex', 0.5 ) app1.SetParameterFloat( - 'transform.type.id.scaley', - 0.5 + 'transform.type.id.scaley', 0.5 ) app1.Execute() -app2 = otbApplication.Registry.CreateApplication( +app2 = otb.Registry.CreateApplication( 'OpticalCalibration' ) app2.ConnectImage('in', app1, 'out') app2.SetParameterString('level', 'toa') app2.Execute() -app3 = otbApplication.Registry.CreateApplication( +app3 = otb.Registry.CreateApplication( 'BinaryMorphologicalOperation' ) -app3.ConnectImage('in', app2, 'out') -app3.SetParameterString('filter', 'dilate') -app3.SetParameterString('structype', 'ball') -app3.SetParameterInt('xradius', 3) -app3.SetParameterInt('yradius', 3) -app3.SetParameterString('out', 'output.tif') +app3.ConnectImage( + 'in', app2, 'out' +) +app3.SetParameterString( + 'filter', 'dilate' +) +app3.SetParameterString( + 'structype', 'ball' +) +app3.SetParameterInt( + 'xradius', 3 +) +app3.SetParameterInt( + 'yradius', 3 +) +app3.SetParameterString( + 'out', 'output.tif' +) app3.SetParameterOutputImagePixelType( - 'out', - otbApplication.ImagePixelType_uint16 + 'out', otb.ImagePixelType_uint16 ) app3.ExecuteAndWriteOutput() ``` @@ -159,28 +179,31 @@ Consider an example where we want to perform the arithmetic operation <td> ```python -import otbApplication +import otbApplication as otb -bmx = otbApplication.Registry.CreateApplication( +bmx = otb.Registry.CreateApplication( 'BandMathX' ) bmx.SetParameterStringList( 'il', - ['image1.tif', 'image2.tif', 'image3.tif'] -) # all images are 3-bands + ['im1.tif', 'im2.tif', 'im3.tif'] +) exp = ('im1b1*im2b1-2*im3b1; ' 'im1b2*im2b2-2*im3b2; ' 'im1b3*im2b3-2*im3b3') bmx.SetParameterString('exp', exp) -bmx.SetParameterString('out', 'output.tif') +bmx.SetParameterString( + 'out', + 'output.tif' +) bmx.SetParameterOutputImagePixelType( - 'out', - otbApplication.ImagePixelType_uint8 + 'out', + otb.ImagePixelType_uint8 ) bmx.ExecuteAndWriteOutput() ``` -In OTB, this code works for 3-bands images. +Note: code limited to 3-bands images. </td> <td> @@ -188,16 +211,19 @@ In OTB, this code works for 3-bands images. ```python import pyotb -# transforming filepaths to pyotb objects -input1 = pyotb.Input('image1.tif') -input2 = pyotb.Input('image2.tif') -input3 = pyotb.Input('image3.tif') +# filepaths --> pyotb objects +in1 = pyotb.Input('im1.tif') +in2 = pyotb.Input('im2.tif') +in3 = pyotb.Input('im3.tif') -res = input1 * input2 - 2 * input2 -res.write('output.tif', pixel_type='uint8') +res = in1 * in2 - 2 * in3 +res.write( + 'output.tif', + pixel_type='uint8' +) ``` -In pyotb,this code works with images of any number of bands. +Note: works with any number of bands. </td> </tr> @@ -215,13 +241,15 @@ In pyotb,this code works with images of any number of bands. ```python -import otbApplication +import otbApplication as otb # first 3 channels -app = otbApplication.Registry.CreateApplication( +app = otb.Registry.CreateApplication( 'ExtractROI' ) -app.SetParameterString('in', 'my_image.tif') +app.SetParameterString( + 'in', 'my_image.tif' +) app.SetParameterStringList( 'cl', ['Channel1', 'Channel2', 'Channel3'] @@ -229,16 +257,30 @@ app.SetParameterStringList( app.Execute() # 1000x1000 roi -app = otbApplication.Registry.CreateApplication( +app = otb.Registry.CreateApplication( 'ExtractROI' ) -app.SetParameterString('in', 'my_image.tif') -app.SetParameterString('mode', 'extent') -app.SetParameterString('mode.extent.unit', 'pxl') -app.SetParameterFloat('mode.extent.ulx', 0) -app.SetParameterFloat('mode.extent.uly', 0) -app.SetParameterFloat('mode.extent.lrx', 999) -app.SetParameterFloat('mode.extent.lry', 999) +app.SetParameterString( + 'in', 'my_image.tif' +) +app.SetParameterString( + 'mode', 'extent' +) +app.SetParameterString( + 'mode.extent.unit', 'pxl' +) +app.SetParameterFloat( + 'mode.extent.ulx', 0 +) +app.SetParameterFloat( + 'mode.extent.uly', 0 +) +app.SetParameterFloat( + 'mode.extent.lrx', 999 +) +app.SetParameterFloat( + 'mode.extent.lry', 999 +) app.Execute() ``` @@ -248,11 +290,11 @@ app.Execute() ```python import pyotb -# transforming filepath to pyotb object +# filepath --> pyotb object inp = pyotb.Input('my_image.tif') -extracted = inp[:, :, :3] # first 3 channels -extracted = inp[:1000, :1000] # 1000x1000 roi +extracted = inp[:, :, :3] # Bands 1,2,3 +extracted = inp[:1000, :1000] # ROI ``` </td> -- GitLab From ef5a1dd024a40cc67fa0e1fe86f168f4d7649ce2 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 7 Jul 2023 10:35:12 +0200 Subject: [PATCH 264/399] DOC: title --- mkdocs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 18cc088..5d1e8b8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -72,8 +72,8 @@ markdown_extensions: - pymdownx.details - pymdownx.superfences -# Rest of the navigation.. -site_name: "pyotb documentation: a Python extension of OTB" +# Rest of the navigation. +site_name: "pyotb: a Python extension of OTB" repo_url: https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb repo_name: pyotb docs_dir: doc/ -- GitLab From bee275a5b23c49ee847d15b58d6b81a4ce0ce96c Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 7 Jul 2023 08:38:56 +0000 Subject: [PATCH 265/399] Apply 1 suggestion(s) to 1 file(s) --- doc/quickstart.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/quickstart.md b/doc/quickstart.md index 111f283..fb15188 100644 --- a/doc/quickstart.md +++ b/doc/quickstart.md @@ -77,7 +77,7 @@ Let's start from our previous example. Consider the case where one wants to resample the image, then apply optical calibration and binary morphological dilatation. We can write the following code to build a pipeline that will generate the output in an end-to-end fashion, without being limited with the -input image size, without writing temporary files. +input image size or writing temporary files. ```python import pyotb -- GitLab From 2c94e26a34421d55728a3241da4c56da731a5aff Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 7 Jul 2023 12:20:42 +0200 Subject: [PATCH 266/399] DOC: fix python syntax highlight --- mkdocs.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 5d1e8b8..88589b1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -67,13 +67,15 @@ markdown_extensions: toc_depth: 1-2 - pymdownx.highlight: anchor_linenums: true - - pymdownx.inlinehilite - - pymdownx.snippets - pymdownx.details - - pymdownx.superfences + - pymdownx.superfences: + custom_fences: + - name: python + class: python + format: !!python/name:pymdownx.superfences.fence_code_format # Rest of the navigation. -site_name: "pyotb: a Python extension of OTB" +site_name: "pyotb: Orfeo ToolBox for Python" repo_url: https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb repo_name: pyotb docs_dir: doc/ -- GitLab From 8b8389aae60ed8a5cb15c67ac1b01956391e9be1 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 7 Jul 2023 17:34:53 +0200 Subject: [PATCH 267/399] DOC: fix summarize api doc --- pyotb/core.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index b7ee57a..f6e3060 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -502,14 +502,20 @@ class OTBObject(ABC): return f"<pyotb.{self.__class__.__name__} object, id {id(self)}>" +def old_otbobj_warn(): + depreciation_warning( + "Since pyotb 2.0.0, otbObject has been renamed OTBObject. " + "otbObject will be removed definitively in future releases." + ) + class otbObject(OTBObject): # noqa, pylint: disable=invalid-name """Class for depreciation of otbObject since pyotb 2.0.0. Will be removed in future releases.""" def __init_subclass__(cls): """Show a warning for depreciation.""" - depreciation_warning( - "Since pyotb 2.0.0, otbObject has been renamed OTBObject. " - "otbObject will be removed definitively in future releases." - ) + old_otbobj_warn() + def __class__(self): + old_otbobj_warn() + return super().__class__ class App(OTBObject): @@ -1744,8 +1750,7 @@ def summarize( useful to remove extended filenames. Returns: - nested dictionary with serialized App(s) containing name and - parameters of an app and its parents + nested dictionary with serialized App(s) containing name and parameters of an app and its parents """ -- GitLab From 199a9c196ef8be6dcc41e2c21ece9b8fc7d85e09 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 7 Jul 2023 17:35:42 +0200 Subject: [PATCH 268/399] DOC: fix summarize api doc --- pyotb/core.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index f9c8aa6..9bcd851 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1660,8 +1660,7 @@ def summarize( useful to remove extended filenames. Returns: - nested dictionary with serialized App(s) containing name and - parameters of an app and its parents + nested dictionary with serialized App(s) containing name and parameters of an app and its parents """ -- GitLab From cabb7f0a7a987b358ad8bfe9fe6af119f7d8c994 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 7 Jul 2023 18:31:54 +0200 Subject: [PATCH 269/399] FIX: AttributeError message --- pyotb/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index d5cca0e..f095125 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -455,7 +455,9 @@ class OTBObject(ABC): ) if hint: depreciation_warning(f"{note} {hint}") - raise AttributeError + raise AttributeError( + f"'{type(self).__name__}' object has no attribute '{item}'" + ) def __getitem__(self, key) -> Any | list[float] | float | Slicer: """Override the default __getitem__ behaviour. -- GitLab From 62bc96a99137467c53d3d6acec44c9d992489a66 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 7 Jul 2023 18:47:21 +0200 Subject: [PATCH 270/399] FIX: AttributeError message --- pyotb/core.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index f095125..1a5859c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -502,22 +502,6 @@ class OTBObject(ABC): return f"<pyotb.{self.__class__.__name__} object, id {id(self)}>" -def old_otbobj_warn(): - depreciation_warning( - "Since pyotb 2.0.0, otbObject has been renamed OTBObject. " - "otbObject will be removed definitively in future releases." - ) - -class otbObject(OTBObject): # noqa, pylint: disable=invalid-name - """Class for depreciation of otbObject since pyotb 2.0.0. Will be removed in future releases.""" - def __init_subclass__(cls): - """Show a warning for depreciation.""" - old_otbobj_warn() - def __class__(self): - old_otbobj_warn() - return super().__class__ - - class App(OTBObject): """Base class that gathers common operations for any OTB application.""" -- GitLab From 56dea1ac019db8c6db47dd26bf2a3c16f0d6f2cd Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Fri, 7 Jul 2023 18:48:00 +0200 Subject: [PATCH 271/399] FIX: remove otbObject --- pyotb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/__init__.py b/pyotb/__init__.py index 62508f5..bf55392 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -4,7 +4,7 @@ __version__ = "2.0.0.dev4" from .helpers import logger, set_logger_level from .apps import * -from .core import App, Input, Output, get_nbchannels, get_pixel_type, summarize, OTBObject, otbObject +from .core import App, Input, Output, get_nbchannels, get_pixel_type, summarize, OTBObject from .functions import ( # pylint: disable=redefined-builtin all, -- GitLab From 1490f60a6601886588ea788e6f492f647cc02514 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 7 Jul 2023 17:01:05 +0000 Subject: [PATCH 272/399] PERF: sub-processed application listing is not required with latest OTBTF --- pyotb/apps.py | 61 ++++++++------------------------------------------- 1 file changed, 9 insertions(+), 52 deletions(-) diff --git a/pyotb/apps.py b/pyotb/apps.py index 6f54b27..cd0c696 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -3,9 +3,6 @@ from __future__ import annotations import os -import subprocess -import sys -from pathlib import Path import otbApplication as otb # pylint: disable=import-error @@ -13,7 +10,7 @@ from .core import App from .helpers import logger -def get_available_applications(as_subprocess: bool = False) -> list[str]: +def get_available_applications() -> tuple[str]: """Find available OTB applications. Args: @@ -23,53 +20,13 @@ def get_available_applications(as_subprocess: bool = False) -> list[str]: tuple of available applications """ - app_list = () - if as_subprocess and sys.executable: - # Currently, there is an incompatibility between OTBTF and Tensorflow that causes segfault - # when OTBTF apps are used in a script where tensorflow has already been imported. - # See https://github.com/remicres/otbtf/issues/28 - # Thus, we run this piece of code in a clean independent `subprocess` that doesn't interact with Tensorflow - env = os.environ.copy() - if "PYTHONPATH" not in env: - env["PYTHONPATH"] = "" - env["PYTHONPATH"] += ":" + str(Path(otb.__file__).parent) - env[ - "OTB_LOGGER_LEVEL" - ] = "CRITICAL" # in order to suppress warnings while listing applications - pycmd = "import otbApplication; print(otbApplication.Registry.GetAvailableApplications())" - cmd_args = [sys.executable, "-c", pycmd] - try: - params = {"env": env, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE} - with subprocess.Popen(cmd_args, **params) as process: - logger.debug("Exec \"%s '%s'\"", " ".join(cmd_args[:-1]), pycmd) - stdout, stderr = process.communicate() - stdout, stderr = stdout.decode(), stderr.decode() - # ast.literal_eval is secure and will raise more handy Exceptions than eval - from ast import literal_eval # pylint: disable=import-outside-toplevel - - app_list = literal_eval(stdout.strip()) - assert isinstance(app_list, (tuple, list)) - except subprocess.SubprocessError: - logger.debug("Failed to call subprocess") - except (ValueError, SyntaxError, AssertionError): - logger.debug( - "Failed to decode output or convert to tuple:\nstdout=%s\nstderr=%s", - stdout, - stderr, - ) - if not app_list: - logger.debug( - "Failed to list applications in an independent process. Falling back to local python import" - ) - # Find applications using the normal way - if not app_list: - app_list = otb.Registry.GetAvailableApplications() - if not app_list: - raise SystemExit( - "Unable to load applications. Set env variable OTB_APPLICATION_PATH and try again." - ) - logger.info("Successfully loaded %s OTB applications", len(app_list)) - return app_list + app_list = otb.Registry.GetAvailableApplications() + if app_list: + logger.info("Successfully loaded %s OTB applications", len(app_list)) + return app_list + raise SystemExit( + "Unable to load applications. Set env variable OTB_APPLICATION_PATH and try again." + ) class OTBTFApp(App): @@ -113,7 +70,7 @@ class OTBTFApp(App): super().__init__(name, *args, **kwargs) -AVAILABLE_APPLICATIONS = get_available_applications(as_subprocess=True) +AVAILABLE_APPLICATIONS = get_available_applications() # This is to enable aliases of Apps, i.e. using apps like `pyotb.AppName(...)` instead of `pyotb.App("AppName", ...)` _CODE_TEMPLATE = ( -- GitLab From 3bedecfc229257cc3908417bf044a25ebcb579ec Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 7 Jul 2023 19:19:01 +0200 Subject: [PATCH 273/399] STYLE: run black --- pyotb/depreciation.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pyotb/depreciation.py b/pyotb/depreciation.py index 859ba60..687e6ca 100644 --- a/pyotb/depreciation.py +++ b/pyotb/depreciation.py @@ -8,9 +8,10 @@ import functools import warnings -WARN = '\033[91m' -ENDC = '\033[0m' -OKAY = '\033[92m' +WARN = "\033[91m" +ENDC = "\033[0m" +OKAY = "\033[92m" + def depreciation_warning(message: str): """Shows a warning message. @@ -24,6 +25,8 @@ def depreciation_warning(message: str): category=DeprecationWarning, stacklevel=3, ) + + def deprecated_alias(**aliases: str) -> Callable: """Decorator for deprecated function and method arguments. @@ -40,6 +43,7 @@ def deprecated_alias(**aliases: str) -> Callable: wrapped function """ + def deco(func: Callable): @functools.wraps(func) def wrapper(*args, **kwargs): @@ -51,11 +55,7 @@ def deprecated_alias(**aliases: str) -> Callable: return deco -def rename_kwargs( - func_name: str, - kwargs: Dict[str, Any], - aliases: Dict[str, str] -): +def rename_kwargs(func_name: str, kwargs: Dict[str, Any], aliases: Dict[str, str]): """Helper function for deprecating function arguments. Args: @@ -78,6 +78,7 @@ def rename_kwargs( depreciation_warning(message) kwargs[new] = kwargs.pop(alias) + def deprecated_attr(replacement: str) -> Callable: """Decorator for deprecated attr. @@ -94,6 +95,7 @@ def deprecated_attr(replacement: str) -> Callable: wrapped function """ + def deco(attr: Any): @functools.wraps(attr) def wrapper(self, *args, **kwargs): -- GitLab From 0d7dc0c4fd47f25d8c7a002f64f4cb085b1556cd Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 7 Jul 2023 19:20:36 +0200 Subject: [PATCH 274/399] STYLE: run black on core.py --- pyotb/core.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 1a5859c..d49581f 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -115,7 +115,6 @@ class OTBObject(ABC): origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2 return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y - def summarize(self, *args, **kwargs): """Recursively summarize parameters and parents. @@ -129,7 +128,6 @@ class OTBObject(ABC): """ return summarize(self, *args, **kwargs) - def get_info(self) -> dict[str, (str, float, list[float])]: """Return a dict output of ReadImageInfo for the first image output.""" return App("ReadImageInfo", self, quiet=True).data @@ -435,8 +433,7 @@ class OTBObject(ABC): # Because otbApplication instances methods names start with an # upper case hint = ( - f"Maybe try `pyotb_app.app.{item}` instead of " - f"`pyotb_app.{item}`? " + f"Maybe try `pyotb_app.app.{item}` instead of " f"`pyotb_app.{item}`? " ) if item.startswith("GetParameter"): hint += ( @@ -1498,7 +1495,6 @@ class Output(OTBObject): _filepath: str | Path = None - @deprecated_alias(app="pyotb_app", output_parameter_key="param_key") def __init__( self, -- GitLab From 9d27acfa592f927c443edc63a896e42125db1a79 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 8 Jul 2023 07:16:06 +0000 Subject: [PATCH 275/399] Apply 1 suggestion(s) to 1 file(s) --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index d49581f..c8e98a7 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -433,7 +433,7 @@ class OTBObject(ABC): # Because otbApplication instances methods names start with an # upper case hint = ( - f"Maybe try `pyotb_app.app.{item}` instead of " f"`pyotb_app.{item}`? " + f"Maybe try `pyotb_app.app.{item}` instead of `pyotb_app.{item}`? " ) if item.startswith("GetParameter"): hint += ( -- GitLab From b5a3c8097bfc463694ccca9bb58e3a491b837585 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Mon, 10 Jul 2023 09:08:31 +0200 Subject: [PATCH 276/399] DOC: add a note for migration in troubleshooting --- doc/troubleshooting.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md index 0025913..ee3bdf4 100644 --- a/doc/troubleshooting.md +++ b/doc/troubleshooting.md @@ -1,4 +1,16 @@ -## Troubleshooting: known limitations with old versions +# Troubleshooting + +## Migration from pyotb 1.5.4 (oct 2022) to 2.x.y + +- `otbObject` has ben renamed `OTBObject` +- use `pyotb_app['paramname']` or `pyotb_app.app.GetParameterValue('paramname')` instead of `pyotb_app.GetParameterValue('paramname')` to access parameter `paramname` value +- use `pyotb_app['paramname']` instead of `pyotb_app.paramname` to access parameter `paramname` value +- `App.output_param` has been replaced with `App.output_image_key` +- `App.write()` argument `filename_extension` has been renamed `ext_fname` +- `Output.__init__()` arguments `app` and `output_parameter_key` have been renamed `pyotb_app` and `param_key` +- `Output.pyotb_app` has been renamed `Output.parent_pyotb_app` + +## Known limitations with old versions !!! note -- GitLab From 19d682d854045f3ac16ff8e7a7a5cb1390ebdc19 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Mon, 10 Jul 2023 09:20:04 +0200 Subject: [PATCH 277/399] DOC: add a note for migration in troubleshooting --- doc/troubleshooting.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md index ee3bdf4..9cf5b84 100644 --- a/doc/troubleshooting.md +++ b/doc/troubleshooting.md @@ -2,13 +2,22 @@ ## Migration from pyotb 1.5.4 (oct 2022) to 2.x.y +List of breaking changes: + - `otbObject` has ben renamed `OTBObject` -- use `pyotb_app['paramname']` or `pyotb_app.app.GetParameterValue('paramname')` instead of `pyotb_app.GetParameterValue('paramname')` to access parameter `paramname` value -- use `pyotb_app['paramname']` instead of `pyotb_app.paramname` to access parameter `paramname` value +- `otbObject.get_infos()` has been renamed `OTBObject.get_info()` +- `otbObject.key_output_image` has been renamed `OTBObject.output_image_key` +- `otbObject.key_input_image` has been renamed `OTBObject.input_image_key` +- `otbObject.read_values_at_coords()` has been renamed `OTBObject.get_values_at_coords()` +- `otbObject.xy_to_rowcol()` has been renamed `OTBObject.get_rowcol_from_xy()` - `App.output_param` has been replaced with `App.output_image_key` - `App.write()` argument `filename_extension` has been renamed `ext_fname` +- `App.save_objects()` has been renamed `App.__sync_parameters()` +- use `pyotb_app['paramname']` or `pyotb_app.app.GetParameterValue('paramname')` instead of `pyotb_app.GetParameterValue('paramname')` to access parameter `paramname` value +- use `pyotb_app['paramname']` instead of `pyotb_app.paramname` to access parameter `paramname` value - `Output.__init__()` arguments `app` and `output_parameter_key` have been renamed `pyotb_app` and `param_key` - `Output.pyotb_app` has been renamed `Output.parent_pyotb_app` +- `logicalOperation` has been renamed `LogicalOperation` ## Known limitations with old versions -- GitLab From b7b7b9cf017f4cc896d25d045f4feff1fc9ed35f Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 11 Jul 2023 15:06:06 +0000 Subject: [PATCH 278/399] CI: add tests coverage + badge + apply black --- .gitlab-ci.yml | 20 +++--- README.md | 1 + pyproject.toml | 2 +- tests/test_core.py | 152 ++++++++++++++++++++++++++++++++++---------- tests/test_numpy.py | 67 ------------------- 5 files changed, 130 insertions(+), 112 deletions(-) delete mode 100644 tests/test_numpy.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3e7f69d..c57f662 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -68,6 +68,7 @@ codespell: rules: - changes: - "**/*.py" + - .gitlab-ci.yml variables: OTB_ROOT: /opt/otb LD_LIBRARY_PATH: /opt/otb/lib @@ -76,19 +77,18 @@ codespell: artifacts: reports: junit: test-*.xml + coverage_report: + coverage_format: cobertura + path: coverage.xml before_script: - - pip install pytest + - pip install pytest pytest-cov - pip install . test_core: extends: .tests + coverage: '/TOTAL.*\s+(\d+%)$/' script: - - pytest -vv --color=yes --junitxml=test-core.xml tests/test_core.py - -test_numpy: - extends: .tests - script: - - pytest -vv --color=yes --junitxml=test-numpy.xml tests/test_numpy.py + - pytest -vv --color=yes --junitxml=test-core.xml --cov=/usr/local/lib/python3.8/dist-packages/pyotb --cov-report term --cov-report xml:coverage.xml tests/test_core.py test_pipeline: extends: .tests @@ -102,9 +102,9 @@ docs: # when: manual rules: - changes: - - mkdocs.yml - - doc/* - - pyotb/*.py + - mkdocs.yml + - doc/* + - pyotb/*.py before_script: - apt update && apt install -y python3.8-venv - python3 -m venv docs_venv diff --git a/README.md b/README.md index d51ba17..6659aab 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/releases) [](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop) +[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop) [](https://pyotb.readthedocs.io/en/master/) **pyotb** wraps the [Orfeo Toolbox](https://www.orfeo-toolbox.org/) (OTB) diff --git a/pyproject.toml b/pyproject.toml index bc2c576..429ed73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ ] [project.optional-dependencies] -dev = ["pytest", "pylint", "codespell", "pydocstyle", "tomli", "requests"] +dev = ["pytest", "pytest-cov", "pylint", "codespell", "pydocstyle", "tomli", "requests"] [project.urls] documentation = "https://pyotb.readthedocs.io" diff --git a/tests/test_core.py b/tests/test_core.py index 2e07b96..bbc35ae 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,4 +1,5 @@ import pytest +import numpy as np from tests_data import * @@ -30,7 +31,10 @@ def test_input_vsi(): assert info.parameters["in"] == "https://fake.com/image.tif" # Compressed remote file info = pyotb.ReadImageInfo("https://fake.com/image.tif.zip", frozen=True) - assert info.app.GetParameterValue("in") == "/vsizip//vsicurl/https://fake.com/image.tif.zip" + assert ( + info.app.GetParameterValue("in") + == "/vsizip//vsicurl/https://fake.com/image.tif.zip" + ) assert info.parameters["in"] == "https://fake.com/image.tif.zip" # Piped curl --> zip --> tiff ziped_tif_urls = ( @@ -93,8 +97,10 @@ def test_metadata(): assert "ProjectionRef", "OVR_RESAMPLING_ALG" in INPUT2.metadata # Metadata with numeric values (e.g. TileHintX) - fp = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/" \ - "Data/Input/radarsat2/RADARSAT2_ALTONA_300_300_VV.tif?inline=false" + fp = ( + "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/" + "Data/Input/radarsat2/RADARSAT2_ALTONA_300_300_VV.tif?inline=false" + ) app = pyotb.BandMath({"il": [fp], "exp": "im1b1"}) assert "TileHintX" in app.metadata @@ -130,7 +136,7 @@ def test_write(): def test_ext_fname(): - def _check(expected: str, key: str = "out", app = INPUT.app): + def _check(expected: str, key: str = "out", app=INPUT.app): fn = app.GetParameterString(key) assert "?&" in fn assert fn.split("?&", 1)[1] == expected @@ -142,32 +148,23 @@ def test_ext_fname(): assert INPUT.write("/tmp/test_write.tif", ext_fname={"nodata": 0}) _check("nodata=0") assert INPUT.write( - "/tmp/test_write.tif", - ext_fname={ - "nodata": 0, - "gdal:co:COMPRESS": "DEFLATE" - } + "/tmp/test_write.tif", ext_fname={"nodata": 0, "gdal:co:COMPRESS": "DEFLATE"} ) _check("nodata=0&gdal:co:COMPRESS=DEFLATE") assert INPUT.write( - "/tmp/test_write.tif", - ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE" + "/tmp/test_write.tif", ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE" ) _check("nodata=0&gdal:co:COMPRESS=DEFLATE") assert INPUT.write( "/tmp/test_write.tif?&box=0:0:10:10", - ext_fname={ - "nodata": "0", - "gdal:co:COMPRESS": "DEFLATE", - "box": "0:0:20:20" - } + ext_fname={"nodata": "0", "gdal:co:COMPRESS": "DEFLATE", "box": "0:0:20:20"}, ) # Check that the bbox is the one specified in the filepath, not the one # specified in `ext_filename` _check("nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:10:10") assert INPUT.write( "/tmp/test_write.tif?&box=0:0:10:10", - ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:20:20" + ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:20:20", ) _check("nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:10:10") @@ -177,12 +174,9 @@ def test_ext_fname(): mss.write( { "fout": "/tmp/test_ext_fn_fout.tif?&nodata=1", - "foutpos": "/tmp/test_ext_fn_foutpos.tif?&nodata=2" + "foutpos": "/tmp/test_ext_fn_foutpos.tif?&nodata=2", }, - ext_fname={ - "nodata": 0, - "gdal:co:COMPRESS": "DEFLATE" - } + ext_fname={"nodata": 0, "gdal:co:COMPRESS": "DEFLATE"}, ) _check("nodata=1&gdal:co:COMPRESS=DEFLATE", key="fout", app=mss.app) _check("nodata=2&gdal:co:COMPRESS=DEFLATE", key="foutpos", app=mss.app) @@ -190,13 +184,14 @@ def test_ext_fname(): mss["foutpos"].filepath.unlink() - def test_frozen_app_write(): app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True) assert app.write("/tmp/test_frozen_app_write.tif") app["out"].filepath.unlink() - app = pyotb.BandMath(INPUT, exp="im1b1", out="/tmp/test_frozen_app_write.tif", frozen=True) + app = pyotb.BandMath( + INPUT, exp="im1b1", out="/tmp/test_frozen_app_write.tif", frozen=True + ) assert app.write() app["out"].filepath.unlink() @@ -244,7 +239,9 @@ def test_rational_operators(): meas = func(INPUT) ref = pyotb.BandMathX({"il": [FILEPATH], "exp": exp}) for i in range(1, 5): - compared = pyotb.CompareImages({"ref.in": ref, "meas.in": meas, "ref.channel": i, "meas.channel": i}) + compared = pyotb.CompareImages( + {"ref.in": ref, "meas.in": meas, "ref.channel": i, "meas.channel": i} + ) assert (compared["count"], compared["mse"]) == (0, 0) _test(lambda x: x + x, "im1 + im1") @@ -279,7 +276,10 @@ def test_rational_operators(): def test_operation(): op = INPUT / 255 * 128 - assert op.exp == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)" + assert ( + op.exp + == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)" + ) assert op.dtype == "float32" @@ -296,7 +296,10 @@ def test_binary_mask_where(): # Create binary mask based on several possible values values = [1, 2, 3, 4] res = pyotb.where(pyotb.any(INPUT[:, :, 0] == value for value in values), 255, 0) - assert res.exp == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)" + assert ( + res.exp + == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)" + ) # Essential apps @@ -323,18 +326,27 @@ def test_read_values_at_coords(): # BandMath NDVI == RadiometricIndices NDVI ? def test_ndvi_comparison(): - ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (INPUT[:, :, -1] + INPUT[:, :, 0]) - ndvi_indices = pyotb.RadiometricIndices(INPUT, {"list": ["Vegetation:NDVI"], "channels.red": 1, "channels.nir": 4}) + ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / ( + INPUT[:, :, -1] + INPUT[:, :, 0] + ) + ndvi_indices = pyotb.RadiometricIndices( + INPUT, {"list": ["Vegetation:NDVI"], "channels.red": 1, "channels.nir": 4} + ) assert ndvi_bandmath.exp == "((im1b4 - im1b1) / (im1b4 + im1b1))" assert ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", "float") assert ndvi_indices.write("/tmp/ndvi_indices.tif", "float") - compared = pyotb.CompareImages({"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"}) + compared = pyotb.CompareImages( + {"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"} + ) assert (compared["count"], compared["mse"]) == (0, 0) thresholded_indices = pyotb.where(ndvi_indices >= 0.3, 1, 0) assert thresholded_indices["exp"] == "((im1b1 >= 0.3) ? 1 : 0)" thresholded_bandmath = pyotb.where(ndvi_bandmath >= 0.3, 1, 0) - assert thresholded_bandmath["exp"] == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)" + assert ( + thresholded_bandmath["exp"] + == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)" + ) def test_summarize_output(): @@ -351,14 +363,16 @@ def test_summarize_strip_output(): (in_fn, out_fn_w_ext, "out", {}, out_fn_w_ext), (in_fn, out_fn_w_ext, "out", {"strip_output_paths": True}, out_fn), (in_fn_w_ext, out_fn, "in", {}, in_fn_w_ext), - (in_fn_w_ext, out_fn, "in", {"strip_input_paths": True}, in_fn) + (in_fn_w_ext, out_fn, "in", {"strip_input_paths": True}, in_fn), ] for inp, out, key, extra_args, expected in baseline: app = pyotb.ExtractROI({"in": inp, "out": out}) summary = pyotb.summarize(app, **extra_args) - assert summary["parameters"][key] == expected, \ - f"Failed for input {inp}, output {out}, args {extra_args}" + assert ( + summary["parameters"][key] == expected + ), f"Failed for input {inp}, output {out}, args {extra_args}" + def test_summarize_consistency(): app_fns = [ @@ -372,6 +386,7 @@ def test_summarize_consistency(): lambda inp: pyotb.BandMathX({"il": [inp], "exp": "im1"}), lambda inp: pyotb.OrthoRectification({"io.in": inp}), ] + def _test(app_fn): """ Here we create 2 summaries: @@ -388,9 +403,12 @@ def test_summarize_consistency(): app[out_key].filepath.unlink() summary_wo_wrt["parameters"].update({out_key: out_file}) assert summary_wo_wrt == summay_w_wrt + for app_fn in app_fns: _test(app_fn) + +@pytest.mark.xfail def test_pipeline_simple(): # BandMath -> OrthoRectification -> ManageNoData app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"}) @@ -400,6 +418,7 @@ def test_pipeline_simple(): assert summary == SIMPLE_SERIALIZATION +@pytest.mark.xfail def test_pipeline_diamond(): # Diamond graph app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"}) @@ -408,3 +427,68 @@ def test_pipeline_diamond(): app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"}) summary = pyotb.summarize(app4) assert summary == COMPLEX_SERIALIZATION + + +# Numpy funcs +def test_export(): + INPUT.export() + array = INPUT.exports_dic[INPUT.output_image_key]["array"] + assert isinstance(array, np.ndarray) + assert array.dtype == "uint8" + del INPUT.exports_dic["out"] + + +def test_output_export(): + INPUT["out"].export() + assert INPUT["out"].output_image_key in INPUT["out"].exports_dic + + +def test_to_numpy(): + array = INPUT.to_numpy() + assert array.dtype == np.uint8 + assert array.shape == INPUT.shape + assert array.min() == 33 + assert array.max() == 255 + + +def test_to_numpy_sliced(): + sliced = INPUT[:100, :200, :3] + array = sliced.to_numpy() + assert array.dtype == np.uint8 + assert array.shape == (100, 200, 3) + + +def test_convert_to_array(): + array = np.array(INPUT) + assert isinstance(array, np.ndarray) + assert INPUT.shape == array.shape + + +def test_pixel_coords_otb_equals_numpy(): + assert INPUT[19, 7] == list(INPUT.to_numpy()[19, 7]) + + +def test_add_noise_array(): + white_noise = np.random.normal(0, 50, size=INPUT.shape) + noisy_image = INPUT + white_noise + assert isinstance(noisy_image, pyotb.core.App) + assert noisy_image.shape == INPUT.shape + + +def test_to_rasterio(): + array, profile = INPUT.to_rasterio() + assert array.dtype == profile["dtype"] == np.uint8 + assert array.shape == (4, 304, 251) + assert profile["transform"] == (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0) + + # CRS test requires GDAL python bindings + try: + from osgeo import osr + + crs = osr.SpatialReference() + crs.ImportFromEPSG(2154) + dest_crs = osr.SpatialReference() + dest_crs.ImportFromWkt(profile["crs"]) + assert dest_crs.IsSame(crs) + except ImportError: + pass diff --git a/tests/test_numpy.py b/tests/test_numpy.py deleted file mode 100644 index d9c3d7b..0000000 --- a/tests/test_numpy.py +++ /dev/null @@ -1,67 +0,0 @@ -import numpy as np -import pyotb -from tests_data import INPUT - - -def test_export(): - INPUT.export() - array = INPUT.exports_dic[INPUT.output_image_key]["array"] - assert isinstance(array, np.ndarray) - assert array.dtype == "uint8" - del INPUT.exports_dic["out"] - - -def test_output_export(): - INPUT["out"].export() - assert INPUT["out"].output_image_key in INPUT["out"].exports_dic - - -def test_to_numpy(): - array = INPUT.to_numpy() - assert array.dtype == np.uint8 - assert array.shape == INPUT.shape - assert array.min() == 33 - assert array.max() == 255 - - -def test_to_numpy_sliced(): - sliced = INPUT[:100, :200, :3] - array = sliced.to_numpy() - assert array.dtype == np.uint8 - assert array.shape == (100, 200, 3) - - -def test_convert_to_array(): - array = np.array(INPUT) - assert isinstance(array, np.ndarray) - assert INPUT.shape == array.shape - - -def test_pixel_coords_otb_equals_numpy(): - assert INPUT[19, 7] == list(INPUT.to_numpy()[19, 7]) - - -def test_add_noise_array(): - white_noise = np.random.normal(0, 50, size=INPUT.shape) - noisy_image = INPUT + white_noise - assert isinstance(noisy_image, pyotb.core.App) - assert noisy_image.shape == INPUT.shape - - -def test_to_rasterio(): - array, profile = INPUT.to_rasterio() - assert array.dtype == profile["dtype"] == np.uint8 - assert array.shape == (4, 304, 251) - assert profile["transform"] == (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0) - - # CRS test requires GDAL python bindings - try: - from osgeo import osr - - crs = osr.SpatialReference() - crs.ImportFromEPSG(2154) - dest_crs = osr.SpatialReference() - dest_crs.ImportFromWkt(profile["crs"]) - assert dest_crs.IsSame(crs) - except ImportError: - pass -- GitLab From ddd61c281351b0f3df83be50d0a972e45a5c2525 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 12 Jul 2023 13:26:48 +0000 Subject: [PATCH 279/399] ADD: tests with OpticalCalibration --- .gitlab-ci.yml | 32 +++++-- pyotb/core.py | 4 +- pyproject.toml | 25 +++-- ...alized_apps.json => pipeline_summary.json} | 92 +++++++++---------- tests/test_core.py | 56 ++++++----- tests/test_pipeline.py | 4 +- tests/tests_data.py | 12 +-- 7 files changed, 127 insertions(+), 98 deletions(-) rename tests/{serialized_apps.json => pipeline_summary.json} (58%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c57f662..18cca61 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -74,26 +74,38 @@ codespell: LD_LIBRARY_PATH: /opt/otb/lib OTB_LOGGER_LEVEL: INFO PYOTB_LOGGER_LEVEL: DEBUG - artifacts: - reports: - junit: test-*.xml - coverage_report: - coverage_format: cobertura - path: coverage.xml + SPOT_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif + PLEIADES_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Baseline/OTB/Images/prTvOrthoRectification_pleiades-1_noDEM.tif before_script: - pip install pytest pytest-cov + +test_install: + extends: .tests + script: - pip install . -test_core: +test_module_core: extends: .tests + artifacts: + reports: + junit: test-module-core.xml + coverage_report: + coverage_format: cobertura + path: coverage.xml coverage: '/TOTAL.*\s+(\d+%)$/' script: - - pytest -vv --color=yes --junitxml=test-core.xml --cov=/usr/local/lib/python3.8/dist-packages/pyotb --cov-report term --cov-report xml:coverage.xml tests/test_core.py + - curl -fsLI $SPOT_IMG_URL + - curl -fsLI $PLEIADES_IMG_URL + - python3 -m pytest -vv --junitxml=test-module-core.xml --cov-report xml:coverage.xml tests/test_core.py -test_pipeline: +test_pipeline_permutations: extends: .tests + artifacts: + reports: + junit: test-pipeline-permutations.xml script: - - pytest -vv --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py + - curl -fsLI $SPOT_IMG_URL + - python3 -m pytest -vv --junitxml=test-pipeline-permutations.xml tests/test_pipeline.py # -------------------------------------- Docs --------------------------------------- diff --git a/pyotb/core.py b/pyotb/core.py index 18ad041..71a032c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -432,9 +432,7 @@ class OTBObject(ABC): if item in dir(self.app): # Because otbApplication instances methods names start with an # upper case - hint = ( - f"Maybe try `pyotb_app.app.{item}` instead of `pyotb_app.{item}`? " - ) + hint = f"Maybe try `pyotb_app.app.{item}` instead of `pyotb_app.{item}`? " if item.startswith("GetParameter"): hint += ( "Note: `pyotb_app.app.GetParameterValue('paramname')` can be " diff --git a/pyproject.toml b/pyproject.toml index 429ed73..30804d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,15 +6,15 @@ build-backend = "setuptools.build_meta" name = "pyotb" description = "Library to enable easy use of the Orfeo ToolBox (OTB) in Python" authors = [ - {name="Rémi Cresson", email="remi.cresson@inrae.fr"}, - {name="Nicolas Narçon"}, - {name="Vincent Delbar"}, + { name = "Rémi Cresson", email = "remi.cresson@inrae.fr" }, + { name = "Nicolas Narçon" }, + { name = "Vincent Delbar" }, ] requires-python = ">=3.7" keywords = ["gis", "remote sensing", "otb", "orfeotoolbox", "orfeo toolbox"] dependencies = ["numpy>=1.16"] readme = "README.md" -license = {text="Apache-2.0"} +license = { text = "Apache-2.0" } dynamic = ["version"] classifiers = [ "Programming Language :: Python :: 3", @@ -30,7 +30,15 @@ classifiers = [ ] [project.optional-dependencies] -dev = ["pytest", "pytest-cov", "pylint", "codespell", "pydocstyle", "tomli", "requests"] +dev = [ + "pytest", + "pytest-cov", + "pylint", + "codespell", + "pydocstyle", + "tomli", + "requests", +] [project.urls] documentation = "https://pyotb.readthedocs.io" @@ -41,7 +49,7 @@ repository = "https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb" packages = ["pyotb"] [tool.setuptools.dynamic] -version = {attr="pyotb.__version__"} +version = { attr = "pyotb.__version__" } [tool.pylint] max-line-length = 88 @@ -61,3 +69,8 @@ convention = "google" [tool.black] line-length = 88 + +[tool.pytest.ini_options] +minversion = "7.0" +addopts = "--color=yes --cov=pyotb --no-cov-on-fail --cov-report term" +testpaths = ["tests"] diff --git a/tests/serialized_apps.json b/tests/pipeline_summary.json similarity index 58% rename from tests/serialized_apps.json rename to tests/pipeline_summary.json index 938958c..8c598ae 100644 --- a/tests/serialized_apps.json +++ b/tests/pipeline_summary.json @@ -2,39 +2,39 @@ "SIMPLE": { "name": "ManageNoData", "parameters": { + "usenan": false, "mode": "buildmask", "mode.buildmask.inv": 1.0, "mode.buildmask.outv": 0.0, - "usenan": false, "in": { - "name": "OrthoRectification", + "name": "BandMath", "parameters": { - "map": "utm", - "map.utm.zone": 31, - "map.utm.northhem": true, - "outputs.ulx": 560000.8125, - "outputs.uly": 5495732.5, - "outputs.sizex": 251, - "outputs.sizey": 304, - "outputs.spacingx": 5.997312068939209, - "outputs.spacingy": -5.997312068939209, - "outputs.lrx": 561506.125, - "outputs.lry": 5493909.5, - "outputs.isotropic": true, - "opt.gridspacing": 4.0, - "opt.rpc": 10, - "outputs.mode": "auto", - "interpolator": "bco", - "interpolator.bco.radius": 2, - "io.in": { - "name": "BandMath", - "parameters": { - "il": [ - "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" - ], - "exp": "im1b1" + "il": [ + { + "name": "OrthoRectification", + "parameters": { + "map": "utm", + "map.utm.zone": 31, + "map.utm.northhem": true, + "outputs.mode": "auto", + "outputs.ulx": 560000.8382510637, + "outputs.uly": 5495732.692591702, + "outputs.sizex": 251, + "outputs.sizey": 304, + "outputs.spacingx": 5.997312290795521, + "outputs.spacingy": -5.997312290795521, + "outputs.lrx": 561506.1636360534, + "outputs.lry": 5493909.5096553, + "outputs.isotropic": true, + "interpolator": "bco", + "interpolator.bco.radius": 2, + "opt.rpc": 10, + "opt.gridspacing": 4.0, + "io.in": "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" + } } - } + ], + "exp": "im1b1" } } } @@ -49,20 +49,20 @@ "map": "utm", "map.utm.zone": 31, "map.utm.northhem": true, - "outputs.ulx": 560000.8125, - "outputs.uly": 5495732.5, + "outputs.mode": "auto", + "outputs.ulx": 560000.8382510637, + "outputs.uly": 5495732.692591702, "outputs.sizex": 251, "outputs.sizey": 304, - "outputs.spacingx": 5.997312068939209, - "outputs.spacingy": -5.997312068939209, - "outputs.lrx": 561506.125, - "outputs.lry": 5493909.5, + "outputs.spacingx": 5.997312290795521, + "outputs.spacingy": -5.997312290795521, + "outputs.lrx": 561506.1636360534, + "outputs.lry": 5493909.5096553, "outputs.isotropic": true, - "opt.gridspacing": 4.0, - "opt.rpc": 10, - "outputs.mode": "auto", "interpolator": "bco", "interpolator.bco.radius": 2, + "opt.rpc": 10, + "opt.gridspacing": 4.0, "io.in": { "name": "BandMath", "parameters": { @@ -77,30 +77,30 @@ { "name": "ManageNoData", "parameters": { + "usenan": false, "mode": "buildmask", "mode.buildmask.inv": 1.0, "mode.buildmask.outv": 0.0, - "usenan": false, "in": { "name": "OrthoRectification", "parameters": { "map": "utm", "map.utm.zone": 31, "map.utm.northhem": true, - "outputs.ulx": 560000.8125, - "outputs.uly": 5495732.5, + "outputs.mode": "auto", + "outputs.ulx": 560000.8382510637, + "outputs.uly": 5495732.692591702, "outputs.sizex": 251, "outputs.sizey": 304, - "outputs.spacingx": 5.997312068939209, - "outputs.spacingy": -5.997312068939209, - "outputs.lrx": 561506.125, - "outputs.lry": 5493909.5, + "outputs.spacingx": 5.997312290795521, + "outputs.spacingy": -5.997312290795521, + "outputs.lrx": 561506.1636360534, + "outputs.lry": 5493909.5096553, "outputs.isotropic": true, - "opt.gridspacing": 4.0, - "opt.rpc": 10, - "outputs.mode": "auto", "interpolator": "bco", "interpolator.bco.radius": 2, + "opt.rpc": 10, + "opt.gridspacing": 4.0, "io.in": { "name": "BandMath", "parameters": { diff --git a/tests/test_core.py b/tests/test_core.py index bbc35ae..8d54003 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -4,11 +4,23 @@ import numpy as np from tests_data import * -# Input settings def test_parameters(): + # Input / ExtractROI assert INPUT.parameters - assert INPUT.parameters["in"] == FILEPATH + assert INPUT.parameters["in"] == SPOT_IMG_URL assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304) + # ManageNoData + app = pyotb.ManageNoData(INPUT) + assert "usenan" in app._auto_parameters + assert "mode.buildmask.inv" in app._auto_parameters + # OpticalCalibration + inp = pyotb.Input(PLEIADES_IMG_URL) + app = pyotb.OpticalCalibration(inp, level="toa") + assert "milli" in app._auto_parameters + assert "clamp" in app._auto_parameters + assert app._auto_parameters["acqui.year"] == 2012 + assert app._auto_parameters["acqui.sun.elev"] == 23.836299896240234 + # OrthoRectification app = pyotb.OrthoRectification(INPUT) assert isinstance(app.parameters["map"], str) assert app.parameters["map"] == "utm" @@ -50,7 +62,7 @@ def test_input_vsi(): def test_input_vsi_from_user(): # Ensure old way is still working: ExtractROI will raise RuntimeError if a path is malformed - pyotb.Input("/vsicurl/" + FILEPATH) + pyotb.Input("/vsicurl/" + SPOT_IMG_URL) def test_wrong_key(): @@ -170,7 +182,7 @@ def test_ext_fname(): INPUT["out"].filepath.unlink() - mss = pyotb.MeanShiftSmoothing(FILEPATH) + mss = pyotb.MeanShiftSmoothing(SPOT_IMG_URL) mss.write( { "fout": "/tmp/test_ext_fn_fout.tif?&nodata=1", @@ -237,7 +249,7 @@ def test_slicer_in_output(): def test_rational_operators(): def _test(func, exp): meas = func(INPUT) - ref = pyotb.BandMathX({"il": [FILEPATH], "exp": exp}) + ref = pyotb.BandMathX({"il": [SPOT_IMG_URL], "exp": exp}) for i in range(1, 5): compared = pyotb.CompareImages( {"ref.in": ref, "meas.in": meas, "ref.channel": i, "meas.channel": i} @@ -248,14 +260,14 @@ def test_rational_operators(): _test(lambda x: x - x, "im1 - im1") _test(lambda x: x / x, "im1 div im1") _test(lambda x: x * x, "im1 mult im1") - _test(lambda x: x + FILEPATH, "im1 + im1") - _test(lambda x: x - FILEPATH, "im1 - im1") - _test(lambda x: x / FILEPATH, "im1 div im1") - _test(lambda x: x * FILEPATH, "im1 mult im1") - _test(lambda x: FILEPATH + x, "im1 + im1") - _test(lambda x: FILEPATH - x, "im1 - im1") - _test(lambda x: FILEPATH / x, "im1 div im1") - _test(lambda x: FILEPATH * x, "im1 mult im1") + _test(lambda x: x + SPOT_IMG_URL, "im1 + im1") + _test(lambda x: x - SPOT_IMG_URL, "im1 - im1") + _test(lambda x: x / SPOT_IMG_URL, "im1 div im1") + _test(lambda x: x * SPOT_IMG_URL, "im1 mult im1") + _test(lambda x: SPOT_IMG_URL + x, "im1 + im1") + _test(lambda x: SPOT_IMG_URL - x, "im1 - im1") + _test(lambda x: SPOT_IMG_URL / x, "im1 div im1") + _test(lambda x: SPOT_IMG_URL * x, "im1 mult im1") _test(lambda x: x + 2, "im1 + {2;2;2;2}") _test(lambda x: x - 2, "im1 - {2;2;2;2}") _test(lambda x: x / 2, "0.5 * im1") @@ -354,8 +366,8 @@ def test_summarize_output(): def test_summarize_strip_output(): - in_fn = FILEPATH - in_fn_w_ext = FILEPATH + "?&skipcarto=1" + in_fn = SPOT_IMG_URL + in_fn_w_ext = SPOT_IMG_URL + "?&skipcarto=1" out_fn = "/tmp/output.tif" out_fn_w_ext = out_fn + "?&box=10:10:10:10" @@ -394,7 +406,7 @@ def test_summarize_consistency(): - summary of the app after write() Then we check that both only differ with the output parameter """ - app = app_fn(inp=FILEPATH) + app = app_fn(inp=SPOT_IMG_URL) out_file = "/dev/shm/out.tif" out_key = app.output_image_key summary_wo_wrt = pyotb.summarize(app) @@ -408,20 +420,18 @@ def test_summarize_consistency(): _test(app_fn) -@pytest.mark.xfail -def test_pipeline_simple(): +def test_summary_pipeline_simple(): # BandMath -> OrthoRectification -> ManageNoData - app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"}) - app2 = pyotb.OrthoRectification({"io.in": app1}) + app1 = pyotb.OrthoRectification({"io.in": SPOT_IMG_URL}) + app2 = pyotb.BandMath({"il": [app1], "exp": "im1b1"}) app3 = pyotb.ManageNoData({"in": app2}) summary = pyotb.summarize(app3) assert summary == SIMPLE_SERIALIZATION -@pytest.mark.xfail -def test_pipeline_diamond(): +def test_summary_pipeline_diamond(): # Diamond graph - app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"}) + app1 = pyotb.BandMath({"il": [SPOT_IMG_URL], "exp": "im1b1"}) app2 = pyotb.OrthoRectification({"io.in": app1}) app3 = pyotb.ManageNoData({"in": app2}) app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"}) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index eb3b51f..0b52ea2 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -2,7 +2,7 @@ import os import itertools import pytest import pyotb -from tests_data import INPUT, FILEPATH +from tests_data import INPUT, SPOT_IMG_URL # List of buildings blocks, we can add other pyotb objects here @@ -79,7 +79,7 @@ def pipeline2str(pipeline): def make_pipelines_list(): """Create a list of pipelines using different lengths and blocks""" - blocks = {FILEPATH: OTBAPPS_BLOCKS, INPUT: ALL_BLOCKS} # for filepath, we can't use Slicer or Operation + blocks = {SPOT_IMG_URL: OTBAPPS_BLOCKS, INPUT: ALL_BLOCKS} # for filepath, we can't use Slicer or Operation pipelines = [] names = [] for inp, blocks in blocks.items(): diff --git a/tests/tests_data.py b/tests/tests_data.py index 359081d..983b190 100644 --- a/tests/tests_data.py +++ b/tests/tests_data.py @@ -1,16 +1,12 @@ import json from pathlib import Path -import requests import pyotb -FILEPATH = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" -response = requests.get(FILEPATH, timeout=5) -code = response.status_code -if code != 200: - raise requests.HTTPError(f"Unable to fetch remote image, GitLab might be offline (HTTP {code}).") +SPOT_IMG_URL = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" +PLEIADES_IMG_URL = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Baseline/OTB/Images/prTvOrthoRectification_pleiades-1_noDEM.tif" -INPUT = pyotb.Input(FILEPATH) +INPUT = pyotb.Input(SPOT_IMG_URL) TEST_IMAGE_STATS = { 'out.mean': [79.5505, 109.225, 115.456, 249.349], 'out.min': [33, 64, 91, 47], @@ -18,7 +14,7 @@ TEST_IMAGE_STATS = { 'out.std': [51.0754, 35.3152, 23.4514, 20.3827] } -json_file = Path(__file__).parent / "serialized_apps.json" +json_file = Path(__file__).parent / "pipeline_summary.json" with json_file.open("r", encoding="utf-8") as js: data = json.load(js) SIMPLE_SERIALIZATION = data["SIMPLE"] -- GitLab From 1249e4a981f55d3e816c502f64de1bd1cabc29da Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 3 Aug 2023 20:43:02 +0000 Subject: [PATCH 280/399] CI: Refactor tests --- .gitlab-ci.yml | 21 +- tests/pipeline_summary.json | 2 +- tests/test_core.py | 458 ++++++++++++++++-------------------- tests/test_pipeline.py | 86 +++++-- tests/tests_data.py | 13 +- 5 files changed, 284 insertions(+), 296 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 18cca61..2c6e521 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -61,6 +61,13 @@ codespell: - codespell {pyotb,tests,doc,README.md} # -------------------------------------- Tests -------------------------------------- +test_install: + stage: Tests + only: + - tags + allow_failure: false + script: + - pip install . .tests: stage: Tests @@ -72,20 +79,16 @@ codespell: variables: OTB_ROOT: /opt/otb LD_LIBRARY_PATH: /opt/otb/lib - OTB_LOGGER_LEVEL: INFO - PYOTB_LOGGER_LEVEL: DEBUG SPOT_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif PLEIADES_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Baseline/OTB/Images/prTvOrthoRectification_pleiades-1_noDEM.tif before_script: - pip install pytest pytest-cov -test_install: - extends: .tests - script: - - pip install . - test_module_core: extends: .tests + variables: + OTB_LOGGER_LEVEL: INFO + PYOTB_LOGGER_LEVEL: DEBUG artifacts: reports: junit: test-module-core.xml @@ -100,6 +103,9 @@ test_module_core: test_pipeline_permutations: extends: .tests + variables: + OTB_LOGGER_LEVEL: WARNING + PYOTB_LOGGER_LEVEL: INFO artifacts: reports: junit: test-pipeline-permutations.xml @@ -111,7 +117,6 @@ test_pipeline_permutations: docs: stage: Documentation - # when: manual rules: - changes: - mkdocs.yml diff --git a/tests/pipeline_summary.json b/tests/pipeline_summary.json index 8c598ae..56ebf8d 100644 --- a/tests/pipeline_summary.json +++ b/tests/pipeline_summary.json @@ -39,7 +39,7 @@ } } }, - "COMPLEX": { + "DIAMOND": { "name": "BandMathX", "parameters": { "il": [ diff --git a/tests/test_core.py b/tests/test_core.py index 8d54003..82173cd 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -4,22 +4,11 @@ import numpy as np from tests_data import * -def test_parameters(): +def test_app_parameters(): # Input / ExtractROI assert INPUT.parameters assert INPUT.parameters["in"] == SPOT_IMG_URL assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304) - # ManageNoData - app = pyotb.ManageNoData(INPUT) - assert "usenan" in app._auto_parameters - assert "mode.buildmask.inv" in app._auto_parameters - # OpticalCalibration - inp = pyotb.Input(PLEIADES_IMG_URL) - app = pyotb.OpticalCalibration(inp, level="toa") - assert "milli" in app._auto_parameters - assert "clamp" in app._auto_parameters - assert app._auto_parameters["acqui.year"] == 2012 - assert app._auto_parameters["acqui.sun.elev"] == 23.836299896240234 # OrthoRectification app = pyotb.OrthoRectification(INPUT) assert isinstance(app.parameters["map"], str) @@ -29,14 +18,37 @@ def test_parameters(): assert app.parameters["map"] == "epsg" assert "map" in app._settings and "map" not in app._auto_parameters assert app.parameters["map.epsg.code"] == app.app.GetParameters()["map.epsg.code"] - - -def test_param_with_underscore(): + # Orthorectification with underscore kwargs app = pyotb.OrthoRectification(io_in=INPUT, map_epsg_code=2154) assert app.parameters["map.epsg.code"] == 2154 + # ManageNoData + app = pyotb.ManageNoData(INPUT) + assert "usenan" in app._auto_parameters + assert "mode.buildmask.inv" in app._auto_parameters + # OpticalCalibration + app = pyotb.OpticalCalibration(pyotb.Input(PLEIADES_IMG_URL), level="toa") + assert "milli" in app._auto_parameters + assert "clamp" in app._auto_parameters + assert app._auto_parameters["acqui.year"] == 2012 + assert app._auto_parameters["acqui.sun.elev"] == 23.836299896240234 + + +def test_app_properties(): + assert INPUT.input_key == INPUT.input_image_key == "in" + assert INPUT.output_key == INPUT.output_image_key == "out" + with pytest.raises(KeyError): + pyotb.BandMath(INPUT, expression="im1b1") + # Test user can set custom name + app = pyotb.App("BandMath", [INPUT], exp="im1b1", name="TestName") + assert app.name == "TestName" + # Test data dict is not empty + app = pyotb.ReadImageInfo(INPUT) + assert app.data + # Test elapsed time is not null + assert 0 < app.elapsed_time < 1 -def test_input_vsi(): +def test_app_input_vsi(): # Simple remote file info = pyotb.ReadImageInfo("https://fake.com/image.tif", frozen=True) assert info.app.GetParameterValue("in") == "/vsicurl/https://fake.com/image.tif" @@ -58,56 +70,27 @@ def test_input_vsi(): for ziped_tif_url in ziped_tif_urls: info = pyotb.ReadImageInfo(ziped_tif_url) assert info["sizex"] == 20 - - -def test_input_vsi_from_user(): # Ensure old way is still working: ExtractROI will raise RuntimeError if a path is malformed pyotb.Input("/vsicurl/" + SPOT_IMG_URL) -def test_wrong_key(): - with pytest.raises(KeyError): - pyotb.BandMath(INPUT, expression="im1b1") - - -# OTBObject properties -def test_name(): - app = pyotb.App("BandMath", [INPUT], exp="im1b1", name="TestName") - assert app.name == "TestName" - - -def test_key_input(): - assert INPUT.input_key == INPUT.input_image_key == "in" - - -def test_key_output(): - assert INPUT.output_image_key == "out" - - -def test_dtype(): +def test_img_properties(): assert INPUT.dtype == "uint8" - - -def test_shape(): assert INPUT.shape == (304, 251, 4) - - -def test_transform(): assert INPUT.transform == (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0) + with pytest.raises(TypeError): + assert pyotb.ReadImageInfo(INPUT).dtype == "uint8" -def test_data(): - assert pyotb.ComputeImagesStatistics(INPUT).data == TEST_IMAGE_STATS - - -def test_metadata(): - INPUT2 = pyotb.Input( +def test_img_metadata(): + assert "ProjectionRef" in INPUT.metadata + assert "TIFFTAG_SOFTWARE" in INPUT.metadata + inp2 = pyotb.Input( "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/" "47/Q/RU/2021/12/S2B_47QRU_20211227_0_L2A/B04.tif" ) - assert "ProjectionRef", "TIFFTAG_SOFTWARE" in INPUT.metadata - assert "ProjectionRef", "OVR_RESAMPLING_ALG" in INPUT2.metadata - + assert "ProjectionRef" in inp2.metadata + assert "OVR_RESAMPLING_ALG" in inp2.metadata # Metadata with numeric values (e.g. TileHintX) fp = ( "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/" @@ -117,16 +100,24 @@ def test_metadata(): assert "TileHintX" in app.metadata -def test_nonraster_property(): - with pytest.raises(TypeError): - assert pyotb.ReadImageInfo(INPUT).dtype == "uint8" +def test_essential_apps(): + readimageinfo = pyotb.ReadImageInfo(INPUT, quiet=True) + assert (readimageinfo["sizex"], readimageinfo["sizey"]) == (251, 304) + assert readimageinfo["numberbands"] == 4 + computeimagestats = pyotb.ComputeImagesStatistics([INPUT], quiet=True) + assert computeimagestats["out.min"] == TEST_IMAGE_STATS["out.min"] + slicer_computeimagestats = pyotb.ComputeImagesStatistics( + il=[INPUT[:10, :10, 0]], quiet=True + ) + assert slicer_computeimagestats["out.min"] == [180] -def test_elapsed_time(): - assert 0 < pyotb.ReadImageInfo(INPUT).elapsed_time < 1 +def test_get_statistics(): + stats_data = pyotb.ComputeImagesStatistics(INPUT).data + assert stats_data == TEST_IMAGE_STATS + assert INPUT.get_statistics() == TEST_IMAGE_STATS -# Other functions def test_get_info(): infos = INPUT.get_info() assert (infos["sizex"], infos["sizey"]) == (251, 304) @@ -134,8 +125,9 @@ def test_get_info(): assert infos == bm_infos -def test_get_statistics(): - assert INPUT.get_statistics() == TEST_IMAGE_STATS +def test_read_values_at_coords(): + assert INPUT[0, 0, 0] == 180 + assert INPUT[10, 20, :] == [207, 192, 172, 255] def test_xy_to_rowcol(): @@ -143,50 +135,59 @@ def test_xy_to_rowcol(): def test_write(): - assert INPUT.write("/tmp/test_write.tif", ext_fname="nodata=0") + assert INPUT.write("/dev/shm/test_write.tif", ext_fname="nodata=0") INPUT["out"].filepath.unlink() + # Frozen + frozen_app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True) + assert frozen_app.write("/dev/shm/test_frozen_app_write.tif") + frozen_app["out"].filepath.unlink() + frozen_app_init_with_outfile = pyotb.BandMath( + INPUT, exp="im1b1", out="/dev/shm/test_frozen_app_write.tif", frozen=True + ) + assert frozen_app_init_with_outfile.write() + frozen_app_init_with_outfile["out"].filepath.unlink() -def test_ext_fname(): +def test_write_ext_fname(): def _check(expected: str, key: str = "out", app=INPUT.app): fn = app.GetParameterString(key) assert "?&" in fn assert fn.split("?&", 1)[1] == expected - assert INPUT.write("/tmp/test_write.tif", ext_fname="nodata=0") + assert INPUT.write("/dev/shm/test_write.tif", ext_fname="nodata=0") _check("nodata=0") - assert INPUT.write("/tmp/test_write.tif", ext_fname={"nodata": "0"}) + assert INPUT.write("/dev/shm/test_write.tif", ext_fname={"nodata": "0"}) _check("nodata=0") - assert INPUT.write("/tmp/test_write.tif", ext_fname={"nodata": 0}) + assert INPUT.write("/dev/shm/test_write.tif", ext_fname={"nodata": 0}) _check("nodata=0") assert INPUT.write( - "/tmp/test_write.tif", ext_fname={"nodata": 0, "gdal:co:COMPRESS": "DEFLATE"} + "/dev/shm/test_write.tif", + ext_fname={"nodata": 0, "gdal:co:COMPRESS": "DEFLATE"}, ) _check("nodata=0&gdal:co:COMPRESS=DEFLATE") assert INPUT.write( - "/tmp/test_write.tif", ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE" + "/dev/shm/test_write.tif", ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE" ) _check("nodata=0&gdal:co:COMPRESS=DEFLATE") assert INPUT.write( - "/tmp/test_write.tif?&box=0:0:10:10", + "/dev/shm/test_write.tif?&box=0:0:10:10", ext_fname={"nodata": "0", "gdal:co:COMPRESS": "DEFLATE", "box": "0:0:20:20"}, ) # Check that the bbox is the one specified in the filepath, not the one # specified in `ext_filename` _check("nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:10:10") assert INPUT.write( - "/tmp/test_write.tif?&box=0:0:10:10", + "/dev/shm/test_write.tif?&box=0:0:10:10", ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:20:20", ) _check("nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:10:10") - INPUT["out"].filepath.unlink() - mss = pyotb.MeanShiftSmoothing(SPOT_IMG_URL) + mss = pyotb.MeanShiftSmoothing(INPUT) mss.write( { - "fout": "/tmp/test_ext_fn_fout.tif?&nodata=1", - "foutpos": "/tmp/test_ext_fn_foutpos.tif?&nodata=2", + "fout": "/dev/shm/test_ext_fn_fout.tif?&nodata=1", + "foutpos": "/dev/shm/test_ext_fn_foutpos.tif?&nodata=2", }, ext_fname={"nodata": 0, "gdal:co:COMPRESS": "DEFLATE"}, ) @@ -196,144 +197,89 @@ def test_ext_fname(): mss["foutpos"].filepath.unlink() -def test_frozen_app_write(): - app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True) - assert app.write("/tmp/test_frozen_app_write.tif") - app["out"].filepath.unlink() - - app = pyotb.BandMath( - INPUT, exp="im1b1", out="/tmp/test_frozen_app_write.tif", frozen=True - ) - assert app.write() - app["out"].filepath.unlink() - - -def test_output_write(): - assert INPUT["out"].write("/tmp/test_output_write.tif") +def test_output(): + assert INPUT["out"].write("/dev/shm/test_output_write.tif") INPUT["out"].filepath.unlink() - - -def test_frozen_output_write(): - app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True) - assert app["out"].write("/tmp/test_frozen_app_write.tif") - app["out"].filepath.unlink() - - -def test_output_in_arg(): - info = pyotb.ReadImageInfo(INPUT["out"]) - assert info.data + frozen_app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True) + assert frozen_app["out"].write("/dev/shm/test_frozen_app_write.tif") + frozen_app["out"].filepath.unlink() + info_from_output_obj = pyotb.ReadImageInfo(INPUT["out"]) + assert info_from_output_obj.data # Slicer -def test_slicer_shape(): - extract = INPUT[:50, :60, :3] - assert extract.shape == (50, 60, 3) - assert extract.parameters["cl"] == ["Channel1", "Channel2", "Channel3"] - - -def test_slicer_preserve_dtype(): - extract = INPUT[:50, :60, :3] - assert extract.dtype == "uint8" - - -def test_slicer_negative_band_index(): - assert INPUT[:50, :60, :-2].shape == (50, 60, 2) - - -def test_slicer_in_output(): - slc = pyotb.BandMath([INPUT], exp="im1b1")["out"][:50, :60, :-2] - assert isinstance(slc, pyotb.core.Slicer) - - -# Arithmetic -def test_rational_operators(): - def _test(func, exp): - meas = func(INPUT) - ref = pyotb.BandMathX({"il": [SPOT_IMG_URL], "exp": exp}) - for i in range(1, 5): - compared = pyotb.CompareImages( - {"ref.in": ref, "meas.in": meas, "ref.channel": i, "meas.channel": i} - ) - assert (compared["count"], compared["mse"]) == (0, 0) - - _test(lambda x: x + x, "im1 + im1") - _test(lambda x: x - x, "im1 - im1") - _test(lambda x: x / x, "im1 div im1") - _test(lambda x: x * x, "im1 mult im1") - _test(lambda x: x + SPOT_IMG_URL, "im1 + im1") - _test(lambda x: x - SPOT_IMG_URL, "im1 - im1") - _test(lambda x: x / SPOT_IMG_URL, "im1 div im1") - _test(lambda x: x * SPOT_IMG_URL, "im1 mult im1") - _test(lambda x: SPOT_IMG_URL + x, "im1 + im1") - _test(lambda x: SPOT_IMG_URL - x, "im1 - im1") - _test(lambda x: SPOT_IMG_URL / x, "im1 div im1") - _test(lambda x: SPOT_IMG_URL * x, "im1 mult im1") - _test(lambda x: x + 2, "im1 + {2;2;2;2}") - _test(lambda x: x - 2, "im1 - {2;2;2;2}") - _test(lambda x: x / 2, "0.5 * im1") - _test(lambda x: x * 2, "im1 * 2") - _test(lambda x: x + 2.0, "im1 + {2.0;2.0;2.0;2.0}") - _test(lambda x: x - 2.0, "im1 - {2.0;2.0;2.0;2.0}") - _test(lambda x: x / 2.0, "0.5 * im1") - _test(lambda x: x * 2.0, "im1 * 2.0") - _test(lambda x: 2 + x, "{2;2;2;2} + im1") - _test(lambda x: 2 - x, "{2;2;2;2} - im1") - _test(lambda x: 2 / x, "{2;2;2;2} div im1") - _test(lambda x: 2 * x, "2 * im1") - _test(lambda x: 2.0 + x, "{2.0;2.0;2.0;2.0} + im1") - _test(lambda x: 2.0 - x, "{2.0;2.0;2.0;2.0} - im1") - _test(lambda x: 2.0 / x, "{2.0;2.0;2.0;2.0} div im1") - _test(lambda x: 2.0 * x, "2.0 * im1") - - -def test_operation(): +def test_slicer(): + sliced = INPUT[:50, :60, :3] + assert sliced.parameters["cl"] == ["Channel1", "Channel2", "Channel3"] + assert sliced.shape == (50, 60, 3) + assert sliced.dtype == "uint8" + sliced_negative_band_idx = INPUT[:50, :60, :-2] + assert sliced_negative_band_idx.shape == (50, 60, 2) + sliced_from_output = pyotb.BandMath([INPUT], exp="im1b1")["out"][:50, :60, :-2] + assert isinstance(sliced_from_output, pyotb.core.Slicer) + + +# Operation and LogicalOperation +def test_operator_expressions(): op = INPUT / 255 * 128 assert ( op.exp == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)" ) assert op.dtype == "float32" - - -def test_func_abs_expression(): assert abs(INPUT).exp == "(abs(im1b1));(abs(im1b2));(abs(im1b3));(abs(im1b4))" - - -def test_sum_bands(): - summed = sum(INPUT[:, :, b] for b in range(INPUT.shape[-1])) - assert summed.exp == "((((0 + im1b1) + im1b2) + im1b3) + im1b4)" - - -def test_binary_mask_where(): - # Create binary mask based on several possible values - values = [1, 2, 3, 4] - res = pyotb.where(pyotb.any(INPUT[:, :, 0] == value for value in values), 255, 0) - assert ( - res.exp - == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)" - ) - - -# Essential apps -def test_app_readimageinfo(): - info = pyotb.ReadImageInfo(INPUT, quiet=True) - assert (info["sizex"], info["sizey"]) == (251, 304) - assert info["numberbands"] == 4 - - -def test_app_computeimagestats(): - stats = pyotb.ComputeImagesStatistics([INPUT], quiet=True) - assert stats["out.min"] == TEST_IMAGE_STATS["out.min"] - - -def test_app_computeimagestats_sliced(): - slicer_stats = pyotb.ComputeImagesStatistics(il=[INPUT[:10, :10, 0]], quiet=True) - assert slicer_stats["out.min"] == [180] - - -def test_read_values_at_coords(): - assert INPUT[0, 0, 0] == 180 - assert INPUT[10, 20, :] == [207, 192, 172, 255] + summed_bands = sum(INPUT[:, :, b] for b in range(INPUT.shape[-1])) + assert summed_bands.exp == "((((0 + im1b1) + im1b2) + im1b3) + im1b4)" + + +def operation_test(func, exp): + meas = func(INPUT) + ref = pyotb.BandMathX({"il": [SPOT_IMG_URL], "exp": exp}) + for i in range(1, 5): + compared = pyotb.CompareImages( + {"ref.in": ref, "meas.in": meas, "ref.channel": i, "meas.channel": i} + ) + assert (compared["count"], compared["mse"]) == (0, 0) + + +def test_operation_add(): + operation_test(lambda x: x + x, "im1 + im1") + operation_test(lambda x: x + INPUT, "im1 + im1") + operation_test(lambda x: INPUT + x, "im1 + im1") + operation_test(lambda x: x + 2, "im1 + {2;2;2;2}") + operation_test(lambda x: x + 2.0, "im1 + {2.0;2.0;2.0;2.0}") + operation_test(lambda x: 2 + x, "{2;2;2;2} + im1") + operation_test(lambda x: 2.0 + x, "{2.0;2.0;2.0;2.0} + im1") + + +def test_operation_sub(): + operation_test(lambda x: x - x, "im1 - im1") + operation_test(lambda x: x - INPUT, "im1 - im1") + operation_test(lambda x: INPUT - x, "im1 - im1") + operation_test(lambda x: x - 2, "im1 - {2;2;2;2}") + operation_test(lambda x: x - 2.0, "im1 - {2.0;2.0;2.0;2.0}") + operation_test(lambda x: 2 - x, "{2;2;2;2} - im1") + operation_test(lambda x: 2.0 - x, "{2.0;2.0;2.0;2.0} - im1") + + +def test_operation_mult(): + operation_test(lambda x: x * x, "im1 mult im1") + operation_test(lambda x: x * INPUT, "im1 mult im1") + operation_test(lambda x: INPUT * x, "im1 mult im1") + operation_test(lambda x: x * 2, "im1 * 2") + operation_test(lambda x: x * 2.0, "im1 * 2.0") + operation_test(lambda x: 2 * x, "2 * im1") + operation_test(lambda x: 2.0 * x, "2.0 * im1") + + +def test_operation_div(): + operation_test(lambda x: x / x, "im1 div im1") + operation_test(lambda x: x / INPUT, "im1 div im1") + operation_test(lambda x: INPUT / x, "im1 div im1") + operation_test(lambda x: x / 2, "im1 * 0.5") + operation_test(lambda x: x / 2.0, "im1 * 0.5") + operation_test(lambda x: 2 / x, "{2;2;2;2} div im1") + operation_test(lambda x: 2.0 / x, "{2.0;2.0;2.0;2.0} div im1") # BandMath NDVI == RadiometricIndices NDVI ? @@ -345,11 +291,11 @@ def test_ndvi_comparison(): INPUT, {"list": ["Vegetation:NDVI"], "channels.red": 1, "channels.nir": 4} ) assert ndvi_bandmath.exp == "((im1b4 - im1b1) / (im1b4 + im1b1))" - assert ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", "float") - assert ndvi_indices.write("/tmp/ndvi_indices.tif", "float") + assert ndvi_bandmath.write("/dev/shm/ndvi_bandmath.tif", "float") + assert ndvi_indices.write("/dev/shm/ndvi_indices.tif", "float") compared = pyotb.CompareImages( - {"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"} + {"ref.in": ndvi_indices, "meas.in": "/dev/shm/ndvi_bandmath.tif"} ) assert (compared["count"], compared["mse"]) == (0, 0) thresholded_indices = pyotb.where(ndvi_indices >= 0.3, 1, 0) @@ -361,14 +307,43 @@ def test_ndvi_comparison(): ) -def test_summarize_output(): +# Tests for functions.py +def test_binary_mask_where(): + # Create binary mask based on several possible values + values = [1, 2, 3, 4] + res = pyotb.where(pyotb.any(INPUT[:, :, 0] == value for value in values), 255, 0) + assert ( + res.exp + == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)" + ) + + +# Tests for summarize() +def test_summarize_pipeline_simple(): + app1 = pyotb.OrthoRectification({"io.in": SPOT_IMG_URL}) + app2 = pyotb.BandMath({"il": [app1], "exp": "im1b1"}) + app3 = pyotb.ManageNoData({"in": app2}) + summary = pyotb.summarize(app3) + assert SIMPLE_SERIALIZATION == summary + + +def test_summarize_pipeline_diamond(): + app1 = pyotb.BandMath({"il": [SPOT_IMG_URL], "exp": "im1b1"}) + app2 = pyotb.OrthoRectification({"io.in": app1}) + app3 = pyotb.ManageNoData({"in": app2}) + app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"}) + summary = pyotb.summarize(app4) + assert DIAMOND_SERIALIZATION == summary + + +def test_summarize_output_obj(): assert pyotb.summarize(INPUT["out"]) def test_summarize_strip_output(): in_fn = SPOT_IMG_URL in_fn_w_ext = SPOT_IMG_URL + "?&skipcarto=1" - out_fn = "/tmp/output.tif" + out_fn = "/dev/shm/output.tif" out_fn_w_ext = out_fn + "?&box=10:10:10:10" baseline = [ @@ -399,7 +374,7 @@ def test_summarize_consistency(): lambda inp: pyotb.OrthoRectification({"io.in": inp}), ] - def _test(app_fn): + def operator_test(app_fn): """ Here we create 2 summaries: - summary of the app before write() @@ -417,75 +392,42 @@ def test_summarize_consistency(): assert summary_wo_wrt == summay_w_wrt for app_fn in app_fns: - _test(app_fn) - - -def test_summary_pipeline_simple(): - # BandMath -> OrthoRectification -> ManageNoData - app1 = pyotb.OrthoRectification({"io.in": SPOT_IMG_URL}) - app2 = pyotb.BandMath({"il": [app1], "exp": "im1b1"}) - app3 = pyotb.ManageNoData({"in": app2}) - summary = pyotb.summarize(app3) - assert summary == SIMPLE_SERIALIZATION + operator_test(app_fn) -def test_summary_pipeline_diamond(): - # Diamond graph - app1 = pyotb.BandMath({"il": [SPOT_IMG_URL], "exp": "im1b1"}) - app2 = pyotb.OrthoRectification({"io.in": app1}) - app3 = pyotb.ManageNoData({"in": app2}) - app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"}) - summary = pyotb.summarize(app4) - assert summary == COMPLEX_SERIALIZATION - - -# Numpy funcs -def test_export(): +# Numpy tests +def test_numpy_exports_dic(): INPUT.export() - array = INPUT.exports_dic[INPUT.output_image_key]["array"] - assert isinstance(array, np.ndarray) - assert array.dtype == "uint8" + exported_array = INPUT.exports_dic[INPUT.output_image_key]["array"] + assert isinstance(exported_array, np.ndarray) + assert exported_array.dtype == "uint8" del INPUT.exports_dic["out"] - - -def test_output_export(): INPUT["out"].export() assert INPUT["out"].output_image_key in INPUT["out"].exports_dic -def test_to_numpy(): +def test_numpy_conversions(): array = INPUT.to_numpy() assert array.dtype == np.uint8 assert array.shape == INPUT.shape - assert array.min() == 33 - assert array.max() == 255 - - -def test_to_numpy_sliced(): + assert (array.min(), array.max()) == (33, 255) + # Sliced img to array sliced = INPUT[:100, :200, :3] - array = sliced.to_numpy() - assert array.dtype == np.uint8 - assert array.shape == (100, 200, 3) - - -def test_convert_to_array(): - array = np.array(INPUT) - assert isinstance(array, np.ndarray) - assert INPUT.shape == array.shape - - -def test_pixel_coords_otb_equals_numpy(): + sliced_array = sliced.to_numpy() + assert sliced_array.dtype == np.uint8 + assert sliced_array.shape == (100, 200, 3) + # Test auto convert to numpy + assert isinstance(np.array(INPUT), np.ndarray) + assert INPUT.shape == np.array(INPUT).shape assert INPUT[19, 7] == list(INPUT.to_numpy()[19, 7]) - - -def test_add_noise_array(): + # Add noise test from the docs white_noise = np.random.normal(0, 50, size=INPUT.shape) noisy_image = INPUT + white_noise assert isinstance(noisy_image, pyotb.core.App) assert noisy_image.shape == INPUT.shape -def test_to_rasterio(): +def test_numpy_to_rasterio(): array, profile = INPUT.to_rasterio() assert array.dtype == profile["dtype"] == np.uint8 assert array.shape == (4, 304, 251) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 0b52ea2..5646674 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -74,12 +74,18 @@ def pipeline2str(pipeline): a string """ - return " > ".join([INPUT.__class__.__name__] + [f"{i}.{app.name.split()[0]}" for i, app in enumerate(pipeline)]) + return " > ".join( + [INPUT.__class__.__name__] + + [f"{i}.{app.name.split()[0]}" for i, app in enumerate(pipeline)] + ) def make_pipelines_list(): """Create a list of pipelines using different lengths and blocks""" - blocks = {SPOT_IMG_URL: OTBAPPS_BLOCKS, INPUT: ALL_BLOCKS} # for filepath, we can't use Slicer or Operation + blocks = { + SPOT_IMG_URL: OTBAPPS_BLOCKS, + INPUT: ALL_BLOCKS, + } # for filepath, we can't use Slicer or Operation pipelines = [] names = [] for inp, blocks in blocks.items(): @@ -96,49 +102,83 @@ def make_pipelines_list(): return pipelines, names -PIPELINES, NAMES = make_pipelines_list() - - -@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES) -def test_pipeline_shape(pipe): +def shape(pipe): for app in pipe: - assert bool(app.shape) + yield bool(app.shape) -@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES) -def test_pipeline_shape_nointermediate(pipe): +def shape_nointermediate(pipe): app = [pipe[-1]][0] - assert bool(app.shape) + yield bool(app.shape) -@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES) -def test_pipeline_shape_backward(pipe): +def shape_backward(pipe): for app in reversed(pipe): - assert bool(app.shape) + yield bool(app.shape) -@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES) -def test_pipeline_write(pipe): +def write(pipe): for i, app in enumerate(pipe): out = f"/tmp/out_{i}.tif" if os.path.isfile(out): os.remove(out) - assert app.write(out) + yield app.write(out) -@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES) -def test_pipeline_write_nointermediate(pipe): +def write_nointermediate(pipe): app = [pipe[-1]][0] out = "/tmp/out_0.tif" if os.path.isfile(out): os.remove(out) - assert app.write(out) + yield app.write(out) -@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES) -def test_pipeline_write_backward(pipe): +def write_backward(pipe): for i, app in enumerate(reversed(pipe)): out = f"/tmp/out_{i}.tif" if os.path.isfile(out): os.remove(out) - assert app.write(out) + yield app.write(out) + + +funcs = [ + shape, + shape_nointermediate, + shape_backward, + write, + write_nointermediate, + write_backward, +] + +PIPELINES, NAMES = make_pipelines_list() + + +@pytest.mark.parametrize("test_func", funcs) +def test(test_func): + fname = test_func.__name__ + successes, failures = 0, 0 + total_successes = [] + for pipeline, blocks in zip(PIPELINES, NAMES): + err = None + try: + # Here we count non-empty shapes or write results, no errors during exec + bool_tests = list(test_func(pipeline)) + overall_success = all(bool_tests) + except Exception as e: + # Unexpected exception in the pipeline, e.g. a RuntimeError we missed + bool_tests = [] + overall_success = False + err = e + if overall_success: + print(f"\033[92m{fname}: success with [{blocks}]\033[0m\n") + successes += 1 + else: + print(f"\033[91m{fname}: failure with [{blocks}]\033[0m {bool_tests}\n") + if err: + print(f"exception thrown: {err}") + failures += 1 + total_successes.append(overall_success) + print(f"\nEnded test {fname} with {successes} successes, {failures} failures") + if err: + raise err + assert all(total_successes) diff --git a/tests/tests_data.py b/tests/tests_data.py index 983b190..2e40c95 100644 --- a/tests/tests_data.py +++ b/tests/tests_data.py @@ -5,17 +5,18 @@ import pyotb SPOT_IMG_URL = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" PLEIADES_IMG_URL = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Baseline/OTB/Images/prTvOrthoRectification_pleiades-1_noDEM.tif" - INPUT = pyotb.Input(SPOT_IMG_URL) + + TEST_IMAGE_STATS = { - 'out.mean': [79.5505, 109.225, 115.456, 249.349], - 'out.min': [33, 64, 91, 47], - 'out.max': [255, 255, 230, 255], - 'out.std': [51.0754, 35.3152, 23.4514, 20.3827] + "out.mean": [79.5505, 109.225, 115.456, 249.349], + "out.min": [33, 64, 91, 47], + "out.max": [255, 255, 230, 255], + "out.std": [51.0754, 35.3152, 23.4514, 20.3827], } json_file = Path(__file__).parent / "pipeline_summary.json" with json_file.open("r", encoding="utf-8") as js: data = json.load(js) SIMPLE_SERIALIZATION = data["SIMPLE"] -COMPLEX_SERIALIZATION = data["COMPLEX"] +DIAMOND_SERIALIZATION = data["DIAMOND"] -- GitLab From 79faeb63a1ba8028517b965726d75b0bb68d7d27 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 5 Aug 2023 17:40:38 +0200 Subject: [PATCH 281/399] STYLE: summarize type hints #111 --- pyotb/core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 71a032c..a39e0cf 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1712,10 +1712,10 @@ def get_out_images_param_keys(app: OTBObject) -> list[str]: def summarize( - obj: App | Output | Any, - strip_input_paths: bool = False, - strip_output_paths: bool = False, -) -> dict[str, str | dict[str, Any]]: + obj: App | Output | str | float | list, + strip_inpath: bool = False, + strip_outpath: bool = False, +) -> dict[str, dict | Any] | str | float | list: """Recursively summarize parameters of an App or Output object and its parents. Args: -- GitLab From 2919258acb2062e1f81607376b5c5aa8c9fdd3b3 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 5 Aug 2023 17:41:06 +0200 Subject: [PATCH 282/399] DOC: better summarize() docstring --- pyotb/core.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index a39e0cf..acf72a1 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1718,25 +1718,23 @@ def summarize( ) -> dict[str, dict | Any] | str | float | list: """Recursively summarize parameters of an App or Output object and its parents. + At the deepest recursion level, this function just return any parameter value, + path stripped if needed, or app summarized in case of a pipeline. + Args: - obj: input object to summarize - strip_input_paths: strip all input paths: If enabled, paths related to + obj: input object / parameter value to summarize + strip_inpath: strip all input paths: If enabled, paths related to inputs are truncated after the first "?" character. Can be useful to remove URLs tokens (e.g. SAS or S3 credentials). - strip_output_paths: strip all output paths: If enabled, paths related + strip_outpath: strip all output paths: If enabled, paths related to outputs are truncated after the first "?" character. Can be useful to remove extended filenames. Returns: - nested dictionary with serialized App(s) containing name and parameters of an app and its parents + nested dictionary containing name and parameters of an app and its parents """ - - def strip_path(param: str | Any): - if not isinstance(param, str): - return summarize(param) - return param.split("?")[0] - + # This is the deepest recursion level if isinstance(obj, list): return [summarize(o) for o in obj] if isinstance(obj, Output): -- GitLab From 39057f6ec3fff8d926d2ca19ed32b754658f1f79 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 5 Aug 2023 17:42:43 +0200 Subject: [PATCH 283/399] ENH: docstrings and readability of summarize(), shorter param names --- pyotb/core.py | 1 + tests/test_core.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index acf72a1..54017d1 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1742,6 +1742,7 @@ def summarize( if not isinstance(obj, App): return obj + if not isinstance(param, str): parameters = {} for key, param in obj.parameters.items(): if ( diff --git a/tests/test_core.py b/tests/test_core.py index 82173cd..d03f31f 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -348,9 +348,9 @@ def test_summarize_strip_output(): baseline = [ (in_fn, out_fn_w_ext, "out", {}, out_fn_w_ext), - (in_fn, out_fn_w_ext, "out", {"strip_output_paths": True}, out_fn), + (in_fn, out_fn_w_ext, "out", {"strip_outpath": True}, out_fn), (in_fn_w_ext, out_fn, "in", {}, in_fn_w_ext), - (in_fn_w_ext, out_fn, "in", {"strip_input_paths": True}, in_fn), + (in_fn_w_ext, out_fn, "in", {"strip_inpath": True}, in_fn), ] for inp, out, key, extra_args, expected in baseline: -- GitLab From 3753779ccdc74b0d0c8555d022bcafa7e3a8c747 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 5 Aug 2023 18:06:22 +0200 Subject: [PATCH 284/399] FIX: incomplete commit --- pyotb/core.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 54017d1..ae6c13b 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1742,20 +1742,19 @@ def summarize( if not isinstance(obj, App): return obj + # Call / top level of recursion : obj is an App + def strip_path(param: str | Any): + if isinstance(param, list): + return [strip_path(p) for p in param] if not isinstance(param, str): + return summarize(param) + return param.split("?")[0] + + # We need to return parameters values, summarized if param is an App parameters = {} for key, param in obj.parameters.items(): - if ( - strip_input_paths - and obj.is_input(key) - or strip_output_paths - and obj.is_output(key) - ): - parameters[key] = ( - [strip_path(p) for p in param] - if isinstance(param, list) - else strip_path(param) - ) + if strip_inpath and obj.is_input(key) or strip_outpath and obj.is_output(key): + parameters[key] = strip_path(param) else: parameters[key] = summarize(param) return {"name": obj.app.GetName(), "parameters": parameters} -- GitLab From c67f1093ce7a1250c0107edc4c69ada78ddd1e4e Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 5 Aug 2023 18:10:15 +0200 Subject: [PATCH 285/399] STYLE: move comment --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index ae6c13b..76eef90 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1742,7 +1742,6 @@ def summarize( if not isinstance(obj, App): return obj - # Call / top level of recursion : obj is an App def strip_path(param: str | Any): if isinstance(param, list): return [strip_path(p) for p in param] @@ -1750,6 +1749,7 @@ def summarize( return summarize(param) return param.split("?")[0] + # Call / top level of recursion : obj is an App # We need to return parameters values, summarized if param is an App parameters = {} for key, param in obj.parameters.items(): -- GitLab From a76171ee95584b700fc965f253677770d604daec Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 7 Aug 2023 11:19:16 +0000 Subject: [PATCH 286/399] ENH: Automatically add vsi prefix for sub path in archived source files --- pyotb/core.py | 81 +++++++++++++++++++++++++--------------------- tests/test_core.py | 16 +++++---- 2 files changed, 55 insertions(+), 42 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 71a032c..0047dd5 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -2,6 +2,7 @@ """This module is the core of pyotb.""" from __future__ import annotations +import re from abc import ABC, abstractmethod from ast import literal_eval from pathlib import Path @@ -731,10 +732,7 @@ class App(OTBObject): ) try: if self.is_input(key): - if self.is_key_images_list(key): - self.__set_param(key, [add_vsi_prefix(p) for p in obj]) - else: - self.__set_param(key, add_vsi_prefix(obj)) + self.__set_param(key, self.__check_input_param(obj)) else: self.__set_param(key, obj) except (RuntimeError, TypeError, ValueError, KeyError) as e: @@ -969,6 +967,49 @@ class App(OTBObject): kwargs.update({self.input_key: arg}) return kwargs + def __check_input_param( + self, obj: list | OTBObject | str | Path + ) -> list | OTBObject | str: + """Check the type and value of an input param. + + Args: + obj: input parameter value + + Returns: + object, string with new /vsi prefix(es) if needed + + """ + if isinstance(obj, list): + return [self.__check_input_param(o) for o in obj] + # May be we could add some checks here + if isinstance(obj, OTBObject): + return obj + if isinstance(obj, Path): + obj = str(obj) + if isinstance(obj, str): + if not obj.startswith("/vsi"): + # Remote file. TODO: add support for S3 / GS / AZ + if obj.startswith(("https://", "http://", "ftp://")): + obj = "/vsicurl/" + obj + # Compressed file + prefixes = { + ".tar": "vsitar", + ".tar.gz": "vsitar", + ".tgz": "vsitar", + ".gz": "vsigzip", + ".7z": "vsi7z", + ".zip": "vsizip", + ".rar": "vsirar", + } + expr = r"(.*?)(\.7z|\.zip|\.rar|\.tar\.gz|\.tgz|\.tar|\.gz)(.*)" + parts = re.match(expr, obj) + if parts: + file, ext = parts.group(1), parts.group(2) + if not Path(file + ext).is_dir(): + obj = f"/{prefixes[ext]}/{obj}" + return obj + raise TypeError(f"{self.name}: wrong input parameter type ({type(obj)})") + def __set_param( self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any] ): @@ -1580,38 +1621,6 @@ class Output(OTBObject): return str(self.filepath) -def add_vsi_prefix(filepath: str | Path) -> str: - """Append vsi prefixes to file URL or path if needed. - - Args: - filepath: file path or URL - - Returns: - string with new /vsi prefix(es) - - """ - if isinstance(filepath, Path): - filepath = str(filepath) - if isinstance(filepath, str) and not filepath.startswith("/vsi"): - # Remote file. TODO: add support for S3 / GS / AZ - if filepath.startswith(("https://", "http://", "ftp://")): - filepath = "/vsicurl/" + filepath - # Compressed file - prefixes = { - ".tar": "vsitar", - ".tgz": "vsitar", - ".gz": "vsigzip", - ".7z": "vsi7z", - ".zip": "vsizip", - ".rar": "vsirar", - } - basename = filepath.split("?")[0] - ext = Path(basename).suffix - if ext in prefixes: - filepath = f"/{prefixes[ext]}/{filepath}" - return filepath - - def get_nbchannels(inp: str | Path | OTBObject) -> int: """Get the nb of bands of input image. diff --git a/tests/test_core.py b/tests/test_core.py index 82173cd..3bfc5dd 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -49,17 +49,23 @@ def test_app_properties(): def test_app_input_vsi(): + # Ensure old way is still working: ExtractROI will raise RuntimeError if a path is malformed + pyotb.Input("/vsicurl/" + SPOT_IMG_URL) # Simple remote file info = pyotb.ReadImageInfo("https://fake.com/image.tif", frozen=True) assert info.app.GetParameterValue("in") == "/vsicurl/https://fake.com/image.tif" assert info.parameters["in"] == "https://fake.com/image.tif" - # Compressed remote file - info = pyotb.ReadImageInfo("https://fake.com/image.tif.zip", frozen=True) + # Compressed single file archive + info = pyotb.ReadImageInfo("image.tif.zip", frozen=True) + assert info.app.GetParameterValue("in") == "/vsizip/image.tif.zip" + assert info.parameters["in"] == "image.tif.zip" + # File within compressed remote archive + info = pyotb.ReadImageInfo("https://fake.com/archive.tar.gz/image.tif", frozen=True) assert ( info.app.GetParameterValue("in") - == "/vsizip//vsicurl/https://fake.com/image.tif.zip" + == "/vsitar//vsicurl/https://fake.com/archive.tar.gz/image.tif" ) - assert info.parameters["in"] == "https://fake.com/image.tif.zip" + assert info.parameters["in"] == "https://fake.com/archive.tar.gz/image.tif" # Piped curl --> zip --> tiff ziped_tif_urls = ( "https://github.com/OSGeo/gdal/raw/master" @@ -70,8 +76,6 @@ def test_app_input_vsi(): for ziped_tif_url in ziped_tif_urls: info = pyotb.ReadImageInfo(ziped_tif_url) assert info["sizex"] == 20 - # Ensure old way is still working: ExtractROI will raise RuntimeError if a path is malformed - pyotb.Input("/vsicurl/" + SPOT_IMG_URL) def test_img_properties(): -- GitLab From e36e4a14ec34d3f2514c4d94e8b09c6ee1582b44 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 18:11:13 +0200 Subject: [PATCH 287/399] FIX: latest OTB version should be used when scanning user dir --- pyotb/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 5363937..aed1bd6 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -274,7 +274,7 @@ def __find_otb_root(scan_userdir: bool = False): prefix = path.parent.absolute() # If possible, use OTB found in user's HOME tree (this may take some time) if scan_userdir: - for path in Path.home().glob("**/OTB-*/lib"): + for path in sorted(Path.home().glob("**/OTB-*/lib/")): logger.info("Found %s", path.parent) prefix = path.parent.absolute() # Return latest found prefix (and version), see precedence in function def find_otb() -- GitLab From 3e9ec90763b4b38f3bf685c3dbf291e01f00ae63 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 18:11:55 +0200 Subject: [PATCH 288/399] STYLE: apply black --- pyotb/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pyotb/__init__.py b/pyotb/__init__.py index bf55392..507a408 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -3,8 +3,16 @@ __version__ = "2.0.0.dev4" from .helpers import logger, set_logger_level +from .core import ( + OTBObject, + App, + Input, + Output, + get_nbchannels, + get_pixel_type, + summarize, +) from .apps import * -from .core import App, Input, Output, get_nbchannels, get_pixel_type, summarize, OTBObject from .functions import ( # pylint: disable=redefined-builtin all, -- GitLab From 4ca2fc0d26ff6fd1d0ea181cae1d3ce249064663 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 18:22:20 +0200 Subject: [PATCH 289/399] ADD: function to install OTB (in interactive mode only) --- pyotb/helpers.py | 118 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 113 insertions(+), 5 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index aed1bd6..825dd9b 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -1,13 +1,19 @@ # -*- coding: utf-8 -*- """This module helps to ensure we properly initialize pyotb: only in case OTB is found and apps are available.""" +import json import logging import os +import re +import subprocess import sys +import tempfile +import urllib.request from pathlib import Path from shutil import which # Allow user to switch between OTB directories without setting every env variable OTB_ROOT = os.environ.get("OTB_ROOT") +DOCS_URL = "https://www.orfeo-toolbox.org/CookBook/Installation.html" # Logging # User can also get logger with `logging.getLogger("pyOTB")` @@ -90,6 +96,22 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru # Else search system logger.info("Failed to import OTB. Searching for it...") prefix = __find_otb_root(scan_userdir) + if not prefix: + if hasattr(sys, "ps1"): + if input("OTB is missing. Do you want to install it ? (y/n): ") == "y": + version = input( + "Choose a version number to install (default is latest): " + ) + path = input( + "Provide a path for installation " + "(default is <user_dir>/Applications/OTB-<version>): " + ) + return find_otb(install_otb(version, path)) + if not prefix: + raise SystemExit( + "OTB not found on disk. " + "To install it, open an interactive python shell and type 'import pyotb'" + ) # Try to import one last time before raising error try: set_environment(prefix) @@ -98,9 +120,6 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru return otb except EnvironmentError as e: raise SystemExit("Auto setup for OTB env failed. Exiting.") from e - # Unknown error - except ModuleNotFoundError as e: - raise SystemExit("Can't run without OTB installed. Exiting.") from e # Help user to fix this except ImportError as e: __suggest_fix_import(str(e), prefix) @@ -166,6 +185,96 @@ def set_environment(prefix: str): os.environ["PROJ_LIB"] = proj_lib +def otb_latest_release_tag(): + """Use gitlab API to find latest release tag name, but skip pre-releases.""" + api_endpoint = "https://gitlab.orfeo-toolbox.org/api/v4/projects/53/repository/tags" + vers_regex = re.compile(r"^\d\.\d\.\d$") + with urllib.request.urlopen(api_endpoint) as stream: + data = json.loads(stream.read()) + releases = sorted( + [tag["name"] for tag in data if vers_regex.match(tag["name"])], + ) + return releases[-1] + + +def install_otb(version: str = "latest", path: str = ""): + """Install pre-compiled OTB binaries in path, use latest release by default. + + Args: + version: OTB version tag, e.g. '8.1.2' + path: installation directory + + Returns: + full path of the new installation + """ + major = sys.version_info.major + if major == 2: + raise SystemExit( + "You need to use python3 in order to import OTB python bindings." + ) + minor = sys.version_info.minor + name_corresp = {"linux": "Linux64", "darwnin": "Darwin64", "win32": "Win64"} + sysname = name_corresp[sys.platform] + if sysname == "Win64": + cmd = which("cmd.exe") + ext = "zip" + if minor != 7: + raise SystemExit( + "Python version 3.7 is required to import python bindings on Windows." + ) + else: + cmd = which("zsh") or which("bash") or which("sh") + ext = "run" + + # Fetch archive and run installer + if not version or version == "latest": + version = otb_latest_release_tag() + filename = f"OTB-{version}-{sysname}.{ext}" + url = f"https://www.orfeo-toolbox.org/packages/archives/OTB/{filename}" + tmpdir = tempfile.gettempdir() + tmpfile = Path(tmpdir) / filename + print(f"Downloading {url}") + if not tmpfile.exists(): + urllib.request.urlretrieve(url, tmpfile) + if path: + path = Path(path) + else: + path = Path.home() / "Applications" / tmpfile.stem + install_cmd = f"{cmd} {tmpfile} --target {path} --accept" + print(f"Executing '{install_cmd}'\n") + subprocess.run(f"{cmd} {tmpfile} --target {path} --accept", shell=True, check=True) + tmpfile.unlink() + + # Add env variable to profile + if sysname != "Win64": + with open(Path.home() / ".profile", "a", encoding="utf-8") as buf: + buf.write(f'\n. "{path}/otbenv.profile"\n') + else: + print( + "In order to speed-up pyotb import, remember to call 'otbenv.bat' " + "before importing pyotb, or add 'OTB_ROOT=\"{path}\"' to your env variables." + ) + if ( + sysname == "Win64" + or (sysname == "Linux64" and minor == 8) + or (sysname == "Darwin64" and minor == 7) + ): + return str(path) + # Recompile bindings : this may fail because of OpenGL... + if sys.executable and which("ctest") and which("python3-config"): + print("\nRecompiling python bindings...") + ctest_cmd = ( + ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV" + ) + subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True) + return str(path) + print( + "\nYou'll need to install 'cmake', 'python3-dev' and 'libgl1-mesa-dev'" + " in order to recompile python bindings. " + ) + raise SystemExit + + def __find_lib(prefix: str = None, otb_module=None): """Try to find OTB external libraries directory. @@ -330,9 +439,8 @@ def __suggest_fix_import(error_message: str, prefix: str): "It seems that your env variables aren't properly set," " first use 'call otbenv.bat' then try to import pyotb once again" ) - docs_link = "https://www.orfeo-toolbox.org/CookBook/Installation.html" logger.critical( - "You can verify installation requirements for your OS at %s", docs_link + "You can verify installation requirements for your OS at %s", DOCS_URL ) -- GitLab From 147aa1132c26bd8562317c2d09c6bdc5453ae430 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 18:24:00 +0200 Subject: [PATCH 290/399] ADD: env variable to allow auto OTB install for non interactive mode --- pyotb/helpers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 825dd9b..942511c 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -445,4 +445,7 @@ def __suggest_fix_import(error_message: str, prefix: str): # Since helpers is the first module to be inititialized, this will prevent pyotb to run if OTB is not found -find_otb() +if os.environ.get("OTB_AUTO_INSTALL") in ("1", "true", "ON", "YES"): + find_otb(install_otb()) +else: + find_otb() -- GitLab From 1cc011b718748af13dfd85942acc98737051da67 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 18:29:15 +0200 Subject: [PATCH 291/399] ENH: inform user that .profile has been modified --- pyotb/helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 942511c..49746b0 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -247,7 +247,9 @@ def install_otb(version: str = "latest", path: str = ""): # Add env variable to profile if sysname != "Win64": - with open(Path.home() / ".profile", "a", encoding="utf-8") as buf: + profile = Path.home() / ".profile" + print(f"Adding new env variables to {profile}") + with open(profile, "a", encoding="utf-8") as buf: buf.write(f'\n. "{path}/otbenv.profile"\n') else: print( -- GitLab From 4512c08c0213d50018959a63752e6ed1dbd7650f Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 21:12:39 +0200 Subject: [PATCH 292/399] FIX: typos and add comments --- pyotb/helpers.py | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 49746b0..5d7c1a8 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -188,7 +188,7 @@ def set_environment(prefix: str): def otb_latest_release_tag(): """Use gitlab API to find latest release tag name, but skip pre-releases.""" api_endpoint = "https://gitlab.orfeo-toolbox.org/api/v4/projects/53/repository/tags" - vers_regex = re.compile(r"^\d\.\d\.\d$") + vers_regex = re.compile(r"^\d\.\d\.\d$") # we ignore rc-* or alpha-* with urllib.request.urlopen(api_endpoint) as stream: data = json.loads(stream.read()) releases = sorted( @@ -209,19 +209,17 @@ def install_otb(version: str = "latest", path: str = ""): """ major = sys.version_info.major if major == 2: - raise SystemExit( - "You need to use python3 in order to import OTB python bindings." - ) + raise SystemExit("Python 3 is required for OTB bindings.") minor = sys.version_info.minor name_corresp = {"linux": "Linux64", "darwnin": "Darwin64", "win32": "Win64"} sysname = name_corresp[sys.platform] if sysname == "Win64": - cmd = which("cmd.exe") - ext = "zip" if minor != 7: raise SystemExit( "Python version 3.7 is required to import python bindings on Windows." ) + cmd = which("cmd.exe") + ext = "zip" else: cmd = which("zsh") or which("bash") or which("sh") ext = "run" @@ -234,8 +232,7 @@ def install_otb(version: str = "latest", path: str = ""): tmpdir = tempfile.gettempdir() tmpfile = Path(tmpdir) / filename print(f"Downloading {url}") - if not tmpfile.exists(): - urllib.request.urlretrieve(url, tmpfile) + urllib.request.urlretrieve(url, tmpfile) if path: path = Path(path) else: @@ -243,7 +240,7 @@ def install_otb(version: str = "latest", path: str = ""): install_cmd = f"{cmd} {tmpfile} --target {path} --accept" print(f"Executing '{install_cmd}'\n") subprocess.run(f"{cmd} {tmpfile} --target {path} --accept", shell=True, check=True) - tmpfile.unlink() + tmpfile.unlink() # cleaning # Add env variable to profile if sysname != "Win64": @@ -254,16 +251,14 @@ def install_otb(version: str = "latest", path: str = ""): else: print( "In order to speed-up pyotb import, remember to call 'otbenv.bat' " - "before importing pyotb, or add 'OTB_ROOT=\"{path}\"' to your env variables." + f"before importing pyotb, or add 'OTB_ROOT=\"{path}\"' to your env variables." ) - if ( - sysname == "Win64" - or (sysname == "Linux64" and minor == 8) - or (sysname == "Darwin64" and minor == 7) - ): return str(path) - # Recompile bindings : this may fail because of OpenGL... - if sys.executable and which("ctest") and which("python3-config"): + # Linux requires 3.8, macOS requires 3.7 + if (sysname == "Linux64" and minor == 8) or (sysname == "Darwin64" and minor == 7): + return str(path) + # Else recompile bindings : this may fail because of OpenGL + if which("ctest") and which("python3-config"): print("\nRecompiling python bindings...") ctest_cmd = ( ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV" @@ -271,7 +266,7 @@ def install_otb(version: str = "latest", path: str = ""): subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True) return str(path) print( - "\nYou'll need to install 'cmake', 'python3-dev' and 'libgl1-mesa-dev'" + "\nYou need to install 'cmake', 'python3-dev' and 'libgl1-mesa-dev'" " in order to recompile python bindings. " ) raise SystemExit -- GitLab From e8e1672af3be1047229e041aa6704a7afd441c57 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 21:33:49 +0200 Subject: [PATCH 293/399] ENH: removed full user dir scan, only check for Applications dir --- pyotb/helpers.py | 42 +++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 5d7c1a8..6319167 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -45,19 +45,19 @@ def set_logger_level(level: str): logger_handler.setLevel(getattr(logging, level)) -def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = True): +def find_otb(prefix: str = OTB_ROOT, scan: bool = True): """Try to load OTB bindings or scan system, help user in case of failure, set env variables. - Path precedence : OTB_ROOT > python bindings directory - OR search for releases installations : HOME - OR (for Linux) : /opt/otbtf > /opt/otb > /usr/local > /usr - OR (for MacOS) : ~/Applications - OR (for Windows) : C:/Program Files + The OTB_ROOT variable allow one to override default OTB version, with auto env setting. + Path precedence : $OTB_ROOT > location of python bindings location + Then, if OTB is not found: + search for releases installations: $HOME/Applications + OR (for Linux): /opt/otbtf > /opt/otb > /usr/local > /usr + OR (for Windows): C:/Program Files Args: prefix: prefix to search OTB in (Default value = OTB_ROOT) scan: find otb in system known locations (Default value = True) - scan_userdir: search for OTB release in user's home directory (Default value = True) Returns: otbApplication module @@ -95,7 +95,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru ) from e # Else search system logger.info("Failed to import OTB. Searching for it...") - prefix = __find_otb_root(scan_userdir) + prefix = __find_otb_root() if not prefix: if hasattr(sys, "ps1"): if input("OTB is missing. Do you want to install it ? (y/n): ") == "y": @@ -341,12 +341,9 @@ def __find_apps_path(lib_dir: Path): return "" -def __find_otb_root(scan_userdir: bool = False): +def __find_otb_root(): """Search for OTB root directory in well known locations. - Args: - scan_userdir: search with glob in $HOME directory - Returns: str path of the OTB directory @@ -369,22 +366,17 @@ def __find_otb_root(scan_userdir: bool = False): prefix = path.parent.parent.parent else: prefix = path.parent.parent - prefix = prefix.absolute() elif sys.platform == "win32": - for path in Path("c:/Program Files").glob("**/OTB-*/lib"): - logger.info("Found %s", path.parent) - prefix = path.parent.absolute() - elif sys.platform == "darwin": - for path in (Path.home() / "Applications").glob("**/OTB-*/lib"): - logger.info("Found %s", path.parent) - prefix = path.parent.absolute() - # If possible, use OTB found in user's HOME tree (this may take some time) - if scan_userdir: - for path in sorted(Path.home().glob("**/OTB-*/lib/")): + for path in sorted(Path("c:/Program Files").glob("**/OTB-*/lib")): logger.info("Found %s", path.parent) - prefix = path.parent.absolute() + prefix = path.parent + # Search for pyotb OTB install, or default on macOS + apps = Path.home() / "Applications" + for path in sorted(apps.glob("OTB-*/lib/")): + logger.info("Found %s", path.parent) + prefix = path.parent # Return latest found prefix (and version), see precedence in function def find_otb() - return prefix + return prefix.absolute() def __suggest_fix_import(error_message: str, prefix: str): -- GitLab From c3f8976301541055d48e00f32809837e9cc4e199 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 21:34:04 +0200 Subject: [PATCH 294/399] ENH: comment about pyotb init mechanism --- pyotb/helpers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 6319167..1a64fa0 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -432,9 +432,10 @@ def __suggest_fix_import(error_message: str, prefix: str): "You can verify installation requirements for your OS at %s", DOCS_URL ) - -# Since helpers is the first module to be inititialized, this will prevent pyotb to run if OTB is not found +# This part of pyotb is the first imported during __init__ and checks if OTB is found +# User may trigger auto installation with if os.environ.get("OTB_AUTO_INSTALL") in ("1", "true", "ON", "YES"): find_otb(install_otb()) +# If OTB is not found, a SystemExit is raised, to prevent execution of the core module else: find_otb() -- GitLab From 36681d77311ec6cd38d4d1a5a85347a7d7cc2a38 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 21:34:15 +0200 Subject: [PATCH 295/399] CI: bump version --- pyotb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/__init__.py b/pyotb/__init__.py index 507a408..4e89789 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """This module provides convenient python wrapping of otbApplications.""" -__version__ = "2.0.0.dev4" +__version__ = "2.0.0.dev5" from .helpers import logger, set_logger_level from .core import ( -- GitLab From e4cf1b08751250b7dfd07bbd5eeb72059ccad2c8 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 22:06:11 +0200 Subject: [PATCH 296/399] Remove OTB_AUTO_INSTALL --- pyotb/helpers.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 1a64fa0..ea791be 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -433,9 +433,5 @@ def __suggest_fix_import(error_message: str, prefix: str): ) # This part of pyotb is the first imported during __init__ and checks if OTB is found -# User may trigger auto installation with -if os.environ.get("OTB_AUTO_INSTALL") in ("1", "true", "ON", "YES"): - find_otb(install_otb()) # If OTB is not found, a SystemExit is raised, to prevent execution of the core module -else: - find_otb() +find_otb() -- GitLab From 619dc50cdc87bf9909d9c342089f68d651f14309 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 22:09:08 +0200 Subject: [PATCH 297/399] FIX: window install just unzip files --- pyotb/helpers.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index ea791be..53d9f9a 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -7,6 +7,7 @@ import re import subprocess import sys import tempfile +import zipfile import urllib.request from pathlib import Path from shutil import which @@ -237,9 +238,13 @@ def install_otb(version: str = "latest", path: str = ""): path = Path(path) else: path = Path.home() / "Applications" / tmpfile.stem - install_cmd = f"{cmd} {tmpfile} --target {path} --accept" - print(f"Executing '{install_cmd}'\n") - subprocess.run(f"{cmd} {tmpfile} --target {path} --accept", shell=True, check=True) + if sysname == "Win64": + with zipfile.ZipFile(tmpfile) as zipf: + zipf.extractall(path) + else: + install_cmd = f"{cmd} {tmpfile} --target {path} --accept" + print(f"Executing '{install_cmd}'\n") + subprocess.run(f"{cmd} {tmpfile} --target {path} --accept", shell=True, check=True) tmpfile.unlink() # cleaning # Add env variable to profile -- GitLab From 65cf76ebe173ca7469ccb58d1826a7c9874ca007 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 22:19:39 +0200 Subject: [PATCH 298/399] FIX: bug if OTB is not installed --- pyotb/helpers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 53d9f9a..9944dd3 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -98,6 +98,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True): logger.info("Failed to import OTB. Searching for it...") prefix = __find_otb_root() if not prefix: + # Python is interactive if hasattr(sys, "ps1"): if input("OTB is missing. Do you want to install it ? (y/n): ") == "y": version = input( @@ -113,7 +114,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True): "OTB not found on disk. " "To install it, open an interactive python shell and type 'import pyotb'" ) - # Try to import one last time before raising error + # If OTB is found on disk, set env and try to import one last time try: set_environment(prefix) import otbApplication as otb # pylint: disable=import-outside-toplevel @@ -381,7 +382,8 @@ def __find_otb_root(): logger.info("Found %s", path.parent) prefix = path.parent # Return latest found prefix (and version), see precedence in function def find_otb() - return prefix.absolute() + if isinstance(prefix, Path): + return prefix.absolute() def __suggest_fix_import(error_message: str, prefix: str): -- GitLab From 9ca34eecd801147f4c38efc17a8f4200d5d93316 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 22:29:20 +0200 Subject: [PATCH 299/399] FIX: zipfile not extracted in subdirectory --- pyotb/helpers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 9944dd3..ed2d96c 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -204,7 +204,7 @@ def install_otb(version: str = "latest", path: str = ""): Args: version: OTB version tag, e.g. '8.1.2' - path: installation directory + path: installation directory, default is $HOME/Applications Returns: full path of the new installation @@ -241,11 +241,12 @@ def install_otb(version: str = "latest", path: str = ""): path = Path.home() / "Applications" / tmpfile.stem if sysname == "Win64": with zipfile.ZipFile(tmpfile) as zipf: - zipf.extractall(path) + print("Extracting zip file...") + zipf.extractall(path.parent) else: install_cmd = f"{cmd} {tmpfile} --target {path} --accept" print(f"Executing '{install_cmd}'\n") - subprocess.run(f"{cmd} {tmpfile} --target {path} --accept", shell=True, check=True) + subprocess.run(install_cmd, shell=True, check=True) tmpfile.unlink() # cleaning # Add env variable to profile -- GitLab From 502fe83c7b7479a131f610358ef90eac487b20c4 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 22:35:49 +0200 Subject: [PATCH 300/399] STYLE: pylint --- pyotb/helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index ed2d96c..ac69729 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -385,6 +385,7 @@ def __find_otb_root(): # Return latest found prefix (and version), see precedence in function def find_otb() if isinstance(prefix, Path): return prefix.absolute() + return None def __suggest_fix_import(error_message: str, prefix: str): @@ -440,6 +441,7 @@ def __suggest_fix_import(error_message: str, prefix: str): "You can verify installation requirements for your OS at %s", DOCS_URL ) + # This part of pyotb is the first imported during __init__ and checks if OTB is found # If OTB is not found, a SystemExit is raised, to prevent execution of the core module find_otb() -- GitLab From bf0030a35921f1b35696fc094abf1c5594c4bbb8 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 22:43:52 +0200 Subject: [PATCH 301/399] STYLE: add print headers for steps in install_otb --- pyotb/helpers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index ac69729..c99bbb8 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -233,7 +233,7 @@ def install_otb(version: str = "latest", path: str = ""): url = f"https://www.orfeo-toolbox.org/packages/archives/OTB/{filename}" tmpdir = tempfile.gettempdir() tmpfile = Path(tmpdir) / filename - print(f"Downloading {url}") + print(f"##### Downloading {url}") urllib.request.urlretrieve(url, tmpfile) if path: path = Path(path) @@ -241,18 +241,18 @@ def install_otb(version: str = "latest", path: str = ""): path = Path.home() / "Applications" / tmpfile.stem if sysname == "Win64": with zipfile.ZipFile(tmpfile) as zipf: - print("Extracting zip file...") + print("##### Extracting zip file...") zipf.extractall(path.parent) else: install_cmd = f"{cmd} {tmpfile} --target {path} --accept" - print(f"Executing '{install_cmd}'\n") + print(f"##### Executing '{install_cmd}'\n") subprocess.run(install_cmd, shell=True, check=True) tmpfile.unlink() # cleaning # Add env variable to profile if sysname != "Win64": profile = Path.home() / ".profile" - print(f"Adding new env variables to {profile}") + print(f"##### Adding new env variables to {profile}") with open(profile, "a", encoding="utf-8") as buf: buf.write(f'\n. "{path}/otbenv.profile"\n') else: @@ -266,7 +266,7 @@ def install_otb(version: str = "latest", path: str = ""): return str(path) # Else recompile bindings : this may fail because of OpenGL if which("ctest") and which("python3-config"): - print("\nRecompiling python bindings...") + print("\n##### Recompiling python bindings...") ctest_cmd = ( ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV" ) -- GitLab From 7d24c797b0c10b0b9027ab0bcc3585d8c489f158 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 22:46:20 +0200 Subject: [PATCH 302/399] FIX: raise system exit with message --- pyotb/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index c99bbb8..120b859 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -272,11 +272,11 @@ def install_otb(version: str = "latest", path: str = ""): ) subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True) return str(path) - print( + msg = ( "\nYou need to install 'cmake', 'python3-dev' and 'libgl1-mesa-dev'" " in order to recompile python bindings. " ) - raise SystemExit + raise SystemExit(msg) def __find_lib(prefix: str = None, otb_module=None): -- GitLab From 116cba62bb956c7072f510ee73babb0270c1d903 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 23:06:10 +0200 Subject: [PATCH 303/399] ENH: ask before editing user env .profile --- pyotb/helpers.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 120b859..9739e62 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -108,7 +108,10 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True): "Provide a path for installation " "(default is <user_dir>/Applications/OTB-<version>): " ) - return find_otb(install_otb(version, path)) + edit_env = input( + "Enable user environment variables for this installation ? (y/n): " + ) + return find_otb(install_otb(version, path, edit_env)) if not prefix: raise SystemExit( "OTB not found on disk. " @@ -199,7 +202,7 @@ def otb_latest_release_tag(): return releases[-1] -def install_otb(version: str = "latest", path: str = ""): +def install_otb(version: str = "latest", path: str = "", edit_env: bool = False): """Install pre-compiled OTB binaries in path, use latest release by default. Args: @@ -250,7 +253,7 @@ def install_otb(version: str = "latest", path: str = ""): tmpfile.unlink() # cleaning # Add env variable to profile - if sysname != "Win64": + if sysname != "Win64" and edit_env: profile = Path.home() / ".profile" print(f"##### Adding new env variables to {profile}") with open(profile, "a", encoding="utf-8") as buf: -- GitLab From 95a41bd6bc73f1510128e164da4d4a2e0ff57bbd Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Thu, 10 Aug 2023 23:07:55 +0200 Subject: [PATCH 304/399] FIX: check user input before editing profile --- pyotb/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 9739e62..c4414b7 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -109,8 +109,8 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True): "(default is <user_dir>/Applications/OTB-<version>): " ) edit_env = input( - "Enable user environment variables for this installation ? (y/n): " - ) + "Modify user environment variables for this installation ? (y/n): " + ) == "y" return find_otb(install_otb(version, path, edit_env)) if not prefix: raise SystemExit( -- GitLab From 0d486b76ceb4df4ebbb379ff55a45429861389f9 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 15:19:37 +0200 Subject: [PATCH 305/399] ADD: new module pyotb.install, clean helpers.py --- pyotb/__init__.py | 1 + pyotb/helpers.py | 127 +++-------------------------------- pyotb/install.py | 165 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 176 insertions(+), 117 deletions(-) create mode 100644 pyotb/install.py diff --git a/pyotb/__init__.py b/pyotb/__init__.py index 4e89789..c134e58 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -2,6 +2,7 @@ """This module provides convenient python wrapping of otbApplications.""" __version__ = "2.0.0.dev5" +from .install import install_otb from .helpers import logger, set_logger_level from .core import ( OTBObject, diff --git a/pyotb/helpers.py b/pyotb/helpers.py index c4414b7..ea14c55 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -1,17 +1,13 @@ # -*- coding: utf-8 -*- """This module helps to ensure we properly initialize pyotb: only in case OTB is found and apps are available.""" -import json import logging import os -import re -import subprocess import sys -import tempfile -import zipfile -import urllib.request from pathlib import Path from shutil import which +from .install import install_otb, interactive_config + # Allow user to switch between OTB directories without setting every env variable OTB_ROOT = os.environ.get("OTB_ROOT") DOCS_URL = "https://www.orfeo-toolbox.org/CookBook/Installation.html" @@ -97,27 +93,16 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True): # Else search system logger.info("Failed to import OTB. Searching for it...") prefix = __find_otb_root() - if not prefix: - # Python is interactive - if hasattr(sys, "ps1"): - if input("OTB is missing. Do you want to install it ? (y/n): ") == "y": - version = input( - "Choose a version number to install (default is latest): " - ) - path = input( - "Provide a path for installation " - "(default is <user_dir>/Applications/OTB-<version>): " - ) - edit_env = input( - "Modify user environment variables for this installation ? (y/n): " - ) == "y" - return find_otb(install_otb(version, path, edit_env)) - if not prefix: + # Try auto install if shell is interactive + if not prefix and hasattr(sys, "ps1"): + if input("OTB is missing. Do you want to install it ? (y/n): ") == "y": + return find_otb(install_otb(*interactive_config())) + elif not prefix: raise SystemExit( - "OTB not found on disk. " - "To install it, open an interactive python shell and type 'import pyotb'" + "OTB libraries not found on disk. " + "To install it, open an interactive python shell and 'import pyotb'" ) - # If OTB is found on disk, set env and try to import one last time + # If OTB was found on disk, set env and try to import one last time try: set_environment(prefix) import otbApplication as otb # pylint: disable=import-outside-toplevel @@ -190,98 +175,6 @@ def set_environment(prefix: str): os.environ["PROJ_LIB"] = proj_lib -def otb_latest_release_tag(): - """Use gitlab API to find latest release tag name, but skip pre-releases.""" - api_endpoint = "https://gitlab.orfeo-toolbox.org/api/v4/projects/53/repository/tags" - vers_regex = re.compile(r"^\d\.\d\.\d$") # we ignore rc-* or alpha-* - with urllib.request.urlopen(api_endpoint) as stream: - data = json.loads(stream.read()) - releases = sorted( - [tag["name"] for tag in data if vers_regex.match(tag["name"])], - ) - return releases[-1] - - -def install_otb(version: str = "latest", path: str = "", edit_env: bool = False): - """Install pre-compiled OTB binaries in path, use latest release by default. - - Args: - version: OTB version tag, e.g. '8.1.2' - path: installation directory, default is $HOME/Applications - - Returns: - full path of the new installation - """ - major = sys.version_info.major - if major == 2: - raise SystemExit("Python 3 is required for OTB bindings.") - minor = sys.version_info.minor - name_corresp = {"linux": "Linux64", "darwnin": "Darwin64", "win32": "Win64"} - sysname = name_corresp[sys.platform] - if sysname == "Win64": - if minor != 7: - raise SystemExit( - "Python version 3.7 is required to import python bindings on Windows." - ) - cmd = which("cmd.exe") - ext = "zip" - else: - cmd = which("zsh") or which("bash") or which("sh") - ext = "run" - - # Fetch archive and run installer - if not version or version == "latest": - version = otb_latest_release_tag() - filename = f"OTB-{version}-{sysname}.{ext}" - url = f"https://www.orfeo-toolbox.org/packages/archives/OTB/{filename}" - tmpdir = tempfile.gettempdir() - tmpfile = Path(tmpdir) / filename - print(f"##### Downloading {url}") - urllib.request.urlretrieve(url, tmpfile) - if path: - path = Path(path) - else: - path = Path.home() / "Applications" / tmpfile.stem - if sysname == "Win64": - with zipfile.ZipFile(tmpfile) as zipf: - print("##### Extracting zip file...") - zipf.extractall(path.parent) - else: - install_cmd = f"{cmd} {tmpfile} --target {path} --accept" - print(f"##### Executing '{install_cmd}'\n") - subprocess.run(install_cmd, shell=True, check=True) - tmpfile.unlink() # cleaning - - # Add env variable to profile - if sysname != "Win64" and edit_env: - profile = Path.home() / ".profile" - print(f"##### Adding new env variables to {profile}") - with open(profile, "a", encoding="utf-8") as buf: - buf.write(f'\n. "{path}/otbenv.profile"\n') - else: - print( - "In order to speed-up pyotb import, remember to call 'otbenv.bat' " - f"before importing pyotb, or add 'OTB_ROOT=\"{path}\"' to your env variables." - ) - return str(path) - # Linux requires 3.8, macOS requires 3.7 - if (sysname == "Linux64" and minor == 8) or (sysname == "Darwin64" and minor == 7): - return str(path) - # Else recompile bindings : this may fail because of OpenGL - if which("ctest") and which("python3-config"): - print("\n##### Recompiling python bindings...") - ctest_cmd = ( - ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV" - ) - subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True) - return str(path) - msg = ( - "\nYou need to install 'cmake', 'python3-dev' and 'libgl1-mesa-dev'" - " in order to recompile python bindings. " - ) - raise SystemExit(msg) - - def __find_lib(prefix: str = None, otb_module=None): """Try to find OTB external libraries directory. diff --git a/pyotb/install.py b/pyotb/install.py new file mode 100644 index 0000000..c4e1c9d --- /dev/null +++ b/pyotb/install.py @@ -0,0 +1,165 @@ +"""This module contains functions for interactive auto installation of OTB.""" +import json +import sys +import re +import subprocess +import tempfile +import zipfile +import urllib.request +from pathlib import Path +from shutil import which + + +def otb_latest_release_tag(): + """Use gitlab API to find latest release tag name, but skip pre-releases.""" + api_endpoint = "https://gitlab.orfeo-toolbox.org/api/v4/projects/53/repository/tags" + vers_regex = re.compile(r"^\d\.\d\.\d$") # we ignore rc-* or alpha-* + with urllib.request.urlopen(api_endpoint) as stream: + data = json.loads(stream.read()) + releases = sorted( + [tag["name"] for tag in data if vers_regex.match(tag["name"])], + ) + return releases[-1] + + +def check_versions(sysname: str, python_minor: int, otb_major: int): + """Verify if python version is compatible with OTB version. + + Args: + sysname: OTB's system name convention (Linux64, Darwin64, Win64) + python_minor: minor version of python + otb_major: major version of OTB to be installed + Returns: + True if requirements are satisfied + """ + if sysname == "Win64": + expected = 5 if otb_major in (6, 7) else 7 + if python_minor == expected: + return True, 0 + elif sysname == "Darwin64": + expected = 7, 0 + if python_minor == expected: + return True, 0 + elif sysname == "Linux64": + expected = 5 if otb_major in (6, 7) else 8 + if python_minor == expected: + return True, 0 + return False, expected + + +def install_otb(version: str = "latest", path: str = "", edit_env: bool = False): + """Install pre-compiled OTB binaries in path, use latest release by default. + + Args: + version: OTB version tag, e.g. '8.1.2' + path: installation directory, default is $HOME/Applications + + Returns: + full path of the new installation + """ + # Read env config + if sys.version_info.major == 2: + raise SystemExit("Python 3 is required for OTB bindings.") + python_minor = sys.version_info.minor + if not version or version == "latest": + version = otb_latest_release_tag() + name_corresp = {"linux": "Linux64", "darwnin": "Darwin64", "win32": "Win64"} + sysname = name_corresp[sys.platform] + ext = "zip" if sysname == "Win64" else "run" + cmd = which("zsh") or which("bash") or which("sh") + otb_major = int(version[0]) + check, expected = check_versions(sysname, python_minor, otb_major) + if sysname == "Win64" and not check: + print(f"Python 3.{expected} is required to import bindings on Windows.") + return + + # Fetch archive and run installer + filename = f"OTB-{version}-{sysname}.{ext}" + url = f"https://www.orfeo-toolbox.org/packages/archives/OTB/{filename}" + tmpdir = tempfile.gettempdir() + tmpfile = Path(tmpdir) / filename + print(f"##### Downloading {url}") + urllib.request.urlretrieve(url, tmpfile) + if path: + default_path = False + path = Path(path) + else: + default_path = True + path = Path.home() / "Applications" / tmpfile.stem + if sysname == "Win64": + with zipfile.ZipFile(tmpfile) as zipf: + print("##### Extracting zip file...") + zipf.extractall(path.parent) + else: + install_cmd = f"{cmd} {tmpfile} --target {path} --accept" + print(f"##### Executing '{install_cmd}'\n") + subprocess.run(install_cmd, shell=True, check=True) + tmpfile.unlink() # cleaning + + # Add env variable to profile + if edit_env: + if sysname == "Win64": + # TODO: import winreg + return str(path) + else: + profile = Path.home() / ".profile" + print(f"##### Adding new env variables to {profile}") + with open(profile, "a", encoding="utf-8") as buf: + buf.write(f'\n. "{path}/otbenv.profile"\n') + elif not default_path: + ext = "bat" if sysname == "Win64" else "profile" + print( + f"Remember to call 'otbenv.{ext}' before importing pyotb, " + f"or add 'OTB_ROOT=\"{path}\"' to your env variables." + ) + # No recompilation or symlink required + if check: + return str(path) + + # Else recompile bindings : this may fail because of OpenGL + target_lib = f"{path}/lib/libpython3.{expected}.so.rh-python3{expected}-1.0" + if which("ctest") and which("python3-config"): + print("\n##### Recompiling python bindings...") + ctest_cmd = ( + ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV" + ) + subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True) + return str(path) + # Use dirty cross python version symlink + elif sys.executable.startswith("/usr/bin"): + lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so" + if Path(lib).exists(): + ln_cmd = f'ln -s "{lib}" "{target_lib}"' + try: + subprocess.run(ln_cmd, executable=cmd, shell=True, check=True) + return str(path) + except subprocess.CalledProcessError as err: + raise SystemExit( + "Unable to recompile python bindings, " + "some dependencies may require manuel installation" + ) from err + else: + print( + f"Unable to automatically locate library for executable {sys.executable}" + f"You'll need to manually symlink that one file to {target_lib}" + ) + # TODO: support for auto build deps install using brew, apt, pacman/yay, yum... + msg = ( + "\nYou need to install 'cmake', 'python3-dev' and 'libgl1-mesa-dev'" + " in order to recompile python bindings. " + ) + raise SystemExit(msg) + + +def interactive_config(): + """Prompt user to configure installation variables.""" + version = input("Choose a version number to install (default is latest): ") + path = input( + "Provide a path for installation " + "(default is <user_dir>/Applications/OTB-<version>): " + ) + edit_env = ( + input("Modify user environment variables for this installation ? (y/n): ") + == "y" + ) + return version, path, edit_env -- GitLab From cdd87a628824199f8f7a1fd7fe81196b5eed6ac6 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 15:27:41 +0200 Subject: [PATCH 306/399] STYLE: enforce max line length=88 --- pyotb/helpers.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index ea14c55..30c3e4a 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -"""This module helps to ensure we properly initialize pyotb: only in case OTB is found and apps are available.""" +"""This module ensure we properly initialize pyotb, or raise SystemExit in case of broken install.""" import logging import os import sys @@ -43,8 +43,9 @@ def set_logger_level(level: str): def find_otb(prefix: str = OTB_ROOT, scan: bool = True): - """Try to load OTB bindings or scan system, help user in case of failure, set env variables. + """Try to load OTB bindings or scan system, help user in case of failure, set env. + If in interactive prompt, user will be asked if he wants to install OTB. The OTB_ROOT variable allow one to override default OTB version, with auto env setting. Path precedence : $OTB_ROOT > location of python bindings location Then, if OTB is not found: @@ -75,7 +76,8 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True): raise SystemExit("Failed to import OTB. Exiting.") from e # Else try import from actual Python path try: - # Here, we can't properly set env variables before OTB import. We assume user did this before running python + # Here, we can't properly set env variables before OTB import. + # We assume user did this before running python # For LD_LIBRARY_PATH problems, use OTB_ROOT instead of PYTHONPATH import otbApplication as otb # pylint: disable=import-outside-toplevel @@ -145,9 +147,9 @@ def set_environment(prefix: str): raise EnvironmentError("Can't find OTB Python API") if otb_api not in sys.path: sys.path.insert(0, otb_api) - # Add /bin first in PATH, in order to avoid conflicts with another GDAL install when using os.system() + # Add /bin first in PATH, in order to avoid conflicts with another GDAL install os.environ["PATH"] = f"{prefix / 'bin'}{os.pathsep}{os.environ['PATH']}" - # Applications path (this can be tricky since OTB import will succeed even without apps) + # Ensure APPLICATION_PATH is set apps_path = __find_apps_path(lib_dir) if Path(apps_path).exists(): os.environ["OTB_APPLICATION_PATH"] = apps_path @@ -180,7 +182,7 @@ def __find_lib(prefix: str = None, otb_module=None): Args: prefix: try with OTB root directory - otb_module: try with OTB python module (otbApplication) library path if found, else None + otb_module: try with otbApplication library path if found, else None Returns: lib path @@ -278,7 +280,7 @@ def __find_otb_root(): for path in sorted(apps.glob("OTB-*/lib/")): logger.info("Found %s", path.parent) prefix = path.parent - # Return latest found prefix (and version), see precedence in function def find_otb() + # Return latest found prefix (and version), see precedence in find_otb() docstrings if isinstance(prefix, Path): return prefix.absolute() return None @@ -297,25 +299,27 @@ def __suggest_fix_import(error_message: str, prefix: str): lib = ( f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so" ) - if which("ctest"): + if which("ctest") and which("python3-config"): logger.critical( - "To recompile python bindings, use 'cd %s ; source otbenv.profile ; " + "To recompile python bindings, use " + "'cd %s ; source otbenv.profile ; " "ctest -S share/otb/swig/build_wrapping.cmake -VV'", prefix, ) elif Path(lib).exists(): - expect_minor = int(error_message[11]) - if expect_minor != sys.version_info.minor: + expected = int(error_message[11]) + if expected != sys.version_info.minor: logger.critical( - "Python library version mismatch (OTB was expecting 3.%s) : " - "a simple symlink may not work, depending on your python version", - expect_minor, + "Python library version mismatch (OTB expected 3.%s) : " + "a symlink may not work, depending on your python version", + expected, ) - target_lib = f"{prefix}/lib/libpython3.{expect_minor}.so.rh-python3{expect_minor}-1.0" - logger.critical("Use 'ln -s %s %s'", lib, target_lib) + target = f"{prefix}/lib/libpython3.{expected}.so.1.0" + logger.critical("Use 'ln -s %s %s'", lib, target) else: logger.critical( - "You may need to install cmake in order to recompile python bindings" + "You may need to install cmake, python3-dev and mesa's libgl" + " in order to recompile python bindings" ) else: logger.critical( @@ -326,7 +330,7 @@ def __suggest_fix_import(error_message: str, prefix: str): if error_message.startswith("DLL load failed"): if sys.version_info.minor != 7: logger.critical( - "You need Python 3.5 (OTB releases 6.4 to 7.4) or Python 3.7 (since OTB 8)" + "You need Python 3.5 (OTB 6.4 to 7.4) or Python 3.7 (since OTB 8)" ) else: logger.critical( -- GitLab From dca3ea5f75f9314b4faf78bc423cc25e88e3d145 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 15:34:22 +0200 Subject: [PATCH 307/399] FIX: raise error when recompilation failed --- pyotb/install.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index c4e1c9d..8c1e797 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -123,21 +123,21 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) ctest_cmd = ( ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV" ) - subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True) - return str(path) + try: + subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True) + return str(path) + except subprocess.CalledProcessError as err: + raise SystemExit( + "Unable to recompile python bindings, " + "some dependencies may require manuel installation." + ) from err # Use dirty cross python version symlink elif sys.executable.startswith("/usr/bin"): lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so" if Path(lib).exists(): ln_cmd = f'ln -s "{lib}" "{target_lib}"' - try: - subprocess.run(ln_cmd, executable=cmd, shell=True, check=True) - return str(path) - except subprocess.CalledProcessError as err: - raise SystemExit( - "Unable to recompile python bindings, " - "some dependencies may require manuel installation" - ) from err + subprocess.run(ln_cmd, executable=cmd, shell=True, check=True) + return str(path) else: print( f"Unable to automatically locate library for executable {sys.executable}" -- GitLab From 37827e778050edf8d51671b338c33e104f724bbe Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 15:35:41 +0200 Subject: [PATCH 308/399] ENH: talk about libgl1 in error message --- pyotb/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/install.py b/pyotb/install.py index 8c1e797..f5edeab 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -129,7 +129,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) except subprocess.CalledProcessError as err: raise SystemExit( "Unable to recompile python bindings, " - "some dependencies may require manuel installation." + "some dependencies (libgl1) may require manual installation." ) from err # Use dirty cross python version symlink elif sys.executable.startswith("/usr/bin"): -- GitLab From b0a907d7003f3a48807ef66da1efe6f9b6bc5098 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 15:37:39 +0200 Subject: [PATCH 309/399] ENH: print message before creating symlinks --- pyotb/install.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index f5edeab..6a7c66d 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -117,7 +117,6 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) return str(path) # Else recompile bindings : this may fail because of OpenGL - target_lib = f"{path}/lib/libpython3.{expected}.so.rh-python3{expected}-1.0" if which("ctest") and which("python3-config"): print("\n##### Recompiling python bindings...") ctest_cmd = ( @@ -133,9 +132,11 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) ) from err # Use dirty cross python version symlink elif sys.executable.startswith("/usr/bin"): + target_lib = f"{path}/lib/libpython3.{expected}.so.rh-python3{expected}-1.0" lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so" if Path(lib).exists(): - ln_cmd = f'ln -s "{lib}" "{target_lib}"' + print(f"Creating symbolic links: {lib} -> {target_lib}") + ln_cmd = f'ln -sf "{lib}" "{target_lib}"' subprocess.run(ln_cmd, executable=cmd, shell=True, check=True) return str(path) else: -- GitLab From 7ecd4a4536b3b63ddbb426aa215abb6ac46f07ce Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 15:45:13 +0200 Subject: [PATCH 310/399] FIX: wrong python lib name for symlink with OTB 8 --- pyotb/install.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index 6a7c66d..4f26afe 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -130,9 +130,10 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) "Unable to recompile python bindings, " "some dependencies (libgl1) may require manual installation." ) from err - # Use dirty cross python version symlink + # Use dirty cross python version symlink (only tested on Ubuntu) elif sys.executable.startswith("/usr/bin"): - target_lib = f"{path}/lib/libpython3.{expected}.so.rh-python3{expected}-1.0" + suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else ".so.1.0" + target_lib = f"{path}/lib/libpython3.{expected}.{suffix}" lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so" if Path(lib).exists(): print(f"Creating symbolic links: {lib} -> {target_lib}") -- GitLab From f7697d0d52341f27399a2a8b858ddc1d862345bd Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 15:46:53 +0200 Subject: [PATCH 311/399] ENH: log message about python lib symlinks --- pyotb/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 30c3e4a..812b72e 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -315,7 +315,7 @@ def __suggest_fix_import(error_message: str, prefix: str): expected, ) target = f"{prefix}/lib/libpython3.{expected}.so.1.0" - logger.critical("Use 'ln -s %s %s'", lib, target) + logger.critical("If using OTB>=8.0, use 'ln -s %s %s'", lib, target) else: logger.critical( "You may need to install cmake, python3-dev and mesa's libgl" -- GitLab From f593595d6e27f58453421c12bd61140f2d998380 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 15:49:53 +0200 Subject: [PATCH 312/399] FIX: case python exe is not default system python --- pyotb/install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index 4f26afe..32a78cb 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -117,6 +117,8 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) return str(path) # Else recompile bindings : this may fail because of OpenGL + suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else ".so.1.0" + target_lib = f"{path}/lib/libpython3.{expected}.{suffix}" if which("ctest") and which("python3-config"): print("\n##### Recompiling python bindings...") ctest_cmd = ( @@ -132,8 +134,6 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) ) from err # Use dirty cross python version symlink (only tested on Ubuntu) elif sys.executable.startswith("/usr/bin"): - suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else ".so.1.0" - target_lib = f"{path}/lib/libpython3.{expected}.{suffix}" lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so" if Path(lib).exists(): print(f"Creating symbolic links: {lib} -> {target_lib}") -- GitLab From 1de178d4a249f306d65d8e928cf4d9bfb459c773 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 15:51:11 +0200 Subject: [PATCH 313/399] FIX: typo in python lib path --- pyotb/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/install.py b/pyotb/install.py index 32a78cb..a02cf9d 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -117,7 +117,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) return str(path) # Else recompile bindings : this may fail because of OpenGL - suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else ".so.1.0" + suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else "so.1.0" target_lib = f"{path}/lib/libpython3.{expected}.{suffix}" if which("ctest") and which("python3-config"): print("\n##### Recompiling python bindings...") -- GitLab From 1458e253250f4be1bd759dfb909a1b390e9b6bb9 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 15:57:37 +0200 Subject: [PATCH 314/399] FIX: do not try to recompile when is colab env --- pyotb/install.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index a02cf9d..2729f22 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -1,6 +1,7 @@ """This module contains functions for interactive auto installation of OTB.""" import json import sys +import os import re import subprocess import tempfile @@ -119,13 +120,20 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) # Else recompile bindings : this may fail because of OpenGL suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else "so.1.0" target_lib = f"{path}/lib/libpython3.{expected}.{suffix}" - if which("ctest") and which("python3-config"): + if ( + which("ctest") + and which("python3-config") + # Google Colab ships with cmake and python3-dev, but not libgl1-mesa-dev + and "COLAB_RELEASE_TAG" not in os.environ + ): print("\n##### Recompiling python bindings...") ctest_cmd = ( ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV" ) try: - subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True) + subprocess.run( + ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True + ) return str(path) except subprocess.CalledProcessError as err: raise SystemExit( -- GitLab From c18843f560e21622f2180ca63efb2aa7db560090 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr> Date: Fri, 11 Aug 2023 14:01:21 +0000 Subject: [PATCH 315/399] Apply 1 suggestion(s) to 1 file(s) --- pyotb/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/install.py b/pyotb/install.py index 2729f22..abf0499 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -64,7 +64,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) python_minor = sys.version_info.minor if not version or version == "latest": version = otb_latest_release_tag() - name_corresp = {"linux": "Linux64", "darwnin": "Darwin64", "win32": "Win64"} + name_corresp = {"linux": "Linux64", "darwin": "Darwin64", "win32": "Win64"} sysname = name_corresp[sys.platform] ext = "zip" if sysname == "Win64" else "run" cmd = which("zsh") or which("bash") or which("sh") -- GitLab From ebd7cb34fe481b247e511d7e6f25149a35da9ad7 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 16:14:07 +0200 Subject: [PATCH 316/399] STYLE: run pylint and isort --- pyotb/install.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index abf0499..76f1feb 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -1,12 +1,12 @@ """This module contains functions for interactive auto installation of OTB.""" import json -import sys import os import re import subprocess +import sys import tempfile -import zipfile import urllib.request +import zipfile from pathlib import Path from shutil import which @@ -72,7 +72,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) check, expected = check_versions(sysname, python_minor, otb_major) if sysname == "Win64" and not check: print(f"Python 3.{expected} is required to import bindings on Windows.") - return + return "" # Fetch archive and run installer filename = f"OTB-{version}-{sysname}.{ext}" @@ -101,7 +101,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) if edit_env: if sysname == "Win64": # TODO: import winreg - return str(path) + ... else: profile = Path.home() / ".profile" print(f"##### Adding new env variables to {profile}") -- GitLab From e85ea8281db2ce285248a89529c3781872207a5c Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 17:08:50 +0200 Subject: [PATCH 317/399] ADD: env setting using registry on windows --- pyotb/install.py | 46 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index 76f1feb..35b57a2 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -48,6 +48,39 @@ def check_versions(sysname: str, python_minor: int, otb_major: int): return False, expected +def update_unix_env(otb_path: Path): + """Update env profile for current user with new otb_env.profile call. + + Args: + otb_path: the path of the new OTB installation + + """ + profile = Path.home() / ".profile" + with open(profile, "a", encoding="utf-8") as buf: + buf.write(f'\n. "{otb_path}/otbenv.profile"\n') + print(f"##### Added new environment variables to {profile}") + + +def update_windows_env(otb_path: Path): + """Update registry hive for current user with new OTB_ROOT env variable. + + Args: + otb_path: path of the new OTB installation + + """ + import winreg # pylint: disable=import-error,import-outside-toplevel + + with winreg.OpenKeyEx( + winreg.HKEY_CURRENT_USER, "Environment", 0, winreg.KEY_SET_VALUE + ) as reg_key: + winreg.SetValueEx(reg_key, "OTB_ROOT", 0, winreg.REG_EXPAND_SZ, str(otb_path)) + print( + "##### Environment variable 'OTB_ROOT' added successfully to user's hive." + ) + key = "HKEY_CURRENT_USER\\Environment\\OTB_ROOT" + print(f"To undo this permanent setting, use 'reg.exe delete \"{key}\"'") + + def install_otb(version: str = "latest", path: str = "", edit_env: bool = False): """Install pre-compiled OTB binaries in path, use latest release by default. @@ -57,6 +90,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) Returns: full path of the new installation + """ # Read env config if sys.version_info.major == 2: @@ -89,7 +123,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) path = Path.home() / "Applications" / tmpfile.stem if sysname == "Win64": with zipfile.ZipFile(tmpfile) as zipf: - print("##### Extracting zip file...") + print("##### Extracting zip file...\n") zipf.extractall(path.parent) else: install_cmd = f"{cmd} {tmpfile} --target {path} --accept" @@ -100,17 +134,13 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) # Add env variable to profile if edit_env: if sysname == "Win64": - # TODO: import winreg - ... + update_windows_env(path) else: - profile = Path.home() / ".profile" - print(f"##### Adding new env variables to {profile}") - with open(profile, "a", encoding="utf-8") as buf: - buf.write(f'\n. "{path}/otbenv.profile"\n') + update_unix_env(path) elif not default_path: ext = "bat" if sysname == "Win64" else "profile" print( - f"Remember to call 'otbenv.{ext}' before importing pyotb, " + f"Remember to call '{path}{os.sep}otbenv.{ext}' before importing pyotb, " f"or add 'OTB_ROOT=\"{path}\"' to your env variables." ) # No recompilation or symlink required -- GitLab From 2e7b43e5de020a097a2e64ac26ffaa8c06e890bf Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 17:18:47 +0200 Subject: [PATCH 318/399] ENH: install_otb smaller functions for env, recompile and symlinks steps --- pyotb/install.py | 97 ++++++++++++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 41 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index 35b57a2..8f6c281 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -11,6 +11,20 @@ from pathlib import Path from shutil import which +def interactive_config(): + """Prompt user to configure installation variables.""" + version = input("Choose a version number to install (default is latest): ") + path = input( + "Provide a path for installation " + "(default is <user_dir>/Applications/OTB-<version>): " + ) + edit_env = ( + input("Modify user environment variables for this installation ? (y/n): ") + == "y" + ) + return version, path, edit_env + + def otb_latest_release_tag(): """Use gitlab API to find latest release tag name, but skip pre-releases.""" api_endpoint = "https://gitlab.orfeo-toolbox.org/api/v4/projects/53/repository/tags" @@ -81,6 +95,40 @@ def update_windows_env(otb_path: Path): print(f"To undo this permanent setting, use 'reg.exe delete \"{key}\"'") +def recompile_python_bindings(path: str, cmd: str): + """Run subprocess command to recompile python bindings. + + Args: + path: path of the new OTB installation + cmd: path of the default system shell command + + """ + print("\n##### Recompiling python bindings...") + ctest_cmd = ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV" + try: + subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True) + except subprocess.CalledProcessError as err: + raise SystemExit( + "Unable to recompile python bindings, " + "some dependencies (libgl1) may require manual installation." + ) from err + + +def symlink_python_library(target_lib: str, cmd: str): + """Run subprocess command to recompile python bindings. + + Args: + path: path of the new OTB installation + cmd: path of the default system shell command + + """ + lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so" + if Path(lib).exists(): + print(f"##### Creating symbolic links: {lib} -> {target_lib}") + ln_cmd = f'ln -sf "{lib}" "{target_lib}"' + subprocess.run(ln_cmd, executable=cmd, shell=True, check=True) + + def install_otb(version: str = "latest", path: str = "", edit_env: bool = False): """Install pre-compiled OTB binaries in path, use latest release by default. @@ -150,34 +198,15 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) # Else recompile bindings : this may fail because of OpenGL suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else "so.1.0" target_lib = f"{path}/lib/libpython3.{expected}.{suffix}" - if ( - which("ctest") - and which("python3-config") - # Google Colab ships with cmake and python3-dev, but not libgl1-mesa-dev - and "COLAB_RELEASE_TAG" not in os.environ - ): - print("\n##### Recompiling python bindings...") - ctest_cmd = ( - ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV" - ) - try: - subprocess.run( - ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True - ) - return str(path) - except subprocess.CalledProcessError as err: - raise SystemExit( - "Unable to recompile python bindings, " - "some dependencies (libgl1) may require manual installation." - ) from err - # Use dirty cross python version symlink (only tested on Ubuntu) + can_compile = which("ctest") and which("python3-config") + # Google Colab ships with cmake and python3-dev, but not libgl1-mesa-dev + if can_compile and "COLAB_RELEASE_TAG" not in os.environ: + recompile_python_bindings(path, cmd) + return str(path) + # Or use dirty cross version python symlink (only tested on Ubuntu) elif sys.executable.startswith("/usr/bin"): - lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so" - if Path(lib).exists(): - print(f"Creating symbolic links: {lib} -> {target_lib}") - ln_cmd = f'ln -sf "{lib}" "{target_lib}"' - subprocess.run(ln_cmd, executable=cmd, shell=True, check=True) - return str(path) + symlink_python_library(target_lib, cmd) + return str(path) else: print( f"Unable to automatically locate library for executable {sys.executable}" @@ -189,17 +218,3 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) " in order to recompile python bindings. " ) raise SystemExit(msg) - - -def interactive_config(): - """Prompt user to configure installation variables.""" - version = input("Choose a version number to install (default is latest): ") - path = input( - "Provide a path for installation " - "(default is <user_dir>/Applications/OTB-<version>): " - ) - edit_env = ( - input("Modify user environment variables for this installation ? (y/n): ") - == "y" - ) - return version, path, edit_env -- GitLab From 6a9a53a5c0422f9af0cee17c64a36f5b3606ef08 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 17:20:16 +0200 Subject: [PATCH 319/399] DOC: update docstrings --- pyotb/install.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index 8f6c281..9d3402e 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -95,7 +95,7 @@ def update_windows_env(otb_path: Path): print(f"To undo this permanent setting, use 'reg.exe delete \"{key}\"'") -def recompile_python_bindings(path: str, cmd: str): +def recompile_python_bindings(path: Path, cmd: str): """Run subprocess command to recompile python bindings. Args: @@ -118,7 +118,7 @@ def symlink_python_library(target_lib: str, cmd: str): """Run subprocess command to recompile python bindings. Args: - path: path of the new OTB installation + target_lib: path of the missing python library cmd: path of the default system shell command """ @@ -135,6 +135,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) Args: version: OTB version tag, e.g. '8.1.2' path: installation directory, default is $HOME/Applications + edit_env: wether or not to permanently modify user's environment variables Returns: full path of the new installation -- GitLab From 1c7bb6fd538f739f5a8307ead782970afbcdec06 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 18:11:30 +0200 Subject: [PATCH 320/399] STYLE: codespell and pylint --- pyotb/install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index 9d3402e..345f43b 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -135,7 +135,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) Args: version: OTB version tag, e.g. '8.1.2' path: installation directory, default is $HOME/Applications - edit_env: wether or not to permanently modify user's environment variables + edit_env: whether or not to permanently modify user's environment variables Returns: full path of the new installation @@ -205,7 +205,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) recompile_python_bindings(path, cmd) return str(path) # Or use dirty cross version python symlink (only tested on Ubuntu) - elif sys.executable.startswith("/usr/bin"): + if sys.executable.startswith("/usr/bin"): symlink_python_library(target_lib, cmd) return str(path) else: -- GitLab From 3099a5bcd7210d7dd7699676e200a398020d60e9 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 18:16:42 +0200 Subject: [PATCH 321/399] FIX: ensure to exit python if version requirement is not satisfied for Windows --- pyotb/install.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index 345f43b..9a59842 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -154,8 +154,9 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) otb_major = int(version[0]) check, expected = check_versions(sysname, python_minor, otb_major) if sysname == "Win64" and not check: - print(f"Python 3.{expected} is required to import bindings on Windows.") - return "" + raise SystemExit( + f"Python 3.{expected} is required to import bindings on Windows." + ) # Fetch archive and run installer filename = f"OTB-{version}-{sysname}.{ext}" -- GitLab From 29530682207006e1ee2343b6c00019bf9c33afa7 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 18:46:24 +0200 Subject: [PATCH 322/399] Fix Windows install custom path + hints about registry --- pyotb/install.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index 9a59842..f6fc07c 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -90,9 +90,10 @@ def update_windows_env(otb_path: Path): winreg.SetValueEx(reg_key, "OTB_ROOT", 0, winreg.REG_EXPAND_SZ, str(otb_path)) print( "##### Environment variable 'OTB_ROOT' added successfully to user's hive." + "You'll need to login / logout to apply this change." ) - key = "HKEY_CURRENT_USER\\Environment\\OTB_ROOT" - print(f"To undo this permanent setting, use 'reg.exe delete \"{key}\"'") + reg_cmd = "reg.exe delete HKEY_CURRENT_USER\\Environment /v OTB_ROOT /f" + print(f"To undo this permanent setting, use '{reg_cmd}'") def recompile_python_bindings(path: Path, cmd: str): @@ -174,7 +175,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) if sysname == "Win64": with zipfile.ZipFile(tmpfile) as zipf: print("##### Extracting zip file...\n") - zipf.extractall(path.parent) + zipf.extractall(path.parent if default_path else path) else: install_cmd = f"{cmd} {tmpfile} --target {path} --accept" print(f"##### Executing '{install_cmd}'\n") -- GitLab From 9609826e8df9318b9250dccdf0ac05117cc13fea Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 18:51:36 +0200 Subject: [PATCH 323/399] FIX: user custom path must be parent dir --- pyotb/install.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index f6fc07c..7f42bc2 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -15,8 +15,8 @@ def interactive_config(): """Prompt user to configure installation variables.""" version = input("Choose a version number to install (default is latest): ") path = input( - "Provide a path for installation " - "(default is <user_dir>/Applications/OTB-<version>): " + "Provide a parent directory for installation " + "(default is <user_dir>/Applications/): " ) edit_env = ( input("Modify user environment variables for this installation ? (y/n): ") @@ -175,6 +175,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) if sysname == "Win64": with zipfile.ZipFile(tmpfile) as zipf: print("##### Extracting zip file...\n") + # Unzip will always create a dir with OTB-version name zipf.extractall(path.parent if default_path else path) else: install_cmd = f"{cmd} {tmpfile} --target {path} --accept" -- GitLab From c72358fc77b36b5b0c8b79a6c23f296815b8ab12 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 18:53:12 +0200 Subject: [PATCH 324/399] STYLE: remove print newline --- pyotb/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/install.py b/pyotb/install.py index 7f42bc2..178d39c 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -174,7 +174,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) path = Path.home() / "Applications" / tmpfile.stem if sysname == "Win64": with zipfile.ZipFile(tmpfile) as zipf: - print("##### Extracting zip file...\n") + print("##### Extracting zip file...") # Unzip will always create a dir with OTB-version name zipf.extractall(path.parent if default_path else path) else: -- GitLab From 2cfeb2ea6af5f05d1877ab74a26031ad34022dd7 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 18:58:19 +0200 Subject: [PATCH 325/399] STYLE: run pylint --- pyotb/install.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index 178d39c..cb7746b 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -211,11 +211,10 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) if sys.executable.startswith("/usr/bin"): symlink_python_library(target_lib, cmd) return str(path) - else: - print( - f"Unable to automatically locate library for executable {sys.executable}" - f"You'll need to manually symlink that one file to {target_lib}" - ) + print( + f"Unable to automatically locate library for executable {sys.executable}" + f"You could manually create a symlink from that file to {target_lib}" + ) # TODO: support for auto build deps install using brew, apt, pacman/yay, yum... msg = ( "\nYou need to install 'cmake', 'python3-dev' and 'libgl1-mesa-dev'" -- GitLab From 7b781eda246262ae098e025c0245f757b485835f Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 19:43:00 +0200 Subject: [PATCH 326/399] ENH: better messages in cli --- pyotb/install.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index cb7746b..6ce23c2 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -13,16 +13,12 @@ from shutil import which def interactive_config(): """Prompt user to configure installation variables.""" - version = input("Choose a version number to install (default is latest): ") + version = input("OTB version to download (default is latest): ") path = input( - "Provide a parent directory for installation " - "(default is <user_dir>/Applications/): " + "Parent directory for installation (default is <user_dir>/Applications/): " ) - edit_env = ( - input("Modify user environment variables for this installation ? (y/n): ") - == "y" - ) - return version, path, edit_env + env = input("Permanently change user's environment variables ? (y/n): ") == "y" + return version, path, env def otb_latest_release_tag(): @@ -89,11 +85,11 @@ def update_windows_env(otb_path: Path): ) as reg_key: winreg.SetValueEx(reg_key, "OTB_ROOT", 0, winreg.REG_EXPAND_SZ, str(otb_path)) print( - "##### Environment variable 'OTB_ROOT' added successfully to user's hive." + "##### Environment variable 'OTB_ROOT' added to user's registry." "You'll need to login / logout to apply this change." ) reg_cmd = "reg.exe delete HKEY_CURRENT_USER\\Environment /v OTB_ROOT /f" - print(f"To undo this permanent setting, use '{reg_cmd}'") + print(f"To undo this, you may use '{reg_cmd}'") def recompile_python_bindings(path: Path, cmd: str): -- GitLab From cd6662d8b3bf3dfcb1dc6cba16ea47abd88a2e74 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 19:45:21 +0200 Subject: [PATCH 327/399] ENH: prompt messages --- pyotb/install.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index 6ce23c2..1e4eb6c 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -13,10 +13,9 @@ from shutil import which def interactive_config(): """Prompt user to configure installation variables.""" - version = input("OTB version to download (default is latest): ") - path = input( - "Parent directory for installation (default is <user_dir>/Applications/): " - ) + version = input("Choose a version to download (default is latest): ") + default_dir = f"<user_dir>{os.path.sep}Applications{os.path.sep}" + path = input(f"Parent directory for installation (default is {default_dir}): ") env = input("Permanently change user's environment variables ? (y/n): ") == "y" return version, path, env -- GitLab From 66c308044da1560dc7ffdfa05675443c01d047e4 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 19:46:45 +0200 Subject: [PATCH 328/399] ENH: use pathlib to print absolute path of default install dir --- pyotb/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/install.py b/pyotb/install.py index 1e4eb6c..ad3cc76 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -14,7 +14,7 @@ from shutil import which def interactive_config(): """Prompt user to configure installation variables.""" version = input("Choose a version to download (default is latest): ") - default_dir = f"<user_dir>{os.path.sep}Applications{os.path.sep}" + default_dir = Path.home() / "Applications" path = input(f"Parent directory for installation (default is {default_dir}): ") env = input("Permanently change user's environment variables ? (y/n): ") == "y" return version, path, env -- GitLab From ddb0863c47bfe184e353cd980b3a69e572f395d2 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 19:49:24 +0200 Subject: [PATCH 329/399] ENH: function names --- pyotb/install.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index ad3cc76..5a08592 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -57,7 +57,7 @@ def check_versions(sysname: str, python_minor: int, otb_major: int): return False, expected -def update_unix_env(otb_path: Path): +def env_config_unix(otb_path: Path): """Update env profile for current user with new otb_env.profile call. Args: @@ -70,7 +70,7 @@ def update_unix_env(otb_path: Path): print(f"##### Added new environment variables to {profile}") -def update_windows_env(otb_path: Path): +def env_config_windows(otb_path: Path): """Update registry hive for current user with new OTB_ROOT env variable. Args: @@ -84,7 +84,7 @@ def update_windows_env(otb_path: Path): ) as reg_key: winreg.SetValueEx(reg_key, "OTB_ROOT", 0, winreg.REG_EXPAND_SZ, str(otb_path)) print( - "##### Environment variable 'OTB_ROOT' added to user's registry." + "##### Environment variable 'OTB_ROOT' added to user's registry. " "You'll need to login / logout to apply this change." ) reg_cmd = "reg.exe delete HKEY_CURRENT_USER\\Environment /v OTB_ROOT /f" @@ -181,9 +181,9 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) # Add env variable to profile if edit_env: if sysname == "Win64": - update_windows_env(path) + env_config_windows(path) else: - update_unix_env(path) + env_config_unix(path) elif not default_path: ext = "bat" if sysname == "Win64" else "profile" print( -- GitLab From 3ac9f25ec53af5d385333fde2e1139a85cbd8f93 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 21:34:15 +0200 Subject: [PATCH 330/399] FIX: always try the symlink trick --- pyotb/install.py | 79 ++++++++++++++++-------------------------------- 1 file changed, 26 insertions(+), 53 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index 5a08592..d8b6c60 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -91,41 +91,7 @@ def env_config_windows(otb_path: Path): print(f"To undo this, you may use '{reg_cmd}'") -def recompile_python_bindings(path: Path, cmd: str): - """Run subprocess command to recompile python bindings. - - Args: - path: path of the new OTB installation - cmd: path of the default system shell command - - """ - print("\n##### Recompiling python bindings...") - ctest_cmd = ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV" - try: - subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True) - except subprocess.CalledProcessError as err: - raise SystemExit( - "Unable to recompile python bindings, " - "some dependencies (libgl1) may require manual installation." - ) from err - - -def symlink_python_library(target_lib: str, cmd: str): - """Run subprocess command to recompile python bindings. - - Args: - target_lib: path of the missing python library - cmd: path of the default system shell command - - """ - lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so" - if Path(lib).exists(): - print(f"##### Creating symbolic links: {lib} -> {target_lib}") - ln_cmd = f'ln -sf "{lib}" "{target_lib}"' - subprocess.run(ln_cmd, executable=cmd, shell=True, check=True) - - -def install_otb(version: str = "latest", path: str = "", edit_env: bool = False): +def install_otb(version: str = "latest", path: str = "", edit_env: bool = True): """Install pre-compiled OTB binaries in path, use latest release by default. Args: @@ -153,7 +119,6 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) raise SystemExit( f"Python 3.{expected} is required to import bindings on Windows." ) - # Fetch archive and run installer filename = f"OTB-{version}-{sysname}.{ext}" url = f"https://www.orfeo-toolbox.org/packages/archives/OTB/{filename}" @@ -194,25 +159,33 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False) if check: return str(path) - # Else recompile bindings : this may fail because of OpenGL + # Here version check failed, try recompile bindings : can fail because of OpenGL suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else "so.1.0" target_lib = f"{path}/lib/libpython3.{expected}.{suffix}" - can_compile = which("ctest") and which("python3-config") - # Google Colab ships with cmake and python3-dev, but not libgl1-mesa-dev - if can_compile and "COLAB_RELEASE_TAG" not in os.environ: - recompile_python_bindings(path, cmd) - return str(path) - # Or use dirty cross version python symlink (only tested on Ubuntu) - if sys.executable.startswith("/usr/bin"): - symlink_python_library(target_lib, cmd) - return str(path) + if which("ctest") and which("python3-config"): + try: + print("\n##### Python version mismatch. Trying to recompile bindings...") + ctest_cmd = ( + ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV" + ) + subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), check=True) + return str(path) + except subprocess.CalledProcessError: + print("\nCompilation failed. ") print( - f"Unable to automatically locate library for executable {sys.executable}" - f"You could manually create a symlink from that file to {target_lib}" + "You need cmake, python3-dev and libgl1-mesa-dev installed. " + "Trying to symlink libraries instead - this may fail with newest versions." ) - # TODO: support for auto build deps install using brew, apt, pacman/yay, yum... - msg = ( - "\nYou need to install 'cmake', 'python3-dev' and 'libgl1-mesa-dev'" - " in order to recompile python bindings. " + # TODO: support for sudo auto build deps install using apt, pacman/yay, brew... + # Else use dirty cross version python symlink (only tested on Ubuntu) + if sys.executable.startswith("/usr/bin"): + lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so" + if Path(lib).exists(): + print(f"##### Creating symbolic links: {lib} -> {target_lib}") + ln_cmd = f'ln -sf "{lib}" "{target_lib}"' + subprocess.run(ln_cmd, executable=cmd, shell=True, check=True) + return str(path) + raise SystemExit( + f"Unable to automatically locate library for executable '{sys.executable}', " + f"you could manually create a symlink from that file to {target_lib}" ) - raise SystemExit(msg) -- GitLab From a4c12ab85249ee95b189d3c4787db66ec861a8ea Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 22:32:52 +0200 Subject: [PATCH 331/399] FIX: shell=True is required for process to complete --- pyotb/install.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index d8b6c60..ed54a25 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -134,7 +134,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True): path = Path.home() / "Applications" / tmpfile.stem if sysname == "Win64": with zipfile.ZipFile(tmpfile) as zipf: - print("##### Extracting zip file...") + print("##### Extracting zip file") # Unzip will always create a dir with OTB-version name zipf.extractall(path.parent if default_path else path) else: @@ -164,14 +164,15 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True): target_lib = f"{path}/lib/libpython3.{expected}.{suffix}" if which("ctest") and which("python3-config"): try: - print("\n##### Python version mismatch. Trying to recompile bindings...") + print("\n!!!!! Python version mismatch, trying to recompile bindings") ctest_cmd = ( - ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV" + ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -V" ) - subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), check=True) + print(f"##### Executing '{ctest_cmd}'") + subprocess.run(ctest_cmd, cwd=path, check=True, shell=True) return str(path) except subprocess.CalledProcessError: - print("\nCompilation failed. ") + print("\nCompilation failed.") print( "You need cmake, python3-dev and libgl1-mesa-dev installed. " "Trying to symlink libraries instead - this may fail with newest versions." -- GitLab From aff854d8395f3c287517ec6da8f8399f29c0a2b5 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 22:34:00 +0200 Subject: [PATCH 332/399] ENH: use SystemError instead of EnvironmentError --- pyotb/helpers.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 812b72e..124abaa 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -69,7 +69,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True): import otbApplication as otb # pylint: disable=import-outside-toplevel return otb - except EnvironmentError as e: + except SystemError as e: raise SystemExit(f"Failed to import OTB with prefix={prefix}") from e except ImportError as e: __suggest_fix_import(str(e), prefix) @@ -110,7 +110,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True): import otbApplication as otb # pylint: disable=import-outside-toplevel return otb - except EnvironmentError as e: + except SystemError as e: raise SystemExit("Auto setup for OTB env failed. Exiting.") from e # Help user to fix this except ImportError as e: @@ -136,7 +136,7 @@ def set_environment(prefix: str): # External libraries lib_dir = __find_lib(prefix) if not lib_dir: - raise EnvironmentError("Can't find OTB external libraries") + raise SystemError("Can't find OTB external libraries") # This does not seems to work if sys.platform == "linux" and built_from_source: new_ld_path = f"{lib_dir}:{os.environ.get('LD_LIBRARY_PATH') or ''}" @@ -144,7 +144,7 @@ def set_environment(prefix: str): # Add python bindings directory first in PYTHONPATH otb_api = __find_python_api(lib_dir) if not otb_api: - raise EnvironmentError("Can't find OTB Python API") + raise SystemError("Can't find OTB Python API") if otb_api not in sys.path: sys.path.insert(0, otb_api) # Add /bin first in PATH, in order to avoid conflicts with another GDAL install @@ -154,7 +154,7 @@ def set_environment(prefix: str): if Path(apps_path).exists(): os.environ["OTB_APPLICATION_PATH"] = apps_path else: - raise EnvironmentError("Can't find OTB applications directory") + raise SystemError("Can't find OTB applications directory") os.environ["LC_NUMERIC"] = "C" os.environ["GDAL_DRIVER_PATH"] = "disable" @@ -170,7 +170,7 @@ def set_environment(prefix: str): gdal_data = str(prefix / "share/data") proj_lib = str(prefix / "share/proj") else: - raise EnvironmentError( + raise SystemError( f"Can't find GDAL location with current OTB prefix '{prefix}' or in /usr" ) os.environ["GDAL_DATA"] = gdal_data -- GitLab From 59a63f396157bca70195adbf772976ed2c920a3e Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 22:34:20 +0200 Subject: [PATCH 333/399] FIX: do not exit python when interactive --- pyotb/helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 124abaa..3560178 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -99,6 +99,8 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True): if not prefix and hasattr(sys, "ps1"): if input("OTB is missing. Do you want to install it ? (y/n): ") == "y": return find_otb(install_otb(*interactive_config())) + else: + raise SystemError("OTB libraries not found on disk. ") elif not prefix: raise SystemExit( "OTB libraries not found on disk. " -- GitLab From 5cc50cea8105f88dcd3d8859e190a2195db3b463 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 22:45:58 +0200 Subject: [PATCH 334/399] STYLE: run pylint --- pyotb/helpers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 3560178..9322041 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -99,8 +99,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True): if not prefix and hasattr(sys, "ps1"): if input("OTB is missing. Do you want to install it ? (y/n): ") == "y": return find_otb(install_otb(*interactive_config())) - else: - raise SystemError("OTB libraries not found on disk. ") + raise SystemError("OTB libraries not found on disk. ") elif not prefix: raise SystemExit( "OTB libraries not found on disk. " -- GitLab From 783d95627abf7077c76a21e05203bd1472ad1894 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 11 Aug 2023 22:49:50 +0200 Subject: [PATCH 335/399] STYLE: remove useless elif --- pyotb/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 9322041..907ab88 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -100,7 +100,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True): if input("OTB is missing. Do you want to install it ? (y/n): ") == "y": return find_otb(install_otb(*interactive_config())) raise SystemError("OTB libraries not found on disk. ") - elif not prefix: + if not prefix: raise SystemExit( "OTB libraries not found on disk. " "To install it, open an interactive python shell and 'import pyotb'" -- GitLab From eef2ccf81751ba6af7b77b065ede257a68f5df1b Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sat, 12 Aug 2023 00:07:49 +0200 Subject: [PATCH 336/399] ENH: show pylint TODO in IDE, but not in CI --- .gitlab-ci.yml | 2 +- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c6e521..f0846f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,7 +47,7 @@ pylint: before_script: - pip install pylint script: - - pylint $PWD/pyotb + - pylint $PWD/pyotb --disable=fixme codespell: extends: .static_analysis diff --git a/pyproject.toml b/pyproject.toml index 30804d4..fdaa0a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,6 @@ max-line-length = 88 max-module-lines = 2000 good-names = ["x", "y", "i", "j", "k", "e"] disable = [ - "fixme", "line-too-long", "too-many-locals", "too-many-branches", -- GitLab From 87de0932f8e5c9dcfc878c2152dd8fcf51e4fa66 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 13 Aug 2023 10:51:58 +0200 Subject: [PATCH 337/399] ENH: comments and docstrings --- pyotb/helpers.py | 3 +-- pyotb/install.py | 16 +++++++--------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 907ab88..b043f13 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -302,8 +302,7 @@ def __suggest_fix_import(error_message: str, prefix: str): ) if which("ctest") and which("python3-config"): logger.critical( - "To recompile python bindings, use " - "'cd %s ; source otbenv.profile ; " + "To compile, use 'cd %s ; source otbenv.profile ; " "ctest -S share/otb/swig/build_wrapping.cmake -VV'", prefix, ) diff --git a/pyotb/install.py b/pyotb/install.py index ed54a25..76919e2 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -40,7 +40,7 @@ def check_versions(sysname: str, python_minor: int, otb_major: int): python_minor: minor version of python otb_major: major version of OTB to be installed Returns: - True if requirements are satisfied + (True, 0) or (False, expected_version) if case of version conflict """ if sysname == "Win64": expected = 5 if otb_major in (6, 7) else 7 @@ -65,7 +65,7 @@ def env_config_unix(otb_path: Path): """ profile = Path.home() / ".profile" - with open(profile, "a", encoding="utf-8") as buf: + with profile.open("a", encoding="utf-8") as buf: buf.write(f'\n. "{otb_path}/otbenv.profile"\n') print(f"##### Added new environment variables to {profile}") @@ -155,18 +155,16 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True): f"Remember to call '{path}{os.sep}otbenv.{ext}' before importing pyotb, " f"or add 'OTB_ROOT=\"{path}\"' to your env variables." ) - # No recompilation or symlink required + # Requirements are met, no recompilation or symlink required if check: return str(path) - # Here version check failed, try recompile bindings : can fail because of OpenGL - suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else "so.1.0" - target_lib = f"{path}/lib/libpython3.{expected}.{suffix}" + # Else try recompile bindings : can fail because of OpenGL if which("ctest") and which("python3-config"): try: print("\n!!!!! Python version mismatch, trying to recompile bindings") ctest_cmd = ( - ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -V" + ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -Vv" ) print(f"##### Executing '{ctest_cmd}'") subprocess.run(ctest_cmd, cwd=path, check=True, shell=True) @@ -174,8 +172,8 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True): except subprocess.CalledProcessError: print("\nCompilation failed.") print( - "You need cmake, python3-dev and libgl1-mesa-dev installed. " - "Trying to symlink libraries instead - this may fail with newest versions." + "You need cmake, python3-dev and libgl1-mesa-dev installed." + "\nTrying to symlink libraries instead - this may fail with newest versions." ) # TODO: support for sudo auto build deps install using apt, pacman/yay, brew... # Else use dirty cross version python symlink (only tested on Ubuntu) -- GitLab From 128794573474a0ce6f4a925b656492c1185729b4 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 13 Aug 2023 10:52:15 +0200 Subject: [PATCH 338/399] FIX: try symlink python libs for executable in /usr/local/bin --- pyotb/__init__.py | 2 +- pyotb/install.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pyotb/__init__.py b/pyotb/__init__.py index c134e58..aff3736 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """This module provides convenient python wrapping of otbApplications.""" -__version__ = "2.0.0.dev5" +__version__ = "2.0.0.dev6" from .install import install_otb from .helpers import logger, set_logger_level diff --git a/pyotb/install.py b/pyotb/install.py index 76919e2..07922c1 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -177,10 +177,15 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True): ) # TODO: support for sudo auto build deps install using apt, pacman/yay, brew... # Else use dirty cross version python symlink (only tested on Ubuntu) - if sys.executable.startswith("/usr/bin"): - lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so" + suffix = "so.1.0" if otb_major >= 8 else f"so.rh-python3{expected}-1.0" + target_lib = f"{path}/lib/libpython3.{expected}.{suffix}" + for prefix in ("/usr", "/usr/local"): + if not sys.executable.startswith(f"{prefix}/bin"): + continue + lib_dir = f"{prefix}/lib" + ("/x86_64-linux-gnu" if prefix == "/usr" else "") + lib = f"{lib_dir}/libpython3.{sys.version_info.minor}.so" if Path(lib).exists(): - print(f"##### Creating symbolic links: {lib} -> {target_lib}") + print(f"##### Creating symbolic link: {lib} -> {target_lib}") ln_cmd = f'ln -sf "{lib}" "{target_lib}"' subprocess.run(ln_cmd, executable=cmd, shell=True, check=True) return str(path) -- GitLab From 67cd415347c9da6b1100018d7829080e0498e6c3 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 13 Aug 2023 11:42:09 +0200 Subject: [PATCH 339/399] ENH: use sysconfig.get_config_var to locate python library --- pyotb/helpers.py | 69 +++++++++++++++++++++++------------------------- pyotb/install.py | 18 ++++++------- 2 files changed, 41 insertions(+), 46 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index b043f13..3fc93b3 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -3,6 +3,7 @@ import logging import os import sys +import sysconfig from pathlib import Path from shutil import which @@ -291,42 +292,7 @@ def __suggest_fix_import(error_message: str, prefix: str): """Help user to fix the OTB installation with appropriate log messages.""" logger.critical("An error occurred while importing OTB Python API") logger.critical("OTB error message was '%s'", error_message) - if sys.platform == "linux": - if error_message.startswith("libpython3."): - logger.critical( - "It seems like you need to symlink or recompile python bindings" - ) - if sys.executable.startswith("/usr/bin"): - lib = ( - f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so" - ) - if which("ctest") and which("python3-config"): - logger.critical( - "To compile, use 'cd %s ; source otbenv.profile ; " - "ctest -S share/otb/swig/build_wrapping.cmake -VV'", - prefix, - ) - elif Path(lib).exists(): - expected = int(error_message[11]) - if expected != sys.version_info.minor: - logger.critical( - "Python library version mismatch (OTB expected 3.%s) : " - "a symlink may not work, depending on your python version", - expected, - ) - target = f"{prefix}/lib/libpython3.{expected}.so.1.0" - logger.critical("If using OTB>=8.0, use 'ln -s %s %s'", lib, target) - else: - logger.critical( - "You may need to install cmake, python3-dev and mesa's libgl" - " in order to recompile python bindings" - ) - else: - logger.critical( - "Unable to automatically locate python dynamic library of %s", - sys.executable, - ) - elif sys.platform == "win32": + if sys.platform == "win32": if error_message.startswith("DLL load failed"): if sys.version_info.minor != 7: logger.critical( @@ -337,6 +303,37 @@ def __suggest_fix_import(error_message: str, prefix: str): "It seems that your env variables aren't properly set," " first use 'call otbenv.bat' then try to import pyotb once again" ) + elif error_message.startswith("libpython3."): + logger.critical( + "It seems like you need to symlink or recompile python bindings" + ) + if ( + sys.executable.startswith("/usr/bin") + and which("ctest") + and which("python3-config") + ): + logger.critical( + "To compile, use 'cd %s ; source otbenv.profile ; " + "ctest -S share/otb/swig/build_wrapping.cmake -VV'", + prefix, + ) + return + logger.critical( + "You may need to install cmake, python3-dev and mesa's libgl" + " in order to recompile python bindings" + ) + expected = int(error_message[11]) + if expected != sys.version_info.minor: + logger.critical( + "Python library version mismatch (OTB expected 3.%s) : " + "a symlink may not work, depending on your python version", + expected, + ) + lib_dir = sysconfig.get_config_var("LIBDIR") + lib = f"{lib_dir}/libpython3.{sys.version_info.minor}.so" + if Path(lib).exists(): + target = f"{prefix}/lib/libpython3.{expected}.so.1.0" + logger.critical("If using OTB>=8.0, try 'ln -sf %s %s'", lib, target) logger.critical( "You can verify installation requirements for your OS at %s", DOCS_URL ) diff --git a/pyotb/install.py b/pyotb/install.py index 07922c1..9de6ac2 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -4,6 +4,7 @@ import os import re import subprocess import sys +import sysconfig import tempfile import urllib.request import zipfile @@ -179,16 +180,13 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True): # Else use dirty cross version python symlink (only tested on Ubuntu) suffix = "so.1.0" if otb_major >= 8 else f"so.rh-python3{expected}-1.0" target_lib = f"{path}/lib/libpython3.{expected}.{suffix}" - for prefix in ("/usr", "/usr/local"): - if not sys.executable.startswith(f"{prefix}/bin"): - continue - lib_dir = f"{prefix}/lib" + ("/x86_64-linux-gnu" if prefix == "/usr" else "") - lib = f"{lib_dir}/libpython3.{sys.version_info.minor}.so" - if Path(lib).exists(): - print(f"##### Creating symbolic link: {lib} -> {target_lib}") - ln_cmd = f'ln -sf "{lib}" "{target_lib}"' - subprocess.run(ln_cmd, executable=cmd, shell=True, check=True) - return str(path) + lib_dir = sysconfig.get_config_var("LIBDIR") + lib = f"{lib_dir}/libpython3.{sys.version_info.minor}.so" + if Path(lib).exists(): + print(f"##### Creating symbolic link: {lib} -> {target_lib}") + ln_cmd = f'ln -sf "{lib}" "{target_lib}"' + subprocess.run(ln_cmd, executable=cmd, shell=True, check=True) + return str(path) raise SystemExit( f"Unable to automatically locate library for executable '{sys.executable}', " f"you could manually create a symlink from that file to {target_lib}" -- GitLab From bf468ceda3b26eae246f14c4edc636e3697d0ae5 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Sun, 13 Aug 2023 13:54:03 +0200 Subject: [PATCH 340/399] DOC: comments --- pyotb/install.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index 9de6ac2..fe42fd7 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -34,7 +34,7 @@ def otb_latest_release_tag(): def check_versions(sysname: str, python_minor: int, otb_major: int): - """Verify if python version is compatible with OTB version. + """Verify if python version is compatible with major OTB version. Args: sysname: OTB's system name convention (Linux64, Darwin64, Win64) @@ -72,7 +72,7 @@ def env_config_unix(otb_path: Path): def env_config_windows(otb_path: Path): - """Update registry hive for current user with new OTB_ROOT env variable. + """Update user's registry hive with new OTB_ROOT env variable. Args: otb_path: path of the new OTB installation @@ -161,23 +161,29 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True): return str(path) # Else try recompile bindings : can fail because of OpenGL - if which("ctest") and which("python3-config"): + # Here we check for /usr/bin because CMake's will find_package() only there + if ( + sys.executable.startswith("/usr/bin") + and which("ctest") + and which("python3-config") + ): try: print("\n!!!!! Python version mismatch, trying to recompile bindings") ctest_cmd = ( - ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -Vv" + ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -V" ) print(f"##### Executing '{ctest_cmd}'") subprocess.run(ctest_cmd, cwd=path, check=True, shell=True) return str(path) except subprocess.CalledProcessError: print("\nCompilation failed.") + # TODO: support for sudo auto build deps install using apt, pacman/yay, brew... print( "You need cmake, python3-dev and libgl1-mesa-dev installed." "\nTrying to symlink libraries instead - this may fail with newest versions." ) - # TODO: support for sudo auto build deps install using apt, pacman/yay, brew... - # Else use dirty cross version python symlink (only tested on Ubuntu) + + # Finally try with cross version python symlink (only tested on Ubuntu) suffix = "so.1.0" if otb_major >= 8 else f"so.rh-python3{expected}-1.0" target_lib = f"{path}/lib/libpython3.{expected}.{suffix}" lib_dir = sysconfig.get_config_var("LIBDIR") -- GitLab From f62ec2ff89d05dd2bf2b180652bda3897ec0a651 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 15 Aug 2023 11:40:32 +0200 Subject: [PATCH 341/399] ENH: do not raise SystemExit in interactive mode --- pyotb/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/install.py b/pyotb/install.py index fe42fd7..47a6ee7 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -193,7 +193,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True): ln_cmd = f'ln -sf "{lib}" "{target_lib}"' subprocess.run(ln_cmd, executable=cmd, shell=True, check=True) return str(path) - raise SystemExit( + raise SystemError( f"Unable to automatically locate library for executable '{sys.executable}', " f"you could manually create a symlink from that file to {target_lib}" ) -- GitLab From 2a357eaa5ad383e821eb883411d5e4a149ff8cea Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 15 Aug 2023 11:41:05 +0200 Subject: [PATCH 342/399] CI: shorter test job names --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f0846f0..27cb2d6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -84,7 +84,7 @@ test_install: before_script: - pip install pytest pytest-cov -test_module_core: +module_core: extends: .tests variables: OTB_LOGGER_LEVEL: INFO @@ -101,7 +101,7 @@ test_module_core: - curl -fsLI $PLEIADES_IMG_URL - python3 -m pytest -vv --junitxml=test-module-core.xml --cov-report xml:coverage.xml tests/test_core.py -test_pipeline_permutations: +pipeline_permutations: extends: .tests variables: OTB_LOGGER_LEVEL: WARNING -- GitLab From 08b9a567dee508666d026d05841398a4a0a925ff Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 16 Aug 2023 10:19:09 +0200 Subject: [PATCH 343/399] CI: remove unused variables --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 27cb2d6..48b6a18 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -79,8 +79,6 @@ test_install: variables: OTB_ROOT: /opt/otb LD_LIBRARY_PATH: /opt/otb/lib - SPOT_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif - PLEIADES_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Baseline/OTB/Images/prTvOrthoRectification_pleiades-1_noDEM.tif before_script: - pip install pytest pytest-cov -- GitLab From b0b59dccd6003d35987719d45eed03813fa2160b Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 16 Aug 2023 10:37:04 +0200 Subject: [PATCH 344/399] Revert "CI: remove unused variables" This reverts commit 08b9a567dee508666d026d05841398a4a0a925ff. --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 48b6a18..27cb2d6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -79,6 +79,8 @@ test_install: variables: OTB_ROOT: /opt/otb LD_LIBRARY_PATH: /opt/otb/lib + SPOT_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif + PLEIADES_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Baseline/OTB/Images/prTvOrthoRectification_pleiades-1_noDEM.tif before_script: - pip install pytest pytest-cov -- GitLab From 2e58ff1a0d9fc455e03b3cf82ec49fe361424e28 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 28 Aug 2023 12:19:22 +0200 Subject: [PATCH 345/399] FIX: #116 --- pyotb/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index e4ca2b6..4d0ab82 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -856,7 +856,9 @@ class App(OTBObject): path, ) elif isinstance(path, (str, Path)) and self.output_key: - kwargs.update({self.output_key: str(path)}) + kwargs[self.output_key] = str(path) + elif not path and self.output_image_key in self.parameters: + kwargs[self.output_key] = self.parameters[self.output_key] elif path is not None: raise TypeError(f"{self.name}: unsupported filepath type ({type(path)})") if not (kwargs or any(k in self._settings for k in self._out_param_types)): -- GitLab From 1dc6f9c60fdf9905f8f90289adbe69635b641187 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 28 Aug 2023 12:19:26 +0200 Subject: [PATCH 346/399] FIX: #117 --- pyotb/core.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 4d0ab82..045721e 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -732,9 +732,10 @@ class App(OTBObject): ) try: if self.is_input(key): - self.__set_param(key, self.__check_input_param(obj)) - else: - self.__set_param(key, obj) + obj = self.__check_input_param(obj) + elif self.is_output(key): + obj = self.__check_output_param(obj) + self.__set_param(key, obj) except (RuntimeError, TypeError, ValueError, KeyError) as e: raise RuntimeError( f"{self.name}: error before execution, while setting parameter '{key}' to '{obj}': {e})" @@ -972,15 +973,7 @@ class App(OTBObject): def __check_input_param( self, obj: list | OTBObject | str | Path ) -> list | OTBObject | str: - """Check the type and value of an input param. - - Args: - obj: input parameter value - - Returns: - object, string with new /vsi prefix(es) if needed - - """ + """Check the type and value of an input parameter, add vsi prefixes if needed.""" if isinstance(obj, list): return [self.__check_input_param(o) for o in obj] # May be we could add some checks here @@ -1012,8 +1005,20 @@ class App(OTBObject): return obj raise TypeError(f"{self.name}: wrong input parameter type ({type(obj)})") + def __check_output_param( + self, obj: list | OTBObject | str | Path + ) -> list | OTBObject | str: + """Check the type and value of an output parameter.""" + if isinstance(obj, list): + return [self.__check_output_param(o) for o in obj] + if isinstance(obj, Path): + obj = str(obj) + if isinstance(obj, str): + return obj + raise TypeError(f"{self.name}: wrong output parameter type ({type(obj)})") + def __set_param( - self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any] + self, key: str, obj: str | float | list | tuple | OTBObject | otb.Application ): """Set one parameter, decide which otb.Application method to use depending on target object.""" if obj is None or (isinstance(obj, (list, tuple)) and not obj): -- GitLab From 23845245262c82522e4ea1366b5661ec06ffe3e5 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 28 Aug 2023 12:22:52 +0200 Subject: [PATCH 347/399] TERST: add test for output Path object param --- tests/test_core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index d172739..5a32a6f 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -141,6 +141,8 @@ def test_xy_to_rowcol(): def test_write(): assert INPUT.write("/dev/shm/test_write.tif", ext_fname="nodata=0") INPUT["out"].filepath.unlink() + assert INPUT.write(Path("/dev/shm/test_write.tif"), ext_fname="nodata=0") + INPUT["out"].filepath.unlink() # Frozen frozen_app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True) assert frozen_app.write("/dev/shm/test_frozen_app_write.tif") -- GitLab From 799fe2e61e36872de2d8651994b4c135abde79c0 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 28 Aug 2023 12:27:09 +0200 Subject: [PATCH 348/399] TEST: check output dtype is set in frozen mode --- tests/test_core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_core.py b/tests/test_core.py index 5a32a6f..1de0937 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -150,7 +150,8 @@ def test_write(): frozen_app_init_with_outfile = pyotb.BandMath( INPUT, exp="im1b1", out="/dev/shm/test_frozen_app_write.tif", frozen=True ) - assert frozen_app_init_with_outfile.write() + assert frozen_app_init_with_outfile.write(pixel_type="uint16") + assert frozen_app_init_with_outfile.dtype == "uint16" frozen_app_init_with_outfile["out"].filepath.unlink() -- GitLab From 5afdc751a87638a8f639743e2e631527643860ce Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 28 Aug 2023 13:28:52 +0200 Subject: [PATCH 349/399] STYLE: black and docstrings --- pyotb/core.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 045721e..8bb493b 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1005,9 +1005,7 @@ class App(OTBObject): return obj raise TypeError(f"{self.name}: wrong input parameter type ({type(obj)})") - def __check_output_param( - self, obj: list | OTBObject | str | Path - ) -> list | OTBObject | str: + def __check_output_param(self, obj: list | str | Path) -> list | str: """Check the type and value of an output parameter.""" if isinstance(obj, list): return [self.__check_output_param(o) for o in obj] -- GitLab From 6a113ac845c7fa81072d42859021a3f31294f5a3 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 28 Aug 2023 13:39:44 +0200 Subject: [PATCH 350/399] TEST: update tests for new summarize behaviour --- tests/pipeline_summary.json | 6 +++--- tests/test_core.py | 21 +++++++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/pipeline_summary.json b/tests/pipeline_summary.json index 56ebf8d..443f9ef 100644 --- a/tests/pipeline_summary.json +++ b/tests/pipeline_summary.json @@ -30,7 +30,7 @@ "interpolator.bco.radius": 2, "opt.rpc": 10, "opt.gridspacing": 4.0, - "io.in": "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" + "io.in": "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" } } ], @@ -67,7 +67,7 @@ "name": "BandMath", "parameters": { "il": [ - "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" + "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" ], "exp": "im1b1" } @@ -105,7 +105,7 @@ "name": "BandMath", "parameters": { "il": [ - "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" + "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif" ], "exp": "im1b1" } diff --git a/tests/test_core.py b/tests/test_core.py index 1de0937..fc4712c 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -7,7 +7,6 @@ from tests_data import * def test_app_parameters(): # Input / ExtractROI assert INPUT.parameters - assert INPUT.parameters["in"] == SPOT_IMG_URL assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304) # OrthoRectification app = pyotb.OrthoRectification(INPUT) @@ -53,19 +52,25 @@ def test_app_input_vsi(): pyotb.Input("/vsicurl/" + SPOT_IMG_URL) # Simple remote file info = pyotb.ReadImageInfo("https://fake.com/image.tif", frozen=True) - assert info.app.GetParameterValue("in") == "/vsicurl/https://fake.com/image.tif" - assert info.parameters["in"] == "https://fake.com/image.tif" + assert ( + info.app.GetParameterValue("in") + == info.parameters["in"] + == "/vsicurl/https://fake.com/image.tif" + ) # Compressed single file archive info = pyotb.ReadImageInfo("image.tif.zip", frozen=True) - assert info.app.GetParameterValue("in") == "/vsizip/image.tif.zip" - assert info.parameters["in"] == "image.tif.zip" + assert ( + info.app.GetParameterValue("in") + == info.parameters["in"] + == "/vsizip/image.tif.zip" + ) # File within compressed remote archive info = pyotb.ReadImageInfo("https://fake.com/archive.tar.gz/image.tif", frozen=True) assert ( info.app.GetParameterValue("in") + == info.parameters["in"] == "/vsitar//vsicurl/https://fake.com/archive.tar.gz/image.tif" ) - assert info.parameters["in"] == "https://fake.com/archive.tar.gz/image.tif" # Piped curl --> zip --> tiff ziped_tif_urls = ( "https://github.com/OSGeo/gdal/raw/master" @@ -348,8 +353,8 @@ def test_summarize_output_obj(): def test_summarize_strip_output(): - in_fn = SPOT_IMG_URL - in_fn_w_ext = SPOT_IMG_URL + "?&skipcarto=1" + in_fn = "/vsicurl/" + SPOT_IMG_URL + in_fn_w_ext = "/vsicurl/" + SPOT_IMG_URL + "?&skipcarto=1" out_fn = "/dev/shm/output.tif" out_fn_w_ext = out_fn + "?&box=10:10:10:10" -- GitLab From d2043b98148b7ef310afe0d2f651ea0dad053456 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 11 Oct 2023 11:50:32 +0200 Subject: [PATCH 351/399] ENH: ensure Input object init is quiet --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 8bb493b..db05bed 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1521,7 +1521,7 @@ class Input(App): filepath: Anything supported by GDAL (local file on the filesystem, remote resource e.g. /vsicurl/.., etc.) """ - super().__init__("ExtractROI", {"in": filepath}, frozen=True) + super().__init__("ExtractROI", {"in": filepath}, quiet=True, frozen=True) self._name = f"Input from {filepath}" if not filepath.startswith(("/vsi", "http://", "https://", "ftp://")): filepath = Path(filepath) -- GitLab From aaf8b9e1d300f319f14013158127ded94c79a456 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 11 Oct 2023 11:55:59 +0200 Subject: [PATCH 352/399] ENH: use attr output_image_key in write() --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index db05bed..15d021c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -859,7 +859,7 @@ class App(OTBObject): elif isinstance(path, (str, Path)) and self.output_key: kwargs[self.output_key] = str(path) elif not path and self.output_image_key in self.parameters: - kwargs[self.output_key] = self.parameters[self.output_key] + kwargs[self.output_key] = self.parameters[self.output_image_key] elif path is not None: raise TypeError(f"{self.name}: unsupported filepath type ({type(path)})") if not (kwargs or any(k in self._settings for k in self._out_param_types)): -- GitLab From def8449334668c26c25df74041a22b67ff9308e8 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 11 Oct 2023 11:57:22 +0200 Subject: [PATCH 353/399] ENH: use output_key to make it work with non image output --- pyotb/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 15d021c..aa5f52e 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -858,8 +858,8 @@ class App(OTBObject): ) elif isinstance(path, (str, Path)) and self.output_key: kwargs[self.output_key] = str(path) - elif not path and self.output_image_key in self.parameters: - kwargs[self.output_key] = self.parameters[self.output_image_key] + elif not path and self.output_key in self.parameters: + kwargs[self.output_key] = self.parameters[self.output_key] elif path is not None: raise TypeError(f"{self.name}: unsupported filepath type ({type(path)})") if not (kwargs or any(k in self._settings for k in self._out_param_types)): -- GitLab From 0404eeb53c8226797b409f3874f15ea2b9f2d02d Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 11 Oct 2023 12:00:41 +0200 Subject: [PATCH 354/399] STYLE: typo --- pyotb/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 5363937..6b53c2b 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -336,5 +336,5 @@ def __suggest_fix_import(error_message: str, prefix: str): ) -# Since helpers is the first module to be inititialized, this will prevent pyotb to run if OTB is not found +# Since helpers is the first module to be initialized, this will prevent pyotb to run if OTB is not found find_otb() -- GitLab From b4b282dd5e2c28172ea6186bf48dcd5146b88d58 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Wed, 11 Oct 2023 12:18:13 +0200 Subject: [PATCH 355/399] CI: bump dev version --- pyotb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/__init__.py b/pyotb/__init__.py index aff3736..40e6434 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """This module provides convenient python wrapping of otbApplications.""" -__version__ = "2.0.0.dev6" +__version__ = "2.0.0.dev7" from .install import install_otb from .helpers import logger, set_logger_level -- GitLab From 291c89648c5204e65e1567e99c522ec848a0fc63 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Tue, 24 Oct 2023 14:56:26 +0000 Subject: [PATCH 356/399] Remove duplicated execution in case of failure, but ensure execution in case of multiple outputs --- pyotb/__init__.py | 2 +- pyotb/core.py | 22 +++++++++------------- tests/test_core.py | 44 ++++++++++++++++++++++++++++++++++++-------- 3 files changed, 46 insertions(+), 22 deletions(-) diff --git a/pyotb/__init__.py b/pyotb/__init__.py index 40e6434..5e74f37 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """This module provides convenient python wrapping of otbApplications.""" -__version__ = "2.0.0.dev7" +__version__ = "2.0.0.dev8" from .install import install_otb from .helpers import logger, set_logger_level diff --git a/pyotb/core.py b/pyotb/core.py index aa5f52e..d05c7d5 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -626,6 +626,10 @@ class App(OTBObject): raise KeyError(f"key {key} not found in the application parameters types") return self._all_param_types[key] in param_types + def __is_multi_output(self): + """Check if app has multiple outputs to ensure execution during write().""" + return len(self.outputs) > 1 + def is_input(self, key: str) -> bool: """Returns True if the key is an input. @@ -803,18 +807,8 @@ class App(OTBObject): def flush(self): """Flush data to disk, this is when WriteOutput is actually called.""" - try: - logger.debug("%s: flushing data to disk", self.name) - self.app.WriteOutput() - except RuntimeError: - logger.debug( - "%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", - self.name, - ) - self._time_start = perf_counter() - self.app.ExecuteAndWriteOutput() - self.__sync_parameters() - self.frozen = False + logger.debug("%s: flushing data to disk", self.name) + self.app.WriteOutput() self._time_end = perf_counter() @deprecated_alias(filename_extension="ext_fname") @@ -927,11 +921,13 @@ class App(OTBObject): if key in data_types: self.propagate_dtype(key, data_types[key]) self.set_parameters({key: filepath}) - if self.frozen: + # TODO: drop multioutput special case when fixed on the OTB side. See discussion in MR !102 + if self.frozen or self.__is_multi_output(): self.execute() self.flush() if not parameters: return True + # Search and log missing files files, missing = [], [] for key, filepath in parameters.items(): diff --git a/tests/test_core.py b/tests/test_core.py index fc4712c..8b3f8fa 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -160,6 +160,31 @@ def test_write(): frozen_app_init_with_outfile["out"].filepath.unlink() +def test_write_multi_output(): + mss = pyotb.MeanShiftSmoothing( + SPOT_IMG_URL, + fout="/dev/shm/test_ext_fn_fout.tif", + foutpos="/dev/shm/test_ext_fn_foutpos.tif", + ) + + mss = pyotb.MeanShiftSmoothing(SPOT_IMG_URL) + assert mss.write( + { + "fout": "/dev/shm/test_ext_fn_fout.tif", + "foutpos": "/dev/shm/test_ext_fn_foutpos.tif", + }, + ext_fname={"nodata": 0, "gdal:co:COMPRESS": "DEFLATE"}, + ) + + dr = pyotb.DimensionalityReduction( + SPOT_IMG_URL, out="/dev/shm/1.tif", outinv="/dev/shm/2.tif" + ) + dr = pyotb.DimensionalityReduction(SPOT_IMG_URL) + assert dr.write( + {"out": "/dev/shm/1.tif", "outinv": "/dev/shm/2.tif"} + ) + + def test_write_ext_fname(): def _check(expected: str, key: str = "out", app=INPUT.app): fn = app.GetParameterString(key) @@ -195,18 +220,21 @@ def test_write_ext_fname(): _check("nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:10:10") INPUT["out"].filepath.unlink() - mss = pyotb.MeanShiftSmoothing(INPUT) - mss.write( + mmsd = pyotb.MorphologicalMultiScaleDecomposition(INPUT) + mmsd.write( { - "fout": "/dev/shm/test_ext_fn_fout.tif?&nodata=1", - "foutpos": "/dev/shm/test_ext_fn_foutpos.tif?&nodata=2", + "outconvex": "/dev/shm/outconvex.tif?&nodata=1", + "outconcave": "/dev/shm/outconcave.tif?&nodata=2", + "outleveling": "/dev/shm/outleveling.tif?&nodata=3", }, ext_fname={"nodata": 0, "gdal:co:COMPRESS": "DEFLATE"}, ) - _check("nodata=1&gdal:co:COMPRESS=DEFLATE", key="fout", app=mss.app) - _check("nodata=2&gdal:co:COMPRESS=DEFLATE", key="foutpos", app=mss.app) - mss["fout"].filepath.unlink() - mss["foutpos"].filepath.unlink() + _check("nodata=1&gdal:co:COMPRESS=DEFLATE", key="outconvex", app=mmsd.app) + _check("nodata=2&gdal:co:COMPRESS=DEFLATE", key="outconcave", app=mmsd.app) + _check("nodata=3&gdal:co:COMPRESS=DEFLATE", key="outleveling", app=mmsd.app) + mmsd["outconvex"].filepath.unlink() + mmsd["outconcave"].filepath.unlink() + mmsd["outleveling"].filepath.unlink() def test_output(): -- GitLab From dfe07485adf004d421eb5ee507ea7bd0bf5b5772 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Tue, 24 Oct 2023 18:47:58 +0200 Subject: [PATCH 357/399] DOC: update release notes --- RELEASE_NOTES.txt | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index 3086078..746a2a6 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -1,3 +1,25 @@ +--------------------------------------------------------------------- +2.00 (Oct XX, 2023) - Changes since version 1.5.4 + +- Major refactoring (see troubleshooting/migration) +- Pythonic extended filenames (can use dict, etc) +- Easy access to image metadata +- CI improvements (tests, coverage, doc, etc) +- Documentation improvement +- Code format +- Allow OTB dotted parameters in kwargs +- Easy access to pixel coordinates +- Add function to transform x,y coordinates into row, col +- Native support of vsicurl inputs +- Fixes in `summarize()` +- Fixes in `shape` +- Add typing to function defs to enhance documentation + +--------------------------------------------------------------------- +1.5.4 (Oct 01, 2022) - Changes since version 1.5.3 + +- Fix slicer wrong end of slicing + --------------------------------------------------------------------- 1.5.3 (Sep 29, 2022) - Changes since version 1.5.2 -- GitLab From 74473e8440f37c04ad061a585ddf55fc7c9c0e2c Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Nov 2023 16:57:27 +0100 Subject: [PATCH 358/399] ENH: remove useless encoding declaration --- pyotb/apps.py | 1 - pyotb/core.py | 1 - pyotb/functions.py | 1 - pyotb/helpers.py | 1 - 4 files changed, 4 deletions(-) diff --git a/pyotb/apps.py b/pyotb/apps.py index cd0c696..e5523a2 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Search for OTB (set env if necessary), subclass core.App for each available application.""" from __future__ import annotations diff --git a/pyotb/core.py b/pyotb/core.py index d05c7d5..6f8c24c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """This module is the core of pyotb.""" from __future__ import annotations diff --git a/pyotb/functions.py b/pyotb/functions.py index 96431cd..31e366f 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """This module provides a set of functions for pyotb.""" from __future__ import annotations diff --git a/pyotb/helpers.py b/pyotb/helpers.py index 3fc93b3..b6c09b3 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """This module ensure we properly initialize pyotb, or raise SystemExit in case of broken install.""" import logging import os -- GitLab From 50a51fd20e9d05b14816c6d520520a2ea58f4983 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Nov 2023 16:58:46 +0100 Subject: [PATCH 359/399] BUG: remove corner case for OTB < 7.4 --- pyotb/functions.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pyotb/functions.py b/pyotb/functions.py index 31e366f..5eff2d4 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -457,12 +457,10 @@ def define_processing_area( # It should replace any 'outside' pixel by some NoData -> add `fillvalue` argument in the function # Applying this bounding box to all inputs + bounds = (ulx, uly, lrx, lry) logger.info( "Cropping all images to extent Upper Left (%s, %s), Lower Right (%s, %s)", - ulx, - uly, - lrx, - lry, + *bounds, ) new_inputs = [] for inp in inputs: @@ -472,11 +470,11 @@ def define_processing_area( "mode": "extent", "mode.extent.unit": "phy", "mode.extent.ulx": ulx, - "mode.extent.uly": lry, # bug in OTB <= 7.3 : + "mode.extent.uly": uly, "mode.extent.lrx": lrx, - "mode.extent.lry": uly, # ULY/LRY are inverted + "mode.extent.lry": lry, } - new_input = App("ExtractROI", params) + new_input = App("ExtractROI", params, quiet=True) # TODO: OTB 7.4 fixes this bug, how to handle different versions of OTB? new_inputs.append(new_input) # Potentially update the reference inputs for later resampling -- GitLab From e670786a81614e0596ba9b722a31f68bd3606b70 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Nov 2023 16:59:25 +0100 Subject: [PATCH 360/399] ENH: update functions.py docstrings and light refac --- pyotb/functions.py | 151 ++++++++++++++++++++------------------------- 1 file changed, 66 insertions(+), 85 deletions(-) diff --git a/pyotb/functions.py b/pyotb/functions.py index 5eff2d4..ad2f24c 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -8,25 +8,28 @@ import sys import textwrap import uuid from collections import Counter +from pathlib import Path from .core import App, Input, LogicalOperation, Operation, get_nbchannels from .helpers import logger -def where( - cond: App | str, x: App | str | int | float, y: App | str | int | float -) -> Operation: +def where(cond: App | str, x: App | str | float, y: App | str | float) -> Operation: """Functionally similar to numpy.where. Where cond is True (!=0), returns x. Else returns y. + If cond is monoband whereas x or y are multiband, cond channels are expanded to match x & y ones. + Args: - cond: condition, must be a raster (filepath, App, Operation...). If cond is monoband whereas x or y are - multiband, cond channels are expanded to match x & y ones. + cond: condition, must be a raster (filepath, App, Operation...). x: value if cond is True. Can be: float, int, App, filepath, Operation... y: value if cond is False. Can be: float, int, App, filepath, Operation... Returns: an output where pixels are x if cond is True, else y + Raises: + ValueError: if x and y have different number of bands + """ # Checking the number of bands of rasters. Several cases : # - if cond is monoband, x and y can be multibands. Then cond will adapt to match x and y nb of bands @@ -61,19 +64,13 @@ def where( " of channels of condition to match the number of channels of X/Y", x_or_y_nb_channels, ) - # Get the number of bands of the result - if x_or_y_nb_channels: # if X or Y is a raster - out_nb_channels = x_or_y_nb_channels - else: # if only cond is a raster - out_nb_channels = cond_nb_channels + out_nb_channels = x_or_y_nb_channels or cond_nb_channels return Operation("?", cond, x, y, nb_bands=out_nb_channels) -def clip( - image: App | str, v_min: App | str | int | float, v_max: App | str | int | float -): +def clip(image: App | str, v_min: App | str | float, v_max: App | str | float): """Clip values of image in a range of values. Args: @@ -85,19 +82,18 @@ def clip( raster whose values are clipped in the range """ - if isinstance(image, str): + if isinstance(image, (str, Path)): image = Input(image) - res = where(image <= v_min, v_min, where(image >= v_max, v_max, image)) - return res + return where(image <= v_min, v_min, where(image >= v_max, v_max, image)) def all(*inputs): # pylint: disable=redefined-builtin """Check if value is different than 0 everywhere along the band axis. - For only one image, this function checks that all bands of the image are True (i.e. !=0) and outputs - a singleband boolean raster + For only one image, this function checks that all bands of the image are True (i.e. !=0) + and outputs a singleband boolean raster For several images, this function checks that all images are True (i.e. !=0) and outputs - a boolean raster, with as many bands as the inputs + a boolean raster, with as many bands as the inputs Args: inputs: inputs can be 1) a single image or 2) several images, either passed as separate arguments @@ -132,18 +128,18 @@ def all(*inputs): # pylint: disable=redefined-builtin res = res & inp[:, :, band] else: res = res & (inp[:, :, band] != 0) + return res + # Checking that all images are True + if isinstance(inputs[0], LogicalOperation): + res = inputs[0] else: - if isinstance(inputs[0], LogicalOperation): - res = inputs[0] + res = inputs[0] != 0 + for inp in inputs[1:]: + if isinstance(inp, LogicalOperation): + res = res & inp else: - res = inputs[0] != 0 - for inp in inputs[1:]: - if isinstance(inp, LogicalOperation): - res = res & inp - else: - res = res & (inp != 0) - + res = res & (inp != 0) return res @@ -158,6 +154,7 @@ def any(*inputs): # pylint: disable=redefined-builtin Args: inputs: inputs can be 1) a single image or 2) several images, either passed as separate arguments or inside a list + Returns: OR intersection @@ -188,19 +185,18 @@ def any(*inputs): # pylint: disable=redefined-builtin res = res | inp[:, :, band] else: res = res | (inp[:, :, band] != 0) + return res # Checking that at least one image is True + if isinstance(inputs[0], LogicalOperation): + res = inputs[0] else: - if isinstance(inputs[0], LogicalOperation): - res = inputs[0] + res = inputs[0] != 0 + for inp in inputs[1:]: + if isinstance(inp, LogicalOperation): + res = res | inp else: - res = inputs[0] != 0 - for inp in inputs[1:]: - if isinstance(inp, LogicalOperation): - res = res | inp - else: - res = res | (inp != 0) - + res = res | (inp != 0) return res @@ -224,17 +220,19 @@ def run_tf_function(func): Returns: wrapper: a function that returns a pyotb object + Raises: + SystemError: if OTBTF apps are missing + """ try: from .apps import ( # pylint: disable=import-outside-toplevel TensorflowModelServe, ) - except ImportError: - logger.error( + except ImportError as err: + raise SystemError( "Could not run Tensorflow function: failed to import TensorflowModelServe." "Check that you have OTBTF configured (https://github.com/remicres/otbtf#how-to-install)" - ) - raise + ) from err def get_tf_pycmd(output_dir, channels, scalar_inputs): """Create a string containing all python instructions necessary to create and save the Keras model. @@ -301,13 +299,13 @@ def run_tf_function(func): raster_inputs = [] for inp in inputs: try: - # this is for raster input + # This is for raster input channel = get_nbchannels(inp) channels.append(channel) scalar_inputs.append(None) raster_inputs.append(inp) except TypeError: - # this is for other inputs (float, int) + # This is for other inputs (float, int) channels.append(None) scalar_inputs.append(inp) @@ -316,6 +314,7 @@ def run_tf_function(func): out_savedmodel = os.path.join(tmp_dir, f"tmp_otbtf_model_{uuid.uuid4()}") pycmd = get_tf_pycmd(out_savedmodel, channels, scalar_inputs) cmd_args = [sys.executable, "-c", pycmd] + # TODO: remove subprocess execution since this issues has been fixed with OTBTF 4.0 try: subprocess.run( cmd_args, @@ -407,41 +406,25 @@ def define_processing_area( # TODO: there seems to have a bug, ImageMetaData is not updated when running an app, # cf https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2234. Should we use ImageOrigin instead? if not all( - metadata["UpperLeftCorner"] == any_metadata["UpperLeftCorner"] - and metadata["LowerRightCorner"] == any_metadata["LowerRightCorner"] - for metadata in metadatas.values() + md["UpperLeftCorner"] == any_metadata["UpperLeftCorner"] + and md["LowerRightCorner"] == any_metadata["LowerRightCorner"] + for md in metadatas.values() ): # Retrieving the bounding box that will be common for all inputs if window_rule == "intersection": # The coordinates depend on the orientation of the axis of projection if any_metadata["GeoTransform"][1] >= 0: - ulx = max( - metadata["UpperLeftCorner"][0] for metadata in metadatas.values() - ) - lrx = min( - metadata["LowerRightCorner"][0] for metadata in metadatas.values() - ) + ulx = max(md["UpperLeftCorner"][0] for md in metadatas.values()) + lrx = min(md["LowerRightCorner"][0] for md in metadatas.values()) else: - ulx = min( - metadata["UpperLeftCorner"][0] for metadata in metadatas.values() - ) - lrx = max( - metadata["LowerRightCorner"][0] for metadata in metadatas.values() - ) + ulx = min(md["UpperLeftCorner"][0] for md in metadatas.values()) + lrx = max(md["LowerRightCorner"][0] for md in metadatas.values()) if any_metadata["GeoTransform"][-1] >= 0: - lry = min( - metadata["LowerRightCorner"][1] for metadata in metadatas.values() - ) - uly = max( - metadata["UpperLeftCorner"][1] for metadata in metadatas.values() - ) + lry = min(md["LowerRightCorner"][1] for md in metadatas.values()) + uly = max(md["UpperLeftCorner"][1] for md in metadatas.values()) else: - lry = max( - metadata["LowerRightCorner"][1] for metadata in metadatas.values() - ) - uly = min( - metadata["UpperLeftCorner"][1] for metadata in metadatas.values() - ) + lry = max(md["LowerRightCorner"][1] for md in metadatas.values()) + uly = min(md["UpperLeftCorner"][1] for md in metadatas.values()) elif window_rule == "same_as_input": ulx = metadatas[reference_window_input]["UpperLeftCorner"][0] @@ -449,12 +432,12 @@ def define_processing_area( lry = metadatas[reference_window_input]["LowerRightCorner"][1] uly = metadatas[reference_window_input]["UpperLeftCorner"][1] elif window_rule == "specify": - pass - # TODO : it is when the user explicitly specifies the bounding box -> add some arguments in the function + # TODO : when the user explicitly specifies the bounding box -> add some arguments in the function + ... elif window_rule == "union": - pass - # TODO : it is when the user wants the final bounding box to be the union of all bounding box + # TODO : when the user wants the final bounding box to be the union of all bounding box # It should replace any 'outside' pixel by some NoData -> add `fillvalue` argument in the function + ... # Applying this bounding box to all inputs bounds = (ulx, uly, lrx, lry) @@ -478,16 +461,14 @@ def define_processing_area( # TODO: OTB 7.4 fixes this bug, how to handle different versions of OTB? new_inputs.append(new_input) # Potentially update the reference inputs for later resampling - if str(inp) == str( - reference_pixel_size_input - ): # we use comparison of string because calling '==' + if str(inp) == str(reference_pixel_size_input): + # We use comparison of string because calling '==' # on pyotb objects implicitly calls BandMathX application, which is not desirable reference_pixel_size_input = new_input - except RuntimeError as e: - logger.error( - "Cannot define the processing area for input %s: %s", inp, e - ) - raise + except RuntimeError as err: + raise ValueError( + f"Cannot define the processing area for input {inp}" + ) from err inputs = new_inputs # Update metadatas metadatas = {input: input.app.GetImageMetaData("out") for input in inputs} @@ -496,9 +477,9 @@ def define_processing_area( any_metadata = next(iter(metadatas.values())) # Handling different pixel sizes if not all( - metadata["GeoTransform"][1] == any_metadata["GeoTransform"][1] - and metadata["GeoTransform"][5] == any_metadata["GeoTransform"][5] - for metadata in metadatas.values() + md["GeoTransform"][1] == any_metadata["GeoTransform"][1] + and md["GeoTransform"][5] == any_metadata["GeoTransform"][5] + for md in metadatas.values() ): # Retrieving the pixel size that will be common for all inputs if pixel_size_rule == "minimal": -- GitLab From e277304176fc09824667ceec2620651bbde51f0d Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Nov 2023 16:59:54 +0100 Subject: [PATCH 361/399] ENH: rename pyOTB logger to just pyotb --- pyotb/helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyotb/helpers.py b/pyotb/helpers.py index b6c09b3..e0feb63 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -15,13 +15,13 @@ DOCS_URL = "https://www.orfeo-toolbox.org/CookBook/Installation.html" # Logging # User can also get logger with `logging.getLogger("pyOTB")` # then use pyotb.set_logger_level() to adjust logger verbosity -logger = logging.getLogger("pyOTB") +logger = logging.getLogger("pyotb") logger_handler = logging.StreamHandler(sys.stdout) formatter = logging.Formatter( - fmt="%(asctime)s (%(levelname)-4s) [pyOTB] %(message)s", datefmt="%Y-%m-%d %H:%M:%S" + fmt="%(asctime)s (%(levelname)-4s) [pyotb] %(message)s", datefmt="%Y-%m-%d %H:%M:%S" ) logger_handler.setFormatter(formatter) -# Search for PYOTB_LOGGER_LEVEL, else use OTB_LOGGER_LEVEL as pyOTB level, or fallback to INFO +# Search for PYOTB_LOGGER_LEVEL, else use OTB_LOGGER_LEVEL as pyotb level, or fallback to INFO LOG_LEVEL = ( os.environ.get("PYOTB_LOGGER_LEVEL") or os.environ.get("OTB_LOGGER_LEVEL") or "INFO" ) -- GitLab From a21aa4dfbcb7212184c3226652c36d5dc1e2abb7 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Nov 2023 17:01:34 +0100 Subject: [PATCH 362/399] ENH: refactor duplicated function in Operation --- pyotb/core.py | 69 ++++++++++++++++++++++----------------------------- 1 file changed, 30 insertions(+), 39 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 6f8c24c..d56e9e9 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1266,6 +1266,32 @@ class Operation(App): appname, il=self.unique_inputs, exp=self.exp, quiet=True, name=name ) + def get_nb_bands(self, inputs: list[OTBObject | str | int | float]) -> int: + """Guess the number of bands of the output image, from the inputs. + + Args: + inputs: the Operation operands + + Raises: + ValueError: if all inputs don't have the same number of bands + + """ + if any( + isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced") + for inp in inputs + ): + return 1 + # Check that all inputs have the same band count + nb_bands_list = [ + get_nbchannels(inp) + for inp in inputs + if not isinstance(inp, (float, int)) + ] + all_same = all(x == nb_bands_list[0] for x in nb_bands_list) + if len(nb_bands_list) > 1 and not all_same: + raise ValueError("All images do not have the same number of bands") + return nb_bands_list[0] + def build_fake_expressions( self, operator: str, @@ -1285,29 +1311,12 @@ class Operation(App): self.inputs.clear() self.nb_channels.clear() logger.debug("%s, %s", operator, inputs) - # This is when we use the ternary operator with `pyotb.where` function. The output nb of bands is already known + # When we use the ternary operator with `pyotb.where` function, the output nb of bands is already known if operator == "?" and nb_bands: pass # For any other operations, the output number of bands is the same as inputs else: - if any( - isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced") - for inp in inputs - ): - nb_bands = 1 - else: - nb_bands_list = [ - get_nbchannels(inp) - for inp in inputs - if not isinstance(inp, (float, int)) - ] - # check that all inputs have the same nb of bands - if len(nb_bands_list) > 1 and not all( - x == nb_bands_list[0] for x in nb_bands_list - ): - raise ValueError("All images do not have the same number of bands") - nb_bands = nb_bands_list[0] - + nb_bands = self.get_nb_bands(inputs) # Create a list of fake expressions, each item of the list corresponding to one band self.fake_exp_bands.clear() for i, band in enumerate(range(1, nb_bands + 1)): @@ -1463,29 +1472,11 @@ class LogicalOperation(Operation): Args: operator: str (one of >, <, >=, <=, ==, !=, &, |) inputs: Can be OTBObject, filepath, int or float - nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where + nb_bands: optionnaly specify the output nb of bands - used only internally by pyotb.where """ - # For any other operations, the output number of bands is the same as inputs - if any( - isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced") - for inp in inputs - ): - nb_bands = 1 - else: - nb_bands_list = [ - get_nbchannels(inp) - for inp in inputs - if not isinstance(inp, (float, int)) - ] - # check that all inputs have the same nb of bands - if len(nb_bands_list) > 1 and not all( - x == nb_bands_list[0] for x in nb_bands_list - ): - raise ValueError("All images do not have the same number of bands") - nb_bands = nb_bands_list[0] # Create a list of fake exp, each item of the list corresponding to one band - for i, band in enumerate(range(1, nb_bands + 1)): + for i, band in enumerate(range(1, self.get_nb_bands(inputs) + 1)): expressions = [] for inp in inputs: fake_exp, corresp_inputs, nb_channels = super().make_fake_exp( -- GitLab From 04d3c20d8441c637d6cbe5801a139b5f52670926 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Nov 2023 17:02:55 +0100 Subject: [PATCH 363/399] ENH: add more exceptions to prevent errors during __set_param --- pyotb/core.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index d56e9e9..2a385ca 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1043,9 +1043,18 @@ class App(OTBObject): else: # here `input` should be an image filepath # Append `input` to the list, do not overwrite any previously set element of the image list self.app.AddParameterStringList(key, inp) + else: + raise TypeError( + f"{self.name}: wrong input parameter type ({type(inp)})" + f" found in '{key}' list: {inp}" + ) # List of any other types (str, int...) - else: + elif self.is_key_list(key): self.app.SetParameterValue(key, obj) + else: + raise TypeError( + f"{self.name}: wrong input parameter type ({type(obj)}) for '{key}'" + ) def __sync_parameters(self): """Save app parameters in _auto_parameters or data dict. -- GitLab From 9bd6c3e5f19d55f10a78ebb2d34146bcfbbad4f3 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Nov 2023 17:11:05 +0100 Subject: [PATCH 364/399] DOC: update or enh comments and docstrings, add "Raises", fix type hints --- pyotb/apps.py | 18 +-- pyotb/core.py | 301 ++++++++++++++++++++++-------------------- pyotb/depreciation.py | 10 +- pyotb/functions.py | 17 +-- pyotb/helpers.py | 22 ++- pyotb/install.py | 9 +- 6 files changed, 202 insertions(+), 175 deletions(-) diff --git a/pyotb/apps.py b/pyotb/apps.py index e5523a2..45e49c4 100644 --- a/pyotb/apps.py +++ b/pyotb/apps.py @@ -12,12 +12,12 @@ from .helpers import logger def get_available_applications() -> tuple[str]: """Find available OTB applications. - Args: - as_subprocess: indicate if function should list available applications using subprocess call - Returns: tuple of available applications + Raises: + SystemExit: if no application is found + """ app_list = otb.Registry.GetAvailableApplications() if app_list: @@ -29,7 +29,7 @@ def get_available_applications() -> tuple[str]: class OTBTFApp(App): - """Helper for OTBTF.""" + """Helper for OTBTF to ensure the nb_sources variable is set.""" @staticmethod def set_nb_sources(*args, n_sources: int = None): @@ -59,10 +59,8 @@ class OTBTFApp(App): Args: name: name of the OTBTF app - *args: arguments (dict). NB: we don't need kwargs because it cannot contain source#.il n_sources: number of sources. Default is None (resolves the number of sources based on the content of the dict passed in args, where some 'source' str is found) - **kwargs: kwargs """ self.set_nb_sources(*args, n_sources=n_sources) @@ -71,16 +69,12 @@ class OTBTFApp(App): AVAILABLE_APPLICATIONS = get_available_applications() -# This is to enable aliases of Apps, i.e. using apps like `pyotb.AppName(...)` instead of `pyotb.App("AppName", ...)` -_CODE_TEMPLATE = ( - """ +# This is to enable aliases of Apps, i.e. `pyotb.AppName(...)` instead of `pyotb.App("AppName", ...)` +_CODE_TEMPLATE = """ class {name}(App): - """ - """ def __init__(self, *args, **kwargs): super().__init__('{name}', *args, **kwargs) """ -) for _app in AVAILABLE_APPLICATIONS: # Customize the behavior for some OTBTF applications. `OTB_TF_NSOURCES` is now handled by pyotb diff --git a/pyotb/core.py b/pyotb/core.py index 2a385ca..0f27547 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -21,12 +21,12 @@ class OTBObject(ABC): @property @abstractmethod def name(self) -> str: - """By default, should return the application name, but a custom name may be passed during init.""" + """Application name by default, but a custom name may be passed during init.""" @property @abstractmethod def app(self) -> otb.Application: - """Reference to the main (or last in pipeline) otb.Application instance linked to this object.""" + """Reference to the otb.Application instance linked to this object.""" @property @abstractmethod @@ -41,11 +41,11 @@ class OTBObject(ABC): @property @abstractmethod def exports_dic(self) -> dict[str, dict]: - """Return an internal dict object containing np.array exports, to avoid duplicated ExportImage() calls.""" + """Ref to an internal dict of np.array exports, to avoid duplicated ExportImage().""" @property def metadata(self) -> dict[str, (str, float, list[float])]: - """Return metadata. + """Return image metadata as dictionary. The returned dict results from the concatenation of the first output image metadata dictionary and the metadata dictionary. @@ -61,10 +61,7 @@ class OTBObject(ABC): if getattr(otb_imd, "has")(key) } - # Metadata dictionary - # Replace items like {"metadata_1": "TIFFTAG_SOFTWARE=CSinG - 13 - # SEPTEMBRE 2012"} with {"TIFFTAG_SOFTWARE": "CSinG - 13 SEPTEMBRE - # 2012"} + # Other metadata dictionary: key-value pairs parsing is required mdd = dict(self.app.GetMetadataDictionary(self.output_image_key)) new_mdd = {} for key, val in mdd.items(): @@ -104,7 +101,9 @@ class OTBObject(ABC): @property def transform(self) -> tuple[int]: - """Get image affine transform, rasterio style (see https://www.perrygeo.com/python-affine-transforms.html). + """Get image affine transform, rasterio style. + + See https://www.perrygeo.com/python-affine-transforms.html Returns: transform: (X spacing, X offset, X origin, Y offset, Y spacing, Y origin) @@ -116,7 +115,7 @@ class OTBObject(ABC): return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y def summarize(self, *args, **kwargs): - """Recursively summarize parameters and parents. + """Recursively summarize an app parameters and its parents. Args: *args: args for `pyotb.summarize()` @@ -138,7 +137,7 @@ class OTBObject(ABC): def get_values_at_coords( self, row: int, col: int, bands: int = None - ) -> list[int | float] | int | float: + ) -> list[float] | float: """Get pixel value(s) at a given YX coordinates. Args: @@ -149,6 +148,9 @@ class OTBObject(ABC): Returns: single numerical value or a list of values for each band + Raises: + TypeError: if bands is not a slice or list + """ channels = [] app = App("PixelValue", self, coordx=col, coordy=row, frozen=True, quiet=True) @@ -170,8 +172,19 @@ class OTBObject(ABC): data = literal_eval(app.app.GetParameterString("value")) return data[0] if len(channels) == 1 else data - def channels_list_from_slice(self, bands: int) -> list[int]: - """Get list of channels to read values at, from a slice.""" + def channels_list_from_slice(self, bands: slice) -> list[int]: + """Get list of channels to read values at, from a slice. + + Args: + bands: slice obtained when using app[:] + + Returns: + list of channels to select + + Raises: + ValueError: if the slice is malformed + + """ nb_channels = self.shape[2] start, stop, step = bands.start, bands.stop, bands.step start = nb_channels + start if isinstance(start, int) and start < 0 else start @@ -226,7 +239,7 @@ class OTBObject(ABC): required to True if preserve_dtype is False and the source app reference is lost Returns: - a numpy array + a numpy array that may already have been cached in self.exports_dic """ data = self.export(key, preserve_dtype) @@ -257,6 +270,7 @@ class OTBObject(ABC): Returns: pixel index: (row, col) + """ spacing_x, _, origin_x, _, spacing_y, origin_y = self.transform row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x @@ -272,8 +286,8 @@ class OTBObject(ABC): x: first element y: second element - Return: - operator + Returns: + an Operation object instance """ if isinstance(y, (np.ndarray, np.generic)): @@ -420,7 +434,6 @@ class OTBObject(ABC): Args: item: attribute name - """ note = ( "Since pyotb 2.0.0, OTBObject instances have stopped to forward " @@ -430,8 +443,6 @@ class OTBObject(ABC): hint = None if item in dir(self.app): - # Because otbApplication instances methods names start with an - # upper case hint = f"Maybe try `pyotb_app.app.{item}` instead of `pyotb_app.{item}`? " if item.startswith("GetParameter"): hint += ( @@ -439,10 +450,8 @@ class OTBObject(ABC): "shorten with `pyotb_app['paramname']` to access parameters " "values." ) - elif item in self.parameters_keys: - # Because in pyotb 1.5.4, applications outputs were added as - # attributes of the instance + # Because in pyotb 1.5.4, app outputs were added as instance attributes hint = ( "Note: `pyotb_app.paramname` is no longer supported. Starting " "from pyotb 2.0.0, `pyotb_app['paramname']` can be used to " @@ -469,6 +478,9 @@ class OTBObject(ABC): Returns: list of pixel values if vector image, or pixel value, or Slicer + Raises: + ValueError: if key is not a valid pixel index or slice + """ # Accessing pixel value(s) using Y/X coordinates if isinstance(key, tuple) and len(key) >= 2: @@ -488,12 +500,15 @@ class OTBObject(ABC): f'"{key}" cannot be interpreted as valid slicing. Slicing should be 2D or 3D.' ) if isinstance(key, tuple) and len(key) == 2: - # Adding a 3rd dimension - key = key + (slice(None, None, None),) + key = key + (slice(None, None, None),) # adding 3rd dimension return Slicer(self, *key) def __repr__(self) -> str: - """Return a string representation with object id, this is a key used to store image ref in Operation dicts.""" + """Return a string representation with object id. + + This is used as key to store image ref in Operation dicts. + + """ return f"<pyotb.{self.__class__.__name__} object, id {id(self)}>" @@ -510,24 +525,20 @@ class App(OTBObject): otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList, ] - OUTPUT_IMAGE_TYPES = [otb.ParameterType_OutputImage] OUTPUT_PARAM_TYPES = OUTPUT_IMAGE_TYPES + [ otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename, ] - - INPUT_LIST_TYPES = [ - otb.ParameterType_InputImageList, - otb.ParameterType_StringList, - otb.ParameterType_InputFilenameList, - otb.ParameterType_ListView, - otb.ParameterType_InputVectorDataList, - ] INPUT_IMAGES_LIST_TYPES = [ otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList, ] + INPUT_LIST_TYPES = [ + otb.ParameterType_StringList, + otb.ParameterType_ListView, + otb.ParameterType_InputVectorDataList, + ] + INPUT_IMAGES_LIST_TYPES def __init__( self, @@ -550,7 +561,6 @@ class App(OTBObject): frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___ quiet: whether to print logs of the OTB app name: custom name that will show up in logs, appname will be used if not provided - **kwargs: used for passing application parameters. e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif' @@ -606,7 +616,7 @@ class App(OTBObject): @property def app(self) -> otb.Application: - """Property to return an internal _app instance.""" + """Reference to this app otb.Application instance.""" return self._app @property @@ -616,41 +626,25 @@ class App(OTBObject): @property def exports_dic(self) -> dict[str, dict]: - """Returns internal _exports_dic object that contains numpy array exports.""" + """Reference to an internal dict object that contains numpy array exports.""" return self._exports_dic def __is_one_of_types(self, key: str, param_types: list[int]) -> bool: - """Helper to factor is_input and is_output.""" + """Helper to check the type of a parameter.""" if key not in self._all_param_types: raise KeyError(f"key {key} not found in the application parameters types") return self._all_param_types[key] in param_types def __is_multi_output(self): - """Check if app has multiple outputs to ensure execution during write().""" + """Check if app has multiple outputs to ensure re-execution during write().""" return len(self.outputs) > 1 def is_input(self, key: str) -> bool: - """Returns True if the key is an input. - - Args: - key: parameter key - - Returns: - True if the parameter is an input, else False - - """ + """Returns True if the parameter key is an input.""" return self.__is_one_of_types(key=key, param_types=self.INPUT_PARAM_TYPES) def is_output(self, key: str) -> bool: - """Returns True if the key is an output. - - Args: - key: parameter key - - Returns: - True if the parameter is an output, else False - - """ + """Returns True if the parameter key is an output.""" return self.__is_one_of_types(key=key, param_types=self.OUTPUT_PARAM_TYPES) def is_key_list(self, key: str) -> bool: @@ -670,7 +664,7 @@ class App(OTBObject): if value == param_type: return key raise TypeError( - f"{self.name}: could not find any parameter key matching the provided types" + f"{self.name}: could not find any key matching the provided types" ) @property @@ -705,14 +699,14 @@ class App(OTBObject): instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths Args: - *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved - (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") + *args: Can be : - dictionary containing key-arguments enumeration. Useful when a keyword is reserved (e.g. "in") - string or OTBObject, useful when the user implicitly wants to set the param "in" - list, useful when the user implicitly wants to set the param "il" **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif' Raises: - Exception: when the setting of a parameter failed + KeyError: when the parameter name wasn't recognized + RuntimeError: failed to set parameter valie """ parameters = kwargs @@ -723,7 +717,8 @@ class App(OTBObject): key = key.replace("_", ".") if key not in self.parameters_keys: raise KeyError( - f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}" + f"{self.name}: parameter '{key}' was not recognized." + f" Available keys are {self.parameters_keys}" ) # When the parameter expects a list, if needed, change the value to list if self.is_key_list(key) and not isinstance(obj, (list, tuple)): @@ -741,7 +736,8 @@ class App(OTBObject): self.__set_param(key, obj) except (RuntimeError, TypeError, ValueError, KeyError) as e: raise RuntimeError( - f"{self.name}: error before execution, while setting parameter '{key}' to '{obj}': {e})" + f"{self.name}: error before execution," + f" while setting '{key}' to '{obj}': {e})" ) from e # Save / update setting value and update the Output object initialized in __init__ without a filepath self._settings[key] = obj @@ -873,7 +869,6 @@ class App(OTBObject): if isinstance(ext_fname, str): ext_fname = _str2dict(ext_fname) - logger.debug("%s: extended filename for all outputs:", self.name) for key, ext in ext_fname.items(): logger.debug("%s: %s", key, ext) @@ -881,14 +876,12 @@ class App(OTBObject): for key, filepath in kwargs.items(): if self._out_param_types[key] == otb.ParameterType_OutputImage: new_ext_fname = ext_fname.copy() - - # grab already set extended filename key/values + # Grab already set extended filename key/values if "?&" in filepath: filepath, already_set_ext = filepath.split("?&", 1) - # extensions in filepath prevail over `new_ext_fname` + # Extensions in filepath prevail over `new_ext_fname` new_ext_fname.update(_str2dict(already_set_ext)) - - # transform dict to str + # tyransform dict to str ext_fname_str = "&".join( [f"{key}={value}" for key, value in new_ext_fname.items()] ) @@ -911,7 +904,7 @@ class App(OTBObject): key: parse_pixel_type(dtype) for key, dtype in pixel_type.items() } elif preserve_dtype: - self.propagate_dtype() # all outputs will have the same type as the main input raster + self.propagate_dtype() # Set parameters and flush to disk for key, filepath in parameters.items(): @@ -942,12 +935,11 @@ class App(OTBObject): ) return bool(files) and not missing - # Private functions def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]: """Gather all input arguments in kwargs dict. Args: - args: the list of arguments passed to set_parameters() + args: the list of arguments passed to set_parameters (__init__ *args) Returns: a dictionary with the right keyword depending on the object @@ -966,12 +958,11 @@ class App(OTBObject): return kwargs def __check_input_param( - self, obj: list | OTBObject | str | Path + self, obj: list | tuple | OTBObject | str | Path ) -> list | OTBObject | str: """Check the type and value of an input parameter, add vsi prefixes if needed.""" - if isinstance(obj, list): + if isinstance(obj, (list, tuple)): return [self.__check_input_param(o) for o in obj] - # May be we could add some checks here if isinstance(obj, OTBObject): return obj if isinstance(obj, Path): @@ -981,7 +972,6 @@ class App(OTBObject): # Remote file. TODO: add support for S3 / GS / AZ if obj.startswith(("https://", "http://", "ftp://")): obj = "/vsicurl/" + obj - # Compressed file prefixes = { ".tar": "vsitar", ".tar.gz": "vsitar", @@ -1000,9 +990,9 @@ class App(OTBObject): return obj raise TypeError(f"{self.name}: wrong input parameter type ({type(obj)})") - def __check_output_param(self, obj: list | str | Path) -> list | str: + def __check_output_param(self, obj: list | tuple | str | Path) -> list | str: """Check the type and value of an output parameter.""" - if isinstance(obj, list): + if isinstance(obj, (list, tuple)): return [self.__check_output_param(o) for o in obj] if isinstance(obj, Path): obj = str(obj) @@ -1020,27 +1010,22 @@ class App(OTBObject): # Single-parameter cases if isinstance(obj, OTBObject): self.app.ConnectImage(key, obj.app, obj.output_image_key) - elif isinstance( - obj, otb.Application - ): # this is for backward comp with plain OTB + elif isinstance(obj, otb.Application): self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0]) - elif ( - key == "ram" - ): # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200 + elif key == "ram": + # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf OTB issue 2200 self.app.SetParameterInt("ram", int(obj)) - elif not isinstance(obj, list): # any other parameters (str, int...) + # Any other parameters (str, int...) + elif not isinstance(obj, (list, tuple)): self.app.SetParameterValue(key, obj) # Images list elif self.is_key_images_list(key): - # To enable possible in-memory connections, we go through the list and set the parameters one by one for inp in obj: if isinstance(inp, OTBObject): self.app.ConnectImage(key, inp.app, inp.output_image_key) - elif isinstance( - inp, otb.Application - ): # this is for backward comp with plain OTB + elif isinstance(inp, otb.Application): self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0]) - else: # here `input` should be an image filepath + elif isinstance(inp, (str, Path)): # Append `input` to the list, do not overwrite any previously set element of the image list self.app.AddParameterStringList(key, inp) else: @@ -1101,7 +1086,9 @@ class App(OTBObject): self.data[key] = value # Special functions - def __getitem__(self, key: str) -> Any | list[int | float] | int | float | Slicer: + def __getitem__( + self, key: str | tuple + ) -> Any | list[int | float] | int | float | Slicer: """This function is called when we use App()[...]. We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer @@ -1141,7 +1128,10 @@ class Slicer(App): obj: input rows: slice along Y / Latitude axis cols: slice along X / Longitude axis - channels: channels, can be slicing, list or int + channels: bands to extract. can be slicing, list or int + + Raises: + TypeError: if channels param isn't slice, list or int """ super().__init__( @@ -1157,20 +1147,18 @@ class Slicer(App): # Channel slicing if channels != slice(None, None, None): - # Trigger source app execution if needed nb_channels = get_nbchannels(obj) self.app.Execute() # this is needed by ExtractROI for setting the `cl` parameter - # if needed, converting int to list if isinstance(channels, int): channels = [channels] - # if needed, converting slice to list elif isinstance(channels, slice): channels = self.channels_list_from_slice(channels) elif isinstance(channels, tuple): channels = list(channels) elif not isinstance(channels, list): - raise ValueError( - f"Invalid type for channels, should be int, slice or list of bands. : {channels}" + raise TypeError( + f"Invalid type for channels ({type(channels)})." + f" Should be int, slice or list of bands." ) # Change the potential negative index values to reverse index channels = [c if c >= 0 else nb_channels + c for c in channels] @@ -1178,28 +1166,24 @@ class Slicer(App): # Spatial slicing spatial_slicing = False - # TODO: handle the step value in the slice so that NN undersampling is possible ? e.g. raster[::2, ::2] if rows.start is not None: parameters.update({"mode.extent.uly": rows.start}) spatial_slicing = True if rows.stop is not None and rows.stop != -1: - parameters.update( - {"mode.extent.lry": rows.stop - 1} - ) # subtract 1 to respect python convention + # Subtract 1 to respect python convention + parameters.update({"mode.extent.lry": rows.stop - 1}) spatial_slicing = True if cols.start is not None: parameters.update({"mode.extent.ulx": cols.start}) spatial_slicing = True if cols.stop is not None and cols.stop != -1: - parameters.update( - {"mode.extent.lrx": cols.stop - 1} - ) # subtract 1 to respect python convention + # Subtract 1 to respect python convention + parameters.update({"mode.extent.lrx": cols.stop - 1}) spatial_slicing = True - # These are some attributes when the user simply wants to extract *one* band to be used in an Operation + # When the user simply wants to extract *one* band to be used in an Operation if not spatial_slicing and isinstance(channels, list) and len(channels) == 1: - self.one_band_sliced = ( - channels[0] + 1 - ) # OTB convention: channels start at 1 + # OTB convention: channels start at 1 + self.one_band_sliced = channels[0] + 1 self.input = obj # Execute app @@ -1230,33 +1214,32 @@ class Operation(App): """ def __init__(self, operator: str, *inputs, nb_bands: int = None, name: str = None): - """Given some inputs and an operator, this function enables to transform this into an OTB application. + """Given some inputs and an operator, this object enables to python operator to a BandMath operation. Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator. It can have 3 inputs for the ternary operator `cond ? x : y`. Args: operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? - *inputs: inputs. Can be OTBObject, filepath, int or float - nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where - name: override the Operation name + *inputs: operands, can be OTBObject, filepath, int or float + nb_bands: optionally specify the output nb of bands - used only internally by pyotb.where + name: override the default Operation name """ self.operator = operator - # We first create a 'fake' expression. E.g for the operation `input1 + input2` , we create a fake expression - # that is like "str(input1) + str(input2)" + # We first create a 'fake' expression. E.g for the operation `input1 + input2` + # we create a fake expression like "str(input1) + str(input2)" self.inputs = [] self.nb_channels = {} self.fake_exp_bands = [] self.build_fake_expressions(operator, inputs, nb_bands=nb_bands) # Transforming images to the adequate im#, e.g. `input1` to "im1" - # creating a dictionary that is like {str(input1): 'im1', 'image2.tif': 'im2', ...}. + # using a dictionary : {str(input1): 'im1', 'image2.tif': 'im2', ...}. # NB: the keys of the dictionary are strings-only, instead of 'complex' objects, to enable easy serialization self.im_dic = {} self.im_count = 1 - map_repr_to_input = ( - {} - ) # to be able to retrieve the real python object from its string representation + # To be able to retrieve the real python object from its string representation + map_repr_to_input = {} for inp in self.inputs: if not isinstance(inp, (int, float)): if str(inp) not in self.im_dic: @@ -1316,6 +1299,9 @@ class Operation(App): inputs: inputs. Can be OTBObject, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where + Raises: + ValueError: if all inputs don't have the same number of bands + """ self.inputs.clear() self.nb_channels.clear() @@ -1336,30 +1322,30 @@ class Operation(App): if len(inputs) == 3 and k == 0: # When cond is monoband whereas the result is multiband, we expand the cond to multiband cond_band = 1 if nb_bands != inp.shape[2] else band - fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp( + fake_exp, corresp_inputs, nb_channels = self.make_fake_exp( inp, cond_band, keep_logical=True ) else: # Any other input - fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp( + fake_exp, corresp_inputs, nb_channels = self.make_fake_exp( inp, band, keep_logical=False ) expressions.append(fake_exp) # Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates) - if i == 0 and corresponding_inputs and nb_channels: - self.inputs.extend(corresponding_inputs) + if i == 0 and corresp_inputs and nb_channels: + self.inputs.extend(corresp_inputs) self.nb_channels.update(nb_channels) # Generating the fake expression of the whole operation - if len(inputs) == 1: # this is only for 'abs' + if len(inputs) == 1: + # This is only for 'abs()' fake_exp = f"({operator}({expressions[0]}))" elif len(inputs) == 2: # We create here the "fake" expression. For example, for a BandMathX expression such as '2 * im1 + im2', # the false expression stores the expression 2 * str(input1) + str(input2) fake_exp = f"({expressions[0]} {operator} {expressions[1]})" - elif ( - len(inputs) == 3 and operator == "?" - ): # this is only for ternary expression + elif len(inputs) == 3 and operator == "?": + # This is only for ternary expression fake_exp = f"({expressions[0]} ? {expressions[1]} : {expressions[2]})" self.fake_exp_bands.append(fake_exp) @@ -1445,10 +1431,12 @@ class Operation(App): class LogicalOperation(Operation): - """A specialization of Operation class for boolean logical operations i.e. >, <, >=, <=, ==, !=, `&` and `|`. + """A specialization of Operation class for boolean logical operations. - The only difference is that not only the BandMath expression is saved (e.g. "im1b1 > 0 ? 1 : 0"), but also the - logical expression (e.g. "im1b1 > 0") + Supported operators are >, <, >=, <=, ==, !=, `&` and `|`. + + The only difference is that not only the BandMath expression is saved (e.g. "im1b1 > 0 ? 1 : 0"), + but also the logical expression (e.g. "im1b1 > 0") """ @@ -1458,7 +1446,7 @@ class LogicalOperation(Operation): Args: operator: string operator (one of >, <, >=, <=, ==, !=, &, |) *inputs: inputs - nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where + nb_bands: optionally specify the output nb of bands - used only by pyotb.where """ self.logical_fake_exp_bands = [] @@ -1481,7 +1469,7 @@ class LogicalOperation(Operation): Args: operator: str (one of >, <, >=, <=, ==, !=, &, |) inputs: Can be OTBObject, filepath, int or float - nb_bands: optionnaly specify the output nb of bands - used only internally by pyotb.where + nb_bands: optionally specify the output nb of bands - used only internally by pyotb.where """ # Create a list of fake exp, each item of the list corresponding to one band @@ -1530,7 +1518,7 @@ class Input(App): class Output(OTBObject): - """Object that behave like a pointer to a specific application output file.""" + """Object that behave like a pointer to a specific application in-memory output or file.""" _filepath: str | Path = None @@ -1547,7 +1535,7 @@ class Output(OTBObject): Args: pyotb_app: The pyotb App to store reference from param_key: Output parameter key of the target app - filepath: path of the output file (if not in memory) + filepath: path of the output file (if not memory) mkdir: create missing parent directories """ @@ -1574,7 +1562,7 @@ class Output(OTBObject): @property def exports_dic(self) -> dict[str, dict]: - """Returns internal _exports_dic object that contains numpy array exports.""" + """Reference to parent _exports_dic object that contains np array exports.""" return self.parent_pyotb_app.exports_dic @property @@ -1597,19 +1585,34 @@ class Output(OTBObject): self._filepath = path def exists(self) -> bool: - """Check file exist.""" + """Check if the output file exist on disk. + + Raises: + ValueError: if filepath is not set or is remote URL + + """ if not isinstance(self.filepath, Path): raise ValueError("Filepath is not set or points to a remote URL") return self.filepath.exists() def make_parent_dirs(self): - """Create missing parent directories.""" + """Create missing parent directories. + + Raises: + ValueError: if filepath is not set or is remote URL + + """ if not isinstance(self.filepath, Path): raise ValueError("Filepath is not set or points to a remote URL") self.filepath.parent.mkdir(parents=True, exist_ok=True) def write(self, filepath: None | str | Path = None, **kwargs) -> bool: - """Write output to disk, filepath is not required if it was provided to parent App during init.""" + """Write output to disk, filepath is not required if it was provided to parent App during init. + + Args: + filepath: path of the output file, can be None if a value was passed during app init + + """ if filepath is None: return self.parent_pyotb_app.write( {self.output_image_key: self.filepath}, **kwargs @@ -1630,6 +1633,9 @@ def get_nbchannels(inp: str | Path | OTBObject) -> int: Returns: number of bands in image + Raises: + TypeError: if inp band count cannot be retrieved + """ if isinstance(inp, OTBObject): return inp.shape[-1] @@ -1657,6 +1663,9 @@ def get_pixel_type(inp: str | Path | OTBObject) -> str: pixel_type: OTB enum e.g. `otbApplication.ImagePixelType_uint8', which actually is an int. For an OTBObject with several outputs, only the pixel type of the first output is returned + Raises: + TypeError: if inp pixel type cannot be retrieved + """ if isinstance(inp, OTBObject): return inp.app.GetParameterOutputImagePixelType(inp.output_image_key) @@ -1679,10 +1688,14 @@ def parse_pixel_type(pixel_type: str | int) -> int: """Convert one str pixel type to OTB integer enum if necessary. Args: - pixel_type: pixel type. can be str, int or dict + pixel_type: pixel type. can be int or str (either OTB or numpy convention) Returns: - pixel_type integer value + pixel_type OTB enum integer value + + Raises: + KeyError: if pixel_type name is unknown + TypeError: if type(pixel_type) isn't int or str """ if isinstance(pixel_type, int): # normal OTB int enum @@ -1704,19 +1717,19 @@ def parse_pixel_type(pixel_type: str | int) -> int: if pixel_type in datatype_to_pixeltype: return getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[pixel_type]}") raise KeyError( - f"Unknown data type `{pixel_type}`. Available ones: {datatype_to_pixeltype}" + f"Unknown dtype `{pixel_type}`. Available ones: {datatype_to_pixeltype}" ) raise TypeError( f"Bad pixel type specification ({pixel_type} of type {type(pixel_type)})" ) -def get_out_images_param_keys(app: OTBObject) -> list[str]: - """Return every output parameter keys of an OTB app.""" +def get_out_images_param_keys(otb_app: otb.Application) -> list[str]: + """Return every output parameter keys of a bare OTB app.""" return [ key - for key in app.GetParametersKeys() - if app.GetParameterType(key) == otb.ParameterType_OutputImage + for key in otb_app.GetParametersKeys() + if otb_app.GetParameterType(key) == otb.ParameterType_OutputImage ] diff --git a/pyotb/depreciation.py b/pyotb/depreciation.py index 687e6ca..6794373 100644 --- a/pyotb/depreciation.py +++ b/pyotb/depreciation.py @@ -1,7 +1,6 @@ """Helps with deprecated classes and methods. -Taken from https://stackoverflow.com/questions/49802412/how-to-implement- -deprecation-in-python-with-argument-alias +Taken from https://stackoverflow.com/questions/49802412/how-to-implement-deprecation-in-python-with-argument-alias """ from typing import Callable, Dict, Any import functools @@ -17,7 +16,7 @@ def depreciation_warning(message: str): """Shows a warning message. Args: - message: message + message: message to log """ warnings.warn( @@ -63,11 +62,14 @@ def rename_kwargs(func_name: str, kwargs: Dict[str, Any], aliases: Dict[str, str kwargs: keyword args aliases: aliases + Raises: + ValueError: if both old and new arguments are provided + """ for alias, new in aliases.items(): if alias in kwargs: if new in kwargs: - raise TypeError( + raise ValueError( f"{func_name} received both {alias} and {new} as arguments!" f" {alias} is deprecated, use {new} instead." ) diff --git a/pyotb/functions.py b/pyotb/functions.py index ad2f24c..8fde2cd 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -247,7 +247,7 @@ def run_tf_function(func): """ # Getting the string definition of the tf function (e.g. "def multiply(x1, x2):...") - # TODO: maybe not entirely foolproof, maybe we should use dill instead? but it would add a dependency + # Maybe not entirely foolproof, maybe we should use dill instead? but it would add a dependency func_def_str = inspect.getsource(func) func_name = func.__name__ @@ -342,7 +342,7 @@ def run_tf_function(func): for i, inp in enumerate(raster_inputs): model_serve.set_parameters({f"source{i + 1}.il": [inp]}) model_serve.execute() - # TODO: handle the deletion of the temporary model ? + # Possible ENH: handle the deletion of the temporary model ? return model_serve @@ -403,7 +403,7 @@ def define_processing_area( ) # Handling different spatial footprints - # TODO: there seems to have a bug, ImageMetaData is not updated when running an app, + # TODO: find possible bug - ImageMetaData is not updated when running an app # cf https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2234. Should we use ImageOrigin instead? if not all( md["UpperLeftCorner"] == any_metadata["UpperLeftCorner"] @@ -432,10 +432,10 @@ def define_processing_area( lry = metadatas[reference_window_input]["LowerRightCorner"][1] uly = metadatas[reference_window_input]["UpperLeftCorner"][1] elif window_rule == "specify": - # TODO : when the user explicitly specifies the bounding box -> add some arguments in the function + # When the user explicitly specifies the bounding box -> add some arguments in the function ... elif window_rule == "union": - # TODO : when the user wants the final bounding box to be the union of all bounding box + # When the user wants the final bounding box to be the union of all bounding box # It should replace any 'outside' pixel by some NoData -> add `fillvalue` argument in the function ... @@ -458,7 +458,7 @@ def define_processing_area( "mode.extent.lry": lry, } new_input = App("ExtractROI", params, quiet=True) - # TODO: OTB 7.4 fixes this bug, how to handle different versions of OTB? + # OTB 7.4 fixes this bug, how to handle different versions of OTB? new_inputs.append(new_input) # Potentially update the reference inputs for later resampling if str(inp) == str(reference_pixel_size_input): @@ -495,8 +495,9 @@ def define_processing_area( elif pixel_size_rule == "same_as_input": reference_input = reference_pixel_size_input elif pixel_size_rule == "specify": - pass - # TODO : when the user explicitly specify the pixel size -> add argument inside the function + # When the user explicitly specify the pixel size -> add argument inside the function + ... + pixel_size = metadatas[reference_input]["GeoTransform"][1] # Perform resampling on inputs that do not comply with the target pixel size diff --git a/pyotb/helpers.py b/pyotb/helpers.py index e0feb63..0e6ea2a 100644 --- a/pyotb/helpers.py +++ b/pyotb/helpers.py @@ -60,6 +60,10 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True): Returns: otbApplication module + Raises: + SystemError: is OTB is not found (when using interactive mode) + SystemExit: if OTB is not found, since pyotb won't be usable + """ otb = None # Try OTB_ROOT env variable first (allow override default OTB version) @@ -125,6 +129,9 @@ def set_environment(prefix: str): Args: prefix: path to OTB root directory + Raises: + SystemError: if OTB or GDAL is not found + """ logger.info("Preparing environment for OTB in %s", prefix) # OTB root directory @@ -138,16 +145,18 @@ def set_environment(prefix: str): lib_dir = __find_lib(prefix) if not lib_dir: raise SystemError("Can't find OTB external libraries") - # This does not seems to work + # LD library path : this does not seems to work if sys.platform == "linux" and built_from_source: new_ld_path = f"{lib_dir}:{os.environ.get('LD_LIBRARY_PATH') or ''}" os.environ["LD_LIBRARY_PATH"] = new_ld_path + # Add python bindings directory first in PYTHONPATH otb_api = __find_python_api(lib_dir) if not otb_api: raise SystemError("Can't find OTB Python API") if otb_api not in sys.path: sys.path.insert(0, otb_api) + # Add /bin first in PATH, in order to avoid conflicts with another GDAL install os.environ["PATH"] = f"{prefix / 'bin'}{os.pathsep}{os.environ['PATH']}" # Ensure APPLICATION_PATH is set @@ -159,6 +168,7 @@ def set_environment(prefix: str): os.environ["LC_NUMERIC"] = "C" os.environ["GDAL_DRIVER_PATH"] = "disable" + # Find GDAL libs if (prefix / "share/gdal").exists(): # Local GDAL (OTB Superbuild, .run, .exe) gdal_data = str(prefix / "share/gdal") @@ -186,7 +196,7 @@ def __find_lib(prefix: str = None, otb_module=None): otb_module: try with otbApplication library path if found, else None Returns: - lib path + lib path, or None if not found """ if prefix is not None: @@ -216,7 +226,7 @@ def __find_python_api(lib_dir: Path): prefix: prefix Returns: - python API path if found, else None + OTB python API path, or None if not found """ otb_api = lib_dir / "python" @@ -235,7 +245,7 @@ def __find_apps_path(lib_dir: Path): lib_dir: library path Returns: - application path if found, else empty string + application path, or empty string if not found """ if lib_dir.exists(): @@ -251,7 +261,7 @@ def __find_otb_root(): """Search for OTB root directory in well known locations. Returns: - str path of the OTB directory + str path of the OTB directory, or None if not found """ prefix = None @@ -339,5 +349,5 @@ def __suggest_fix_import(error_message: str, prefix: str): # This part of pyotb is the first imported during __init__ and checks if OTB is found -# If OTB is not found, a SystemExit is raised, to prevent execution of the core module +# If OTB isn't found, a SystemExit is raised to prevent execution of the core module find_otb() diff --git a/pyotb/install.py b/pyotb/install.py index 47a6ee7..219967a 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -33,15 +33,17 @@ def otb_latest_release_tag(): return releases[-1] -def check_versions(sysname: str, python_minor: int, otb_major: int): +def check_versions(sysname: str, python_minor: int, otb_major: int) -> tuple[bool, int]: """Verify if python version is compatible with major OTB version. Args: sysname: OTB's system name convention (Linux64, Darwin64, Win64) python_minor: minor version of python otb_major: major version of OTB to be installed + Returns: (True, 0) or (False, expected_version) if case of version conflict + """ if sysname == "Win64": expected = 5 if otb_major in (6, 7) else 7 @@ -103,6 +105,10 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True): Returns: full path of the new installation + Raises: + SystemExit: if python version is not compatible with major OTB version + SystemError: if automatic env config failed + """ # Read env config if sys.version_info.major == 2: @@ -120,6 +126,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True): raise SystemExit( f"Python 3.{expected} is required to import bindings on Windows." ) + # Fetch archive and run installer filename = f"OTB-{version}-{sysname}.{ext}" url = f"https://www.orfeo-toolbox.org/packages/archives/OTB/{filename}" -- GitLab From 3da1b9f61c955faa7e3a9db7fa0ac1217dc0bcd8 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Nov 2023 17:14:13 +0100 Subject: [PATCH 365/399] CI: codespell typos --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 0f27547..d6336b5 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -706,7 +706,7 @@ class App(OTBObject): Raises: KeyError: when the parameter name wasn't recognized - RuntimeError: failed to set parameter valie + RuntimeError: failed to set parameter value """ parameters = kwargs -- GitLab From 0770dac0db935663b42193ed457ffb6ca97af7ab Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Nov 2023 17:16:32 +0100 Subject: [PATCH 366/399] ENH: use integer in check_versions --- pyotb/install.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index 219967a..2d5561c 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -33,7 +33,7 @@ def otb_latest_release_tag(): return releases[-1] -def check_versions(sysname: str, python_minor: int, otb_major: int) -> tuple[bool, int]: +def check_versions(sysname: str, python_minor: int, otb_major: int) -> tuple[int]: """Verify if python version is compatible with major OTB version. Args: @@ -42,22 +42,22 @@ def check_versions(sysname: str, python_minor: int, otb_major: int) -> tuple[boo otb_major: major version of OTB to be installed Returns: - (True, 0) or (False, expected_version) if case of version conflict + (1, 0) or (0, expected_version) if case of version conflict """ if sysname == "Win64": expected = 5 if otb_major in (6, 7) else 7 if python_minor == expected: - return True, 0 + return 1, 0 elif sysname == "Darwin64": expected = 7, 0 if python_minor == expected: - return True, 0 + return 1, 0 elif sysname == "Linux64": expected = 5 if otb_major in (6, 7) else 8 if python_minor == expected: - return True, 0 - return False, expected + return 1, 0 + return 0, expected def env_config_unix(otb_path: Path): -- GitLab From 853b206fba8701ff304a726fc44c8de8dd761159 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Nov 2023 17:28:47 +0100 Subject: [PATCH 367/399] FIX: regression in __set_param --- pyotb/core.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index d6336b5..80916d0 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1034,12 +1034,9 @@ class App(OTBObject): f" found in '{key}' list: {inp}" ) # List of any other types (str, int...) - elif self.is_key_list(key): - self.app.SetParameterValue(key, obj) else: - raise TypeError( - f"{self.name}: wrong input parameter type ({type(obj)}) for '{key}'" - ) + # TODO: use self.is_key_list, but this is not working for ExtractROI param "cl" which is ParameterType_UNKNOWN + self.app.SetParameterValue(key, obj) def __sync_parameters(self): """Save app parameters in _auto_parameters or data dict. -- GitLab From e1791ea4b0ceb3069ae6012a11b8c9842f70a944 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Nov 2023 18:06:47 +0100 Subject: [PATCH 368/399] BUG: add missing annotations import from __future__ --- pyotb/install.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyotb/install.py b/pyotb/install.py index 2d5561c..5a8256a 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -1,4 +1,6 @@ """This module contains functions for interactive auto installation of OTB.""" +from __future__ import annotations + import json import os import re -- GitLab From d09c718ad945fc0a0632a1248cd28fef9fe415c2 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Nov 2023 18:07:39 +0100 Subject: [PATCH 369/399] DOC: remove useless comment since bug is fixed --- pyotb/functions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyotb/functions.py b/pyotb/functions.py index 8fde2cd..fe93641 100644 --- a/pyotb/functions.py +++ b/pyotb/functions.py @@ -458,7 +458,6 @@ def define_processing_area( "mode.extent.lry": lry, } new_input = App("ExtractROI", params, quiet=True) - # OTB 7.4 fixes this bug, how to handle different versions of OTB? new_inputs.append(new_input) # Potentially update the reference inputs for later resampling if str(inp) == str(reference_pixel_size_input): -- GitLab From 5c8f8dcccc73ddbbe1042208488f299aea44c03f Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Nov 2023 18:52:24 +0100 Subject: [PATCH 370/399] ENH: back to bool since type hints bug is fixed --- pyotb/install.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyotb/install.py b/pyotb/install.py index 5a8256a..0cf139e 100644 --- a/pyotb/install.py +++ b/pyotb/install.py @@ -35,7 +35,7 @@ def otb_latest_release_tag(): return releases[-1] -def check_versions(sysname: str, python_minor: int, otb_major: int) -> tuple[int]: +def check_versions(sysname: str, python_minor: int, otb_major: int) -> tuple[bool, int]: """Verify if python version is compatible with major OTB version. Args: @@ -44,22 +44,22 @@ def check_versions(sysname: str, python_minor: int, otb_major: int) -> tuple[int otb_major: major version of OTB to be installed Returns: - (1, 0) or (0, expected_version) if case of version conflict + (True, 0) if compatible or (False, expected_version) in case of conflicts """ if sysname == "Win64": expected = 5 if otb_major in (6, 7) else 7 if python_minor == expected: - return 1, 0 + return True, 0 elif sysname == "Darwin64": expected = 7, 0 if python_minor == expected: - return 1, 0 + return True, 0 elif sysname == "Linux64": expected = 5 if otb_major in (6, 7) else 8 if python_minor == expected: - return 1, 0 - return 0, expected + return True, 0 + return False, expected def env_config_unix(otb_path: Path): -- GitLab From 014a07574aa05a0cffb7c573564b5815e77ef0a8 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 3 Nov 2023 19:35:57 +0100 Subject: [PATCH 371/399] ENH: remove useless exceptions since already checked in set_parameters --- pyotb/core.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 80916d0..df3297a 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1025,14 +1025,9 @@ class App(OTBObject): self.app.ConnectImage(key, inp.app, inp.output_image_key) elif isinstance(inp, otb.Application): self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0]) - elif isinstance(inp, (str, Path)): + else: # Append `input` to the list, do not overwrite any previously set element of the image list self.app.AddParameterStringList(key, inp) - else: - raise TypeError( - f"{self.name}: wrong input parameter type ({type(inp)})" - f" found in '{key}' list: {inp}" - ) # List of any other types (str, int...) else: # TODO: use self.is_key_list, but this is not working for ExtractROI param "cl" which is ParameterType_UNKNOWN -- GitLab From 848f4e4aab6d81ad773f81fcf15cef9d2e23fb87 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Nov 2023 14:15:31 +0100 Subject: [PATCH 372/399] ENH: add exception, use if self.is_key_list(key) in __set_param --- pyotb/core.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index df3297a..c0a824b 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -534,11 +534,12 @@ class App(OTBObject): otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList, ] - INPUT_LIST_TYPES = [ + INPUT_LIST_TYPES = INPUT_IMAGES_LIST_TYPES + [ otb.ParameterType_StringList, otb.ParameterType_ListView, otb.ParameterType_InputVectorDataList, - ] + INPUT_IMAGES_LIST_TYPES + otb.ParameterType_Band, + ] def __init__( self, @@ -1025,13 +1026,17 @@ class App(OTBObject): self.app.ConnectImage(key, inp.app, inp.output_image_key) elif isinstance(inp, otb.Application): self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0]) + # Here inp is either str or Path, already checked by __check_*_param else: - # Append `input` to the list, do not overwrite any previously set element of the image list + # Append it to the list, do not overwrite any previously set element of the image list self.app.AddParameterStringList(key, inp) # List of any other types (str, int...) - else: - # TODO: use self.is_key_list, but this is not working for ExtractROI param "cl" which is ParameterType_UNKNOWN + elif self.is_key_list(key): self.app.SetParameterValue(key, obj) + else: + raise TypeError( + f"{self.name}: wrong parameter type ({type(obj)}) for '{key}'" + ) def __sync_parameters(self): """Save app parameters in _auto_parameters or data dict. -- GitLab From 8b257b5073c5ce5ce142a73eff8e3f32cb569c14 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Nov 2023 14:40:24 +0100 Subject: [PATCH 373/399] ENH: type hints (float | int) -> (float) --- pyotb/core.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index c0a824b..f03e9e1 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -294,35 +294,35 @@ class OTBObject(ABC): return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__ return op_cls(name, x, y) - def __add__(self, other: OTBObject | str | int | float) -> Operation: + def __add__(self, other: OTBObject | str | float) -> Operation: """Addition.""" return self.__create_operator(Operation, "+", self, other) - def __sub__(self, other: OTBObject | str | int | float) -> Operation: + def __sub__(self, other: OTBObject | str | float) -> Operation: """Subtraction.""" return self.__create_operator(Operation, "-", self, other) - def __mul__(self, other: OTBObject | str | int | float) -> Operation: + def __mul__(self, other: OTBObject | str | float) -> Operation: """Multiplication.""" return self.__create_operator(Operation, "*", self, other) - def __truediv__(self, other: OTBObject | str | int | float) -> Operation: + def __truediv__(self, other: OTBObject | str | float) -> Operation: """Division.""" return self.__create_operator(Operation, "/", self, other) - def __radd__(self, other: OTBObject | str | int | float) -> Operation: + def __radd__(self, other: OTBObject | str | float) -> Operation: """Right addition.""" return self.__create_operator(Operation, "+", other, self) - def __rsub__(self, other: OTBObject | str | int | float) -> Operation: + def __rsub__(self, other: OTBObject | str | float) -> Operation: """Right subtraction.""" return self.__create_operator(Operation, "-", other, self) - def __rmul__(self, other: OTBObject | str | int | float) -> Operation: + def __rmul__(self, other: OTBObject | str | float) -> Operation: """Right multiplication.""" return self.__create_operator(Operation, "*", other, self) - def __rtruediv__(self, other: OTBObject | str | int | float) -> Operation: + def __rtruediv__(self, other: OTBObject | str | float) -> Operation: """Right division.""" return self.__create_operator(Operation, "/", other, self) @@ -330,35 +330,35 @@ class OTBObject(ABC): """Absolute value.""" return Operation("abs", self) - def __ge__(self, other: OTBObject | str | int | float) -> LogicalOperation: + def __ge__(self, other: OTBObject | str | float) -> LogicalOperation: """Greater of equal than.""" return self.__create_operator(LogicalOperation, ">=", self, other) - def __le__(self, other: OTBObject | str | int | float) -> LogicalOperation: + def __le__(self, other: OTBObject | str | float) -> LogicalOperation: """Lower of equal than.""" return self.__create_operator(LogicalOperation, "<=", self, other) - def __gt__(self, other: OTBObject | str | int | float) -> LogicalOperation: + def __gt__(self, other: OTBObject | str | float) -> LogicalOperation: """Greater than.""" return self.__create_operator(LogicalOperation, ">", self, other) - def __lt__(self, other: OTBObject | str | int | float) -> LogicalOperation: + def __lt__(self, other: OTBObject | str | float) -> LogicalOperation: """Lower than.""" return self.__create_operator(LogicalOperation, "<", self, other) - def __eq__(self, other: OTBObject | str | int | float) -> LogicalOperation: + def __eq__(self, other: OTBObject | str | float) -> LogicalOperation: """Equality.""" return self.__create_operator(LogicalOperation, "==", self, other) - def __ne__(self, other: OTBObject | str | int | float) -> LogicalOperation: + def __ne__(self, other: OTBObject | str | float) -> LogicalOperation: """Inequality.""" return self.__create_operator(LogicalOperation, "!=", self, other) - def __or__(self, other: OTBObject | str | int | float) -> LogicalOperation: + def __or__(self, other: OTBObject | str | float) -> LogicalOperation: """Logical or.""" return self.__create_operator(LogicalOperation, "||", self, other) - def __and__(self, other: OTBObject | str | int | float) -> LogicalOperation: + def __and__(self, other: OTBObject | str | float) -> LogicalOperation: """Logical and.""" return self.__create_operator(LogicalOperation, "&&", self, other) @@ -1085,7 +1085,7 @@ class App(OTBObject): # Special functions def __getitem__( self, key: str | tuple - ) -> Any | list[int | float] | int | float | Slicer: + ) -> Any | list[float] | float | Slicer: """This function is called when we use App()[...]. We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer @@ -1255,7 +1255,7 @@ class Operation(App): appname, il=self.unique_inputs, exp=self.exp, quiet=True, name=name ) - def get_nb_bands(self, inputs: list[OTBObject | str | int | float]) -> int: + def get_nb_bands(self, inputs: list[OTBObject | str | float]) -> int: """Guess the number of bands of the output image, from the inputs. Args: @@ -1284,7 +1284,7 @@ class Operation(App): def build_fake_expressions( self, operator: str, - inputs: list[OTBObject | str | int | float], + inputs: list[OTBObject | str | float], nb_bands: int = None, ): """Create a list of 'fake' expressions, one for each band. @@ -1455,7 +1455,7 @@ class LogicalOperation(Operation): def build_fake_expressions( self, operator: str, - inputs: list[OTBObject | str | int | float], + inputs: list[OTBObject | str | float], nb_bands: int = None, ): """Create a list of 'fake' expressions, one for each band. -- GitLab From d4384d14129d1c5843e8bb191a52bb24a879b6e1 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Nov 2023 15:06:48 +0100 Subject: [PATCH 374/399] ENH: summarize comments --- pyotb/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index f03e9e1..b614a4d 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1758,6 +1758,7 @@ def summarize( return [summarize(o) for o in obj] if isinstance(obj, Output): return summarize(obj.parent_pyotb_app) + # => This is the deepest recursion level if not isinstance(obj, App): return obj @@ -1769,8 +1770,8 @@ def summarize( return param.split("?")[0] # Call / top level of recursion : obj is an App - # We need to return parameters values, summarized if param is an App parameters = {} + # We need to return parameters values, summarized if param is an App for key, param in obj.parameters.items(): if strip_inpath and obj.is_input(key) or strip_outpath and obj.is_output(key): parameters[key] = strip_path(param) -- GitLab From b219d44312a63d0e01372f3d1310c5a4f40e2d9b Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Nov 2023 15:06:50 +0100 Subject: [PATCH 375/399] DOC: avoid multiline argument text in docstrings --- pyotb/core.py | 104 +++++++++++++++++++++++++------------------------- 1 file changed, 51 insertions(+), 53 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index b614a4d..5528039 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -136,14 +136,14 @@ class OTBObject(ABC): return App("ComputeImagesStatistics", self, quiet=True).data def get_values_at_coords( - self, row: int, col: int, bands: int = None + self, row: int, col: int, bands: int | list[int] = None ) -> list[float] | float: """Get pixel value(s) at a given YX coordinates. Args: row: index along Y / latitude axis col: index along X / longitude axis - bands: band number, list or slice to fetch values from + bands: band number(s) to fetch values from Returns: single numerical value or a list of values for each band @@ -209,8 +209,7 @@ class OTBObject(ABC): Args: key: parameter key to export, if None then the default one will be used - preserve_dtype: when set to True, the numpy array is converted to the same pixel type as - the App first output. Default is True + preserve_dtype: convert the array to the same pixel type as the App first output Returns: the exported numpy array @@ -231,12 +230,13 @@ class OTBObject(ABC): ) -> np.ndarray: """Export a pyotb object to numpy array. + A copy is avoided by default, but may be required if preserve_dtype is False + and the source app reference is lost. + Args: key: the output parameter name to export as numpy array - preserve_dtype: when set to True, the numpy array is converted to the same pixel type as - the App first output. Default is True - copy: whether to copy the output array, default is False - required to True if preserve_dtype is False and the source app reference is lost + preserve_dtype: convert the array to the same pixel type as the App first output + copy: whether to copy the output array instead of returning a reference Returns: a numpy array that may already have been cached in self.exports_dic @@ -378,12 +378,12 @@ class OTBObject(ABC): """This is called whenever a numpy function is called on a pyotb object. Operation is performed in numpy, then imported back to pyotb with the same georeference as input. + At least one obj is unputs has to be an OTBObject. Args: ufunc: numpy function method: an internal numpy argument - inputs: inputs, at least one being pyotb object. If there are several pyotb objects, they must all have - the same georeference and pixel size. + inputs: inputs, with equal shape in case of several images / OTBObject **kwargs: kwargs of the numpy function Returns: @@ -552,18 +552,19 @@ class App(OTBObject): ): """Common constructor for OTB applications. Handles in-memory connection between apps. + There are several ways to pass parameters to init the app. *args can be : + - dictionary containing key-arguments enumeration. Useful when a key is python-reserved + (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") + - string, App or Output, useful when the user wants to specify the input "in" + - list, useful when the user wants to specify the input list 'il' + Args: appname: name of the OTB application to initialize, e.g. 'BandMath' - *args: used for passing application parameters. Can be : - - dictionary containing key-arguments enumeration. Useful when a key is python-reserved - (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") - - string, App or Output, useful when the user wants to specify the input "in" - - list, useful when the user wants to specify the input list 'il' + *args: used to pass an app input as argument and ommiting the key frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___ quiet: whether to print logs of the OTB app name: custom name that will show up in logs, appname will be used if not provided - **kwargs: used for passing application parameters. - e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif' + **kwargs: used for passing application parameters (e.g. il=["image_1.tif", "image_1.tif"]) """ # Attributes and data structures used by properties @@ -700,10 +701,8 @@ class App(OTBObject): instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths Args: - *args: Can be : - dictionary containing key-arguments enumeration. Useful when a keyword is reserved (e.g. "in") - - string or OTBObject, useful when the user implicitly wants to set the param "in" - - list, useful when the user implicitly wants to set the param "il" - **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif' + *args: any input OTBObject, filepath or images list, or a dict of parameters + **kwargs: app parameters, with "_" instead of dots e.g. io_in="image.tif" Raises: KeyError: when the parameter name wasn't recognized @@ -818,20 +817,19 @@ class App(OTBObject): ) -> bool: """Set output pixel type and write the output raster files. + The first argument is expected to be: + - filepath, useful when there is only one output, e.g. 'output.tif' + - dictionary containing output filepath + - None if output file was passed during App init + In case of multiple outputs, pixel_type may also be a dictionary with parameter names as keys. + Accepted pixel types : uint8, uint16, uint32, int16, int32, float, double, cint16, cint32, cfloat, cdouble + Args: - path: Can be : - filepath, useful when there is only one output, e.g. 'output.tif' - - dictionary containing key-arguments enumeration. Useful when a key contains - non-standard characters such as a point, e.g. {'io.out':'output.tif'} - - None if output file was passed during App init - pixel_type: Can be : - dictionary {out_param_key: pixeltype} when specifying for several outputs - - str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several - outputs, all outputs are written with this unique type. - Valid pixel types are uint8, uint16, uint32, int16, int32, float, double, - cint16, cint32, cfloat, cdouble. (Default value = None) + path: output filepath or dict of filepath with param keys + pixel_type: pixel type string representation preserve_dtype: propagate main input pixel type to outputs, in case pixel_type is None - ext_fname: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES") - Will be used for all outputs (Default value = None) - **kwargs: keyword arguments e.g. out='output.tif' + ext_fname: an OTB extended filename, will be applied to every output (but won't overwrite existing keys in output filepath) + **kwargs: keyword arguments e.g. out='output.tif' or io_out='output.tif' Returns: True if all files are found on disk @@ -1125,7 +1123,7 @@ class Slicer(App): obj: input rows: slice along Y / Latitude axis cols: slice along X / Longitude axis - channels: bands to extract. can be slicing, list or int + channels: bands to extract Raises: TypeError: if channels param isn't slice, list or int @@ -1218,7 +1216,7 @@ class Operation(App): Args: operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? - *inputs: operands, can be OTBObject, filepath, int or float + *inputs: operands of the expression to build nb_bands: optionally specify the output nb of bands - used only internally by pyotb.where name: override the default Operation name @@ -1292,7 +1290,7 @@ class Operation(App): E.g for the operation input1 + input2, we create a fake expression that is like "str(input1) + str(input2)" Args: - operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? + operator: one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? inputs: inputs. Can be OTBObject, filepath, int or float nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where @@ -1375,14 +1373,14 @@ class Operation(App): """This an internal function, only to be used by `build_fake_expressions`. Enable to create a fake expression just for one input and one band. + Regarding the "keep_logical" param: + - if True, for `input1 > input2`, returned fake expression is "str(input1) > str(input2)" + - if False, for `input1 > input2`, returned fake exp is "str(input1) > str(input2) ? 1 : 0"] Default False Args: x: input band: which band to consider (bands start at 1) keep_logical: whether to keep the logical expressions "as is" in case the input is a logical operation. - ex: if True, for `input1 > input2`, returned fake expression is "str(input1) > str(input2)" - if False, for `input1 > input2`, returned fake exp is "str(input1) > str(input2) ? 1 : 0"] - Default False Returns: fake_exp: the fake expression for this band and input @@ -1498,7 +1496,7 @@ class Input(App): """Default constructor. Args: - filepath: Anything supported by GDAL (local file on the filesystem, remote resource e.g. /vsicurl/.., etc.) + filepath: Anything supported by GDAL (local file on the filesystem, remote resource, etc.) """ super().__init__("ExtractROI", {"in": filepath}, quiet=True, frozen=True) @@ -1625,7 +1623,7 @@ def get_nbchannels(inp: str | Path | OTBObject) -> int: """Get the nb of bands of input image. Args: - inp: can be filepath or OTBObject object + inp: input file or OTBObject Returns: number of bands in image @@ -1651,14 +1649,16 @@ def get_nbchannels(inp: str | Path | OTBObject) -> int: def get_pixel_type(inp: str | Path | OTBObject) -> str: - """Get the encoding of input image pixels. + """Get the encoding of input image pixels as integer enum. + + OTB enum e.g. `otbApplication.ImagePixelType_uint8'. + For an OTBObject with several outputs, only the pixel type of the first output is returned Args: - inp: can be filepath or pyotb object + inp: input file or OTBObject Returns: - pixel_type: OTB enum e.g. `otbApplication.ImagePixelType_uint8', which actually is an int. - For an OTBObject with several outputs, only the pixel type of the first output is returned + OTB enum Raises: TypeError: if inp pixel type cannot be retrieved @@ -1685,7 +1685,7 @@ def parse_pixel_type(pixel_type: str | int) -> int: """Convert one str pixel type to OTB integer enum if necessary. Args: - pixel_type: pixel type. can be int or str (either OTB or numpy convention) + pixel_type: pixel type to parse Returns: pixel_type OTB enum integer value @@ -1739,21 +1739,19 @@ def summarize( At the deepest recursion level, this function just return any parameter value, path stripped if needed, or app summarized in case of a pipeline. + If strip_path is enabled, paths are truncated after the first "?" character. + Can be useful to remove URLs tokens from inputs (e.g. SAS or S3 credentials), + or extended filenames from outputs. Args: obj: input object / parameter value to summarize - strip_inpath: strip all input paths: If enabled, paths related to - inputs are truncated after the first "?" character. Can be useful - to remove URLs tokens (e.g. SAS or S3 credentials). - strip_outpath: strip all output paths: If enabled, paths related - to outputs are truncated after the first "?" character. Can be - useful to remove extended filenames. + strip_inpath: strip all input paths + strip_outpath: strip all output paths Returns: nested dictionary containing name and parameters of an app and its parents """ - # This is the deepest recursion level if isinstance(obj, list): return [summarize(o) for o in obj] if isinstance(obj, Output): -- GitLab From d729f02568fc4949538be2b413630efc418c0976 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Nov 2023 15:11:40 +0100 Subject: [PATCH 376/399] STYLE: apply black --- pyotb/core.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 5528039..5c33706 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1081,9 +1081,7 @@ class App(OTBObject): self.data[key] = value # Special functions - def __getitem__( - self, key: str | tuple - ) -> Any | list[float] | float | Slicer: + def __getitem__(self, key: str | tuple) -> Any | list[float] | float | Slicer: """This function is called when we use App()[...]. We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer @@ -1270,9 +1268,7 @@ class Operation(App): return 1 # Check that all inputs have the same band count nb_bands_list = [ - get_nbchannels(inp) - for inp in inputs - if not isinstance(inp, (float, int)) + get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int)) ] all_same = all(x == nb_bands_list[0] for x in nb_bands_list) if len(nb_bands_list) > 1 and not all_same: @@ -1373,7 +1369,7 @@ class Operation(App): """This an internal function, only to be used by `build_fake_expressions`. Enable to create a fake expression just for one input and one band. - Regarding the "keep_logical" param: + Regarding the "keep_logical" param: - if True, for `input1 > input2`, returned fake expression is "str(input1) > str(input2)" - if False, for `input1 > input2`, returned fake exp is "str(input1) > str(input2) ? 1 : 0"] Default False @@ -1689,7 +1685,7 @@ def parse_pixel_type(pixel_type: str | int) -> int: Returns: pixel_type OTB enum integer value - + Raises: KeyError: if pixel_type name is unknown TypeError: if type(pixel_type) isn't int or str -- GitLab From 09bfa60e027fb6bc7fd6b036e2d97649e4eb5952 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Nov 2023 15:17:17 +0100 Subject: [PATCH 377/399] DOC: typo --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 5c33706..b6655f3 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -560,7 +560,7 @@ class App(OTBObject): Args: appname: name of the OTB application to initialize, e.g. 'BandMath' - *args: used to pass an app input as argument and ommiting the key + *args: used to pass an app input as argument and omitting the key frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___ quiet: whether to print logs of the OTB app name: custom name that will show up in logs, appname will be used if not provided -- GitLab From 3ee79f5672397fd4d985a7c43f96d9c6a0182482 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Nov 2023 15:38:43 +0100 Subject: [PATCH 378/399] DOC: test move docs from __init__ to class docstring --- pyotb/core.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index b6655f3..f578a66 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -513,7 +513,24 @@ class OTBObject(ABC): class App(OTBObject): - """Base class that gathers common operations for any OTB application.""" + """Wrapper around otb.Application to handle settings and execution. + + Base class that gathers common operations for any OTB application lifetime (settings, exec, export, etc.) + There are several ways to pass parameters to init the app. *args can be : + - dictionary containing key-arguments enumeration. Useful when a key is python-reserved + (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") + - string, App or Output, useful when the user wants to specify the input "in" + - list, useful when the user wants to specify the input list 'il' + + Args: + appname: name of the OTB application to initialize, e.g. 'BandMath' + *args: used to pass an app input as argument and omitting the key + frozen: freeze OTB app in order avoid blocking during __init___ + quiet: whether to print logs of the OTB app + name: custom name that will show up in logs, appname will be used if not provided + **kwargs: used for passing application parameters (e.g. il=["image_1.tif", "image_1.tif"]) + + """ INPUT_IMAGE_TYPES = [ otb.ParameterType_InputImage, @@ -550,23 +567,7 @@ class App(OTBObject): name: str = "", **kwargs, ): - """Common constructor for OTB applications. Handles in-memory connection between apps. - - There are several ways to pass parameters to init the app. *args can be : - - dictionary containing key-arguments enumeration. Useful when a key is python-reserved - (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") - - string, App or Output, useful when the user wants to specify the input "in" - - list, useful when the user wants to specify the input list 'il' - - Args: - appname: name of the OTB application to initialize, e.g. 'BandMath' - *args: used to pass an app input as argument and omitting the key - frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___ - quiet: whether to print logs of the OTB app - name: custom name that will show up in logs, appname will be used if not provided - **kwargs: used for passing application parameters (e.g. il=["image_1.tif", "image_1.tif"]) - - """ + """Common constructor for OTB applications, automatically handles in-memory connections.""" # Attributes and data structures used by properties create = ( otb.Registry.CreateApplicationWithoutLogger -- GitLab From d44ffae4691678b535c067effa31a3d4677e5371 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Nov 2023 15:51:57 +0100 Subject: [PATCH 379/399] ENH: App docstring --- pyotb/core.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index f578a66..d6a60d6 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -16,7 +16,7 @@ from .depreciation import deprecated_alias, depreciation_warning, deprecated_att class OTBObject(ABC): - """Abstraction of an image object.""" + """Abstraction of an image object, for a whole app or one specific output.""" @property @abstractmethod @@ -269,7 +269,7 @@ class OTBObject(ABC): y: latitude or projected Y Returns: - pixel index: (row, col) + pixel index as (row, col) """ spacing_x, _, origin_x, _, spacing_y, origin_y = self.transform @@ -516,17 +516,17 @@ class App(OTBObject): """Wrapper around otb.Application to handle settings and execution. Base class that gathers common operations for any OTB application lifetime (settings, exec, export, etc.) - There are several ways to pass parameters to init the app. *args can be : - - dictionary containing key-arguments enumeration. Useful when a key is python-reserved - (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit") - - string, App or Output, useful when the user wants to specify the input "in" - - list, useful when the user wants to specify the input list 'il' + Any app parameter may be passed either using a dict of parameters or keyword argument. + The first argument can be : + - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in") + - string, App or Output, useful when the user wants to specify the input "in" + - list, useful when the user wants to specify the input list 'il' Args: appname: name of the OTB application to initialize, e.g. 'BandMath' *args: used to pass an app input as argument and omitting the key frozen: freeze OTB app in order avoid blocking during __init___ - quiet: whether to print logs of the OTB app + quiet: whether to print logs of the OTB app and the default progress bar name: custom name that will show up in logs, appname will be used if not provided **kwargs: used for passing application parameters (e.g. il=["image_1.tif", "image_1.tif"]) -- GitLab From 12881408bcb8308f9834d0df6e9eaebb6ed3e147 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Nov 2023 15:57:02 +0100 Subject: [PATCH 380/399] DOC: enh class docstring --- pyotb/core.py | 106 ++++++++++++++++++++++++-------------------------- 1 file changed, 51 insertions(+), 55 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index d6a60d6..29535f3 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -517,7 +517,7 @@ class App(OTBObject): Base class that gathers common operations for any OTB application lifetime (settings, exec, export, etc.) Any app parameter may be passed either using a dict of parameters or keyword argument. - The first argument can be : + The first argument can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in") - string, App or Output, useful when the user wants to specify the input "in" - list, useful when the user wants to specify the input list 'il' @@ -1103,7 +1103,22 @@ class App(OTBObject): class Slicer(App): - """Slicer objects i.e. when we call something like raster[:, :, 2] from Python.""" + """Slicer objects, automatically created when using slicing e.g. app[:, :, 2]. + + It contains : + - an ExtractROI app that handles extracting bands and ROI and can be written to disk or used in pipelines + - in case the user only wants to extract one band, an expression such as "im1b#" + + Args: + obj: input + rows: slice along Y / Latitude axis + cols: slice along X / Longitude axis + channels: bands to extract + + Raises: + TypeError: if channels param isn't slice, list or int + + """ def __init__( self, @@ -1112,22 +1127,7 @@ class Slicer(App): cols: slice, channels: slice | list[int] | int, ): - """Create a slicer object, that can be used directly for writing or inside a BandMath. - - It contains : - - an ExtractROI app that handles extracting bands and ROI and can be written to disk or used in pipelines - - in case the user only wants to extract one band, an expression such as "im1b#" - - Args: - obj: input - rows: slice along Y / Latitude axis - cols: slice along X / Longitude axis - channels: bands to extract - - Raises: - TypeError: if channels param isn't slice, list or int - - """ + """Create a slicer object, that can be used directly for writing or inside a BandMath.""" super().__init__( "ExtractROI", obj, @@ -1189,6 +1189,16 @@ class Slicer(App): class Operation(App): """Class for arithmetic/math operations done in Python. + Given some inputs and an operator, this object enables to python operator to a BandMath operation. + Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator. + It can have 3 inputs for the ternary operator `cond ? x : y`. + + Args: + operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? + *inputs: operands of the expression to build + nb_bands: optionally specify the output nb of bands - used only internally by pyotb.where + name: override the default Operation name + Example: Consider the python expression (input1 + 2 * input2) > 0. This class enables to create a BandMathX app, with expression such as (im2 + 2 * im1) > 0 ? 1 : 0 @@ -1208,18 +1218,7 @@ class Operation(App): """ def __init__(self, operator: str, *inputs, nb_bands: int = None, name: str = None): - """Given some inputs and an operator, this object enables to python operator to a BandMath operation. - - Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator. - It can have 3 inputs for the ternary operator `cond ? x : y`. - - Args: - operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ? - *inputs: operands of the expression to build - nb_bands: optionally specify the output nb of bands - used only internally by pyotb.where - name: override the default Operation name - - """ + """Operation constructor, one part of the logic is handled by App.__create_operator""" self.operator = operator # We first create a 'fake' expression. E.g for the operation `input1 + input2` # we create a fake expression like "str(input1) + str(input2)" @@ -1426,21 +1425,18 @@ class LogicalOperation(Operation): """A specialization of Operation class for boolean logical operations. Supported operators are >, <, >=, <=, ==, !=, `&` and `|`. - The only difference is that not only the BandMath expression is saved (e.g. "im1b1 > 0 ? 1 : 0"), but also the logical expression (e.g. "im1b1 > 0") + Args: + operator: string operator (one of >, <, >=, <=, ==, !=, &, |) + *inputs: inputs + nb_bands: optionally specify the output nb of bands - used only by pyotb.where + """ def __init__(self, operator: str, *inputs, nb_bands: int = None): - """Constructor for a LogicalOperation object. - - Args: - operator: string operator (one of >, <, >=, <=, ==, !=, &, |) - *inputs: inputs - nb_bands: optionally specify the output nb of bands - used only by pyotb.where - - """ + """Constructor for a LogicalOperation object.""" self.logical_fake_exp_bands = [] super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation") self.logical_exp_bands, self.logical_exp = self.get_real_exp( @@ -1487,15 +1483,15 @@ class LogicalOperation(Operation): class Input(App): - """Class for transforming a filepath to pyOTB object.""" + """Class for transforming a filepath to pyOTB object. - def __init__(self, filepath: str): - """Default constructor. + Args: + filepath: Anything supported by GDAL (local file on the filesystem, remote resource, etc.) - Args: - filepath: Anything supported by GDAL (local file on the filesystem, remote resource, etc.) + """ - """ + def __init__(self, filepath: str): + """Initialize an ExtractROI OTB app from a filepath, set dtype and store filepath.""" super().__init__("ExtractROI", {"in": filepath}, quiet=True, frozen=True) self._name = f"Input from {filepath}" if not filepath.startswith(("/vsi", "http://", "https://", "ftp://")): @@ -1510,7 +1506,15 @@ class Input(App): class Output(OTBObject): - """Object that behave like a pointer to a specific application in-memory output or file.""" + """Object that behave like a pointer to a specific application in-memory output or file. + + Args: + pyotb_app: The pyotb App to store reference from + param_key: Output parameter key of the target app + filepath: path of the output file (if not memory) + mkdir: create missing parent directories + + """ _filepath: str | Path = None @@ -1522,15 +1526,7 @@ class Output(OTBObject): filepath: str = None, mkdir: bool = True, ): - """Constructor for an Output object. - - Args: - pyotb_app: The pyotb App to store reference from - param_key: Output parameter key of the target app - filepath: path of the output file (if not memory) - mkdir: create missing parent directories - - """ + """Constructor for an Output object, initialized during App.__init__.""" self.parent_pyotb_app = pyotb_app # keep trace of parent app self.param_key = param_key self.filepath = filepath -- GitLab From 212e40feebade49a6cb65a35bd788fb3805050a5 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Nov 2023 16:04:18 +0100 Subject: [PATCH 381/399] DOC: format indent to fix bulletpoints --- pyotb/core.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 29535f3..7f60d0b 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -467,10 +467,8 @@ class OTBObject(ABC): """Override the default __getitem__ behaviour. This function enables 2 things : - - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3] - selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]] - selecting 1000x1000 subset : object[:1000, :1000] - - access pixel value(s) at a specified row, col index + - slicing, i.e. selecting ROI/bands + - access pixel value(s) at a specified row, col index Args: key: attribute key @@ -517,10 +515,10 @@ class App(OTBObject): Base class that gathers common operations for any OTB application lifetime (settings, exec, export, etc.) Any app parameter may be passed either using a dict of parameters or keyword argument. - The first argument can be : - - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in") - - string, App or Output, useful when the user wants to specify the input "in" - - list, useful when the user wants to specify the input list 'il' + The first argument can be: + - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in") + - string, App or Output, useful when the user wants to specify the input "in" + - list, useful when the user wants to specify the input list 'il' Args: appname: name of the OTB application to initialize, e.g. 'BandMath' @@ -1105,9 +1103,9 @@ class App(OTBObject): class Slicer(App): """Slicer objects, automatically created when using slicing e.g. app[:, :, 2]. - It contains : - - an ExtractROI app that handles extracting bands and ROI and can be written to disk or used in pipelines - - in case the user only wants to extract one band, an expression such as "im1b#" + It contains: + - an ExtractROI app that handles extracting bands and ROI and can be written to disk or used in pipelines + - in case the user only wants to extract one band, an expression such as "im1b#" Args: obj: input -- GitLab From 36d0892b9303c6a948e8a3af16465898f0885ce7 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Nov 2023 16:17:21 +0100 Subject: [PATCH 382/399] DOC: enh docstrings with bulletpoints --- pyotb/core.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 7f60d0b..47109da 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -515,6 +515,7 @@ class App(OTBObject): Base class that gathers common operations for any OTB application lifetime (settings, exec, export, etc.) Any app parameter may be passed either using a dict of parameters or keyword argument. + The first argument can be: - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in") - string, App or Output, useful when the user wants to specify the input "in" @@ -820,6 +821,7 @@ class App(OTBObject): - filepath, useful when there is only one output, e.g. 'output.tif' - dictionary containing output filepath - None if output file was passed during App init + In case of multiple outputs, pixel_type may also be a dictionary with parameter names as keys. Accepted pixel types : uint8, uint16, uint32, int16, int32, float, double, cint16, cint32, cfloat, cdouble @@ -1103,9 +1105,8 @@ class App(OTBObject): class Slicer(App): """Slicer objects, automatically created when using slicing e.g. app[:, :, 2]. - It contains: - - an ExtractROI app that handles extracting bands and ROI and can be written to disk or used in pipelines - - in case the user only wants to extract one band, an expression such as "im1b#" + Can be used to select a subset of pixel and / or bands in the image. + This is a shortcut to an ExtractROI app that can be written to disk or used in pipelines. Args: obj: input @@ -1423,8 +1424,8 @@ class LogicalOperation(Operation): """A specialization of Operation class for boolean logical operations. Supported operators are >, <, >=, <=, ==, !=, `&` and `|`. - The only difference is that not only the BandMath expression is saved (e.g. "im1b1 > 0 ? 1 : 0"), - but also the logical expression (e.g. "im1b1 > 0") + The only difference is that not only the BandMath expression is saved + (e.g. "im1b1 > 0 ? 1 : 0"), but also the logical expression (e.g. "im1b1 > 0") Args: operator: string operator (one of >, <, >=, <=, ==, !=, &, |) @@ -1449,8 +1450,8 @@ class LogicalOperation(Operation): ): """Create a list of 'fake' expressions, one for each band. - e.g for the operation input1 > input2, we create a fake expression that is like - "str(input1) > str(input2) ? 1 : 0" and a logical fake expression that is like "str(input1) > str(input2)" + For the operation input1 > input2, we create a fake expression like `str(input1) > str(input2) ? 1 : 0` + and a logical fake expression like `str(input1) > str(input2)` Args: operator: str (one of >, <, >=, <=, ==, !=, &, |) @@ -1630,9 +1631,7 @@ def get_nbchannels(inp: str | Path | OTBObject) -> int: try: info = App("ReadImageInfo", inp, quiet=True) return info["numberbands"] - except ( - RuntimeError - ) as info_err: # this happens when we pass a str that is not a filepath + except RuntimeError as info_err: # e.g. file is missing raise TypeError( f"Could not get the number of channels file '{inp}' ({info_err})" ) from info_err -- GitLab From 8a131d01d0b642a8971489763973a77821a4b422 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Nov 2023 16:23:29 +0100 Subject: [PATCH 383/399] DOC: add missing dot --- pyotb/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/core.py b/pyotb/core.py index 47109da..346dc31 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -1217,7 +1217,7 @@ class Operation(App): """ def __init__(self, operator: str, *inputs, nb_bands: int = None, name: str = None): - """Operation constructor, one part of the logic is handled by App.__create_operator""" + """Operation constructor, one part of the logic is handled by App.__create_operator.""" self.operator = operator # We first create a 'fake' expression. E.g for the operation `input1 + input2` # we create a fake expression like "str(input1) + str(input2)" -- GitLab From b16538db952eec7e012816f4e7d88993c1860d8a Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Nov 2023 16:37:59 +0100 Subject: [PATCH 384/399] ENH: docstrings --- pyotb/core.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 346dc31..bc6529e 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -517,9 +517,9 @@ class App(OTBObject): Any app parameter may be passed either using a dict of parameters or keyword argument. The first argument can be: - - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in") - - string, App or Output, useful when the user wants to specify the input "in" + - string, App or Output, the main input parameter name is automatically set - list, useful when the user wants to specify the input list 'il' + - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in", "map") Args: appname: name of the OTB application to initialize, e.g. 'BandMath' @@ -695,8 +695,10 @@ class App(OTBObject): return self._time_end - self._time_start def set_parameters(self, *args, **kwargs): - """Set some parameters of the app. + """Set parameters, using the right OTB API function depending on the key and type. + Parameters with dots may be passed as keyword arguments using "_", e.g. map_epsg_code=4326. + Additional checks are done for input and output (in-memory objects, remote filepaths, etc.). When useful, e.g. for images list, this function appends the parameters instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths @@ -1482,7 +1484,7 @@ class LogicalOperation(Operation): class Input(App): - """Class for transforming a filepath to pyOTB object. + """Class for transforming a filepath to pyotb object. Args: filepath: Anything supported by GDAL (local file on the filesystem, remote resource, etc.) -- GitLab From 238dce9911046b714eab8baea38ed24dedaeeaf1 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Nov 2023 16:49:11 +0100 Subject: [PATCH 385/399] ENH: App docstring --- pyotb/core.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index bc6529e..8afd168 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -517,17 +517,18 @@ class App(OTBObject): Any app parameter may be passed either using a dict of parameters or keyword argument. The first argument can be: - - string, App or Output, the main input parameter name is automatically set - - list, useful when the user wants to specify the input list 'il' - - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in", "map") + - filepath or OTBObject, the main input parameter name is automatically used + - list of inputs, useful when the user wants to specify the input list `il` + - dictionary of parameters, useful when a key is python-reserved (e.g. `in`, `map`) + Any key except reserved keywards may also be passed via kwargs, if you replace dots with "_" e.g `map_epsg_code=4326` Args: appname: name of the OTB application to initialize, e.g. 'BandMath' - *args: used to pass an app input as argument and omitting the key + *args: can be a filepath, OTB object or a dict or parameters, several dicts will be merged in **kwargs frozen: freeze OTB app in order avoid blocking during __init___ quiet: whether to print logs of the OTB app and the default progress bar name: custom name that will show up in logs, appname will be used if not provided - **kwargs: used for passing application parameters (e.g. il=["image_1.tif", "image_1.tif"]) + **kwargs: any OTB application parameter key is accepted except "in" """ -- GitLab From e55ec155aa0ca9338a66c02d53b6341a081c2fbb Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Nov 2023 17:00:25 +0100 Subject: [PATCH 386/399] STYLE: use comprehensive list instead of filter + whitespace --- pyotb/core.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 8afd168..2b7af8c 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -581,6 +581,7 @@ class App(OTBObject): self._time_start, self._time_end = 0.0, 0.0 self.data, self.outputs = {}, {} self.quiet, self.frozen = quiet, frozen + # Param keys and types self.parameters_keys = tuple(self.app.GetParametersKeys()) self._all_param_types = { @@ -596,15 +597,18 @@ class App(OTBObject): for key in self.parameters_keys if self.app.GetParameterType(key) == otb.ParameterType_Choice } + # Init, execute and write (auto flush only when output param was provided) if args or kwargs: self.set_parameters(*args, **kwargs) # Create Output image objects - for key in filter( - lambda k: self._out_param_types[k] == otb.ParameterType_OutputImage, - self._out_param_types, + for key in ( + key + for key, param in self._out_param_types.items() + if param == otb.ParameterType_OutputImage ): self.outputs[key] = Output(self, key, self._settings.get(key)) + if not self.frozen: self.execute() if any(key in self._settings for key in self._out_param_types): -- GitLab From e18bcfc3394a4b1cca5fbe4c22eaee74cb328cf2 Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Mon, 6 Nov 2023 18:37:43 +0100 Subject: [PATCH 387/399] CI: typo --- pyotb/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyotb/core.py b/pyotb/core.py index 2b7af8c..9cd9d4e 100644 --- a/pyotb/core.py +++ b/pyotb/core.py @@ -520,7 +520,7 @@ class App(OTBObject): - filepath or OTBObject, the main input parameter name is automatically used - list of inputs, useful when the user wants to specify the input list `il` - dictionary of parameters, useful when a key is python-reserved (e.g. `in`, `map`) - Any key except reserved keywards may also be passed via kwargs, if you replace dots with "_" e.g `map_epsg_code=4326` + Any key except "in" or "map" can also be passed via kwargs, replace "." with "_" e.g `map_epsg_code=4326` Args: appname: name of the OTB application to initialize, e.g. 'BandMath' @@ -528,7 +528,7 @@ class App(OTBObject): frozen: freeze OTB app in order avoid blocking during __init___ quiet: whether to print logs of the OTB app and the default progress bar name: custom name that will show up in logs, appname will be used if not provided - **kwargs: any OTB application parameter key is accepted except "in" + **kwargs: any OTB application parameter key is accepted except "in" or "map" """ -- GitLab From ccb4fe2a073604079fabf8cd1145144d394f1b67 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 8 Nov 2023 17:07:20 +0100 Subject: [PATCH 388/399] DOC: add sumarize() documentation --- doc/index.md | 1 + doc/summarize.md | 167 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 doc/summarize.md diff --git a/doc/index.md b/doc/index.md index 7861878..d76ef33 100644 --- a/doc/index.md +++ b/doc/index.md @@ -22,6 +22,7 @@ to make OTB more Python friendly. ## Advanced use - [Comparison between pyotb and OTB native library](comparison_otb.md) +- [Summarize applications](summarize.md) - [OTB versions](otb_versions.md) - [Managing loggers](managing_loggers.md) - [Troubleshooting & limitations](troubleshooting.md) diff --git a/doc/summarize.md b/doc/summarize.md new file mode 100644 index 0000000..13f3021 --- /dev/null +++ b/doc/summarize.md @@ -0,0 +1,167 @@ +## Summarize applications + +pyotb enables to summarize applications as a dictionary with keys/values for +parameters. This feature can be used to keep track of a process, composed of +multiple applications chained together. + +### Single application + +Let's take the example of one single application. + +```python +import pyotb + +app = pyotb.RigidTransformResample({ + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 +}) +``` + +The application can be summarized using `pyotb.summarize()` or +`app.summary()`, which are equivalent. + +```python +print(app.summarize()) +``` + +Results in the following (lines have been pretty printed for the sake of +documentation): + +```json lines +{ + 'name': 'RigidTransformResample', + 'parameters': { + 'transform.type': 'id', + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 + } +} +``` + +Note that we can also summarize an application after it has been executed: + +```python +app.write('output.tif', pixel_type='uint16') +print(app.summarize()) +``` + +Which results in the following: + +```json lines +{ + 'name': 'RigidTransformResample', + 'parameters': { + 'transform.type': 'id', + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5, + 'out': 'output.tif' + } +} +``` + +Now `'output.tif'` has been added to the application parameters. + +### Multiple applications chained together (pipeline) + +When multiple applications are chained together, the summary of the last +application will describe all upstream processes. + +```python +import pyotb + +app1 = pyotb.RigidTransformResample({ + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 +}) +app2 = pyotb.Smoothing(app1) +print(app2.summarize()) +``` + +Results in: + +```json lines +{ + 'name': 'Smoothing', + 'parameters': { + 'type': 'anidif', + 'type.anidif.timestep': 0.125, + 'type.anidif.nbiter': 10, + 'type.anidif.conductance': 1.0, + 'in': { + 'name': 'RigidTransformResample', + 'parameters': { + 'transform.type': 'id', + 'in': 'my_image.tif', + 'interpolator': 'linear', + 'transform.type.id.scaley': 0.5, + 'transform.type.id.scalex': 0.5 + } + } + } +} +``` + +### Remote files URL stripping + +Cloud-based raster URLs often include tokens or random strings resulting from +the URL signing. +Those can be removed from the summarized paths, using the `strip_inpath` +and/or `strip_outpath` arguments respectively for inputs and/or outputs. + +Here is an example with Microsoft Planetary Computer: + +```python +import planetary_computer +import pyotb + +url = ( + "https://sentinel2l2a01.blob.core.windows.net/sentinel2-l2/31/N/EA/2023/" + "11/03/S2A_MSIL2A_20231103T095151_N0509_R079_T31NEA_20231103T161409.SAFE/" + "GRANULE/L2A_T31NEA_A043691_20231103T100626/IMG_DATA/R10m/T31NEA_20231103" + "T095151_B02_10m.tif" +) +signed_url = planetary_computer.sign_inplace(url) +app = pyotb.Smoothing(signed_url) +``` + +By default, the summary does not strip the URL. + +```python +print(app.summarize()["parameters"]["in"]) +``` + +This results in: + +``` +/vsicurl/https://sentinel2l2a01.blob.core.windows.net/sentinel2-l2/31/N/EA/... +2023/11/03/S2A_MSIL2A_20231103T095151_N0509_R079_T31NEA_20231103T161409.SAFE... +/GRANULE/L2A_T31NEA_A043691_20231103T100626/IMG_DATA/R10m/T31NEA_20231103T... +095151_B02_10m.tif?st=2023-11-07T15%3A52%3A47Z&se=2023-11-08T16%3A37%3A47Z&... +sp=rl&sv=2021-06-08&sr=c&skoid=c85c15d6-d1ae-42d4-af60-e2ca0f81359b&sktid=... +72f988bf-86f1-41af-91ab-2d7cd011db47&skt=2023-11-08T11%3A41%3A41Z&ske=2023-... +11-15T11%3A41%3A41Z&sks=b&skv=2021-06-08&sig=xxxxxxxxxxx...xxxxx +``` + +Now we can strip the URL to keep only the resource identifier and get rid of +the token: + +```python +print(app.summarize(strip_inpath=True)["parameters"]["in"]) +``` + +Which now results in: + +``` +/vsicurl/https://sentinel2l2a01.blob.core.windows.net/sentinel2-l2/31/N/EA/... +2023/11/03/S2A_MSIL2A_20231103T095151_N0509_R079_T31NEA_20231103T161409.SAFE... +/GRANULE/L2A_T31NEA_A043691_20231103T100626/IMG_DATA/R10m/T31NEA_20231103T... +095151_B02_10m.tif +``` \ No newline at end of file -- GitLab From bd2bbc419dc80b584886773f293c183c47101786 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 8 Nov 2023 18:11:20 +0100 Subject: [PATCH 389/399] DOC: features (dtype, transform, etc) --- doc/features.md | 58 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/doc/features.md b/doc/features.md index 2058af5..6af1355 100644 --- a/doc/features.md +++ b/doc/features.md @@ -32,18 +32,64 @@ inp = pyotb.Input('my_image.tif') inp[:, :, :3] # selecting first 3 bands inp[:, :, [0, 1, 4]] # selecting bands 1, 2 & 5 -inp[:1000, :1000] # selecting 1000x1000 subset, same as inp[:1000, :1000, :] +inp[:, :, 1:-1] # removing first and last band +inp[:, :, ::2] # selecting one band every 2 bands +inp[:100, :100] # selecting 100x100 subset, same as inp[:100, :100, :] inp[:100, :100].write('my_image_roi.tif') # write cropped image to disk ``` -## Shape attributes +## Retrieving a pixel location in image coordinates -You can access the shape of any in-memory pyotb object. +One can retrieve a pixel location in image coordinates (i.e. row and column +indices) using `get_rowcol_from_xy()`: ```python -import pyotb +inp.get_rowcol_from_xy(760086.0, 6948092.0) # (333, 5) +``` -# transforming filepath to pyotb object -inp = pyotb.Input('my_image.tif') +## Reading a pixel value + +One can read a pixel value of a pyotb object using brackets, as if it was a +common array. Returned is a list of pixel values for each band: + +```python +inp[10, 10] # [217, 202, 182, 255] +``` + +!!! warning + + Accessing multiple pixels values if not computationally efficient. Please + use this with moderation, or consider numpy or pyotb applications to + process efficiently blocks of pixels. + +## Attributes + +### Shape + +The shape of pyotb objects can be retrieved using `shape`. + +```python print(inp[:1000, :500].shape) # (1000, 500, 4) ``` + +### Pixel type + +The pixel type of pyotb objects can be retrieved using `dtype`. + +```python +inp.dtype # e.g. 'uint8' +``` + +!!! note + + The `dtype` returns a `str` corresponding to values accepted by the + `pixel_type` of `write()` + +### Transform + +The transform, as defined in GDAL, can be retrieved with the `transform` +attribute: + +```python +inp.transform # (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0) +``` \ No newline at end of file -- GitLab From 4e9d21ee76eae47f5521178e58b0c08af37e8b29 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 8 Nov 2023 18:12:27 +0100 Subject: [PATCH 390/399] DOC: sections titles --- doc/interaction.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/interaction.md b/doc/interaction.md index 0195b70..dcb8729 100644 --- a/doc/interaction.md +++ b/doc/interaction.md @@ -1,4 +1,6 @@ -## Export to Numpy +## Numpy + +### Export to numpy arrays pyotb objects can be exported to numpy array. @@ -16,7 +18,7 @@ arr = np.asarray(calibrated) arr = calibrated.to_numpy() ``` -## Interaction with Numpy +### Interact with numpy functions pyotb objects can be transparently used in numpy functions. @@ -47,9 +49,9 @@ noisy_image.write('image_plus_noise.tif') - The georeference can not be modified. Thus, numpy operations can not change the image or pixel size -## Export to rasterio +## Rasterio -pyotb objects can also be exported in a format that is usable by rasterio. +pyotb objects can also be exported in a format usable by rasterio. For example: @@ -88,7 +90,7 @@ as the user gets the `profile` dictionary. If the georeference or pixel size is modified, the user can update the `profile` accordingly. -## Interaction with Tensorflow +## Tensorflow We saw that numpy operations had some limitations. To bypass those limitations, it is possible to use some Tensorflow operations on pyotb objects. -- GitLab From 1f718193d6c20c9bae2a699597260bf372811d59 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Wed, 8 Nov 2023 18:13:11 +0100 Subject: [PATCH 391/399] DOC: note for linked python API --- doc/otb_versions.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/otb_versions.md b/doc/otb_versions.md index 50f30ee..aa8889d 100644 --- a/doc/otb_versions.md +++ b/doc/otb_versions.md @@ -40,8 +40,10 @@ Here is the path precedence for this automatic env configuration : OR (for windows) : C:/Program Files ``` -N.B. : in case `otbApplication` is found in `PYTHONPATH` (and if `OTB_ROOT` -was not set), the OTB which the python API is linked to will be used. +!!! Note + + When `otbApplication` is found in `PYTHONPATH` (and `OTB_ROOT` not set), + the OTB installation where the python API is linked, will be used. ## Fresh OTB installation -- GitLab From a8b5bf9550fc2b1efd97319c785cc3df694b1d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr> Date: Wed, 8 Nov 2023 17:43:19 +0000 Subject: [PATCH 392/399] DOC: Update RELEASE_NOTES.txt --- RELEASE_NOTES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index 746a2a6..9b1bad1 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -11,7 +11,7 @@ - Easy access to pixel coordinates - Add function to transform x,y coordinates into row, col - Native support of vsicurl inputs -- Fixes in `summarize()` +- Fixes and enhancements in `summarize()` - Fixes in `shape` - Add typing to function defs to enhance documentation -- GitLab From 87ab40a12c9062c6f6be7d7ff0e94e7bb7dc0d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr> Date: Thu, 9 Nov 2023 12:50:10 +0000 Subject: [PATCH 393/399] CI: doc stage rules --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 27cb2d6..2a2bf38 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -119,6 +119,8 @@ docs: stage: Documentation rules: - changes: + - *.txt + - *.md - mkdocs.yml - doc/* - pyotb/*.py -- GitLab From 4f6874d46b10c72d192036d378253fa3e821fb55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr> Date: Thu, 9 Nov 2023 12:50:56 +0000 Subject: [PATCH 394/399] CI: doc stage rules --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2a2bf38..57c9c42 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -119,8 +119,8 @@ docs: stage: Documentation rules: - changes: - - *.txt - - *.md + - "*.txt" + - "*.md" - mkdocs.yml - doc/* - pyotb/*.py -- GitLab From 0221a9e3dc4feab370292f5ef44b2912905a7f69 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 9 Nov 2023 14:57:21 +0100 Subject: [PATCH 395/399] DOC: update readme --- README.md | 46 +++++++++++++--------------------------------- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 6659aab..f5aa184 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,20 @@ -# pyotb: a pythonic extension of Orfeo Toolbox +# pyotb: Orfeo ToolBox for Python [](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/releases) [](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop) [](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop) [](https://pyotb.readthedocs.io/en/master/) -**pyotb** wraps the [Orfeo Toolbox](https://www.orfeo-toolbox.org/) (OTB) -python bindings to make it more developer friendly. +**pyotb** wraps the [Orfeo Toolbox](https://www.orfeo-toolbox.org/) in a pythonic, developer friendly +fashion. ## Key features -- Easy use of OTB applications from python +- Easy use of Orfeo ToolBox (OTB) applications from python - Simplify common sophisticated I/O features of OTB -- Lazy execution of in-memory pipelines with OTB streaming mechanism -- Interoperable with popular python libraries (numpy, rasterio) +- Lazy execution of operations thanks to OTB streaming mechanism +- Interoperable with popular python libraries ([numpy](https://numpy.org/) and +[rasterio](https://rasterio.readthedocs.io/)) - Extensible Documentation hosted at [pyotb.readthedocs.io](https://pyotb.readthedocs.io/). @@ -25,44 +26,23 @@ Building a simple pipeline with OTB applications ```py import pyotb -# RigidTransformResample application, with input parameters as dict +# RigidTransformResample, with input parameters as dict resampled = pyotb.RigidTransformResample({ - "in": "https://some.remote.data/input.tif", # Note: no /vsicurl/... + "in": "https://myserver.ia/input.tif", # Note: no /vsicurl/ "interpolator": "linear", "transform.type.id.scaley": 0.5, "transform.type.id.scalex": 0.5 }) -# OpticalCalibration, with automatic input parameters resolution +# OpticalCalibration, with input parameters as args calib = pyotb.OpticalCalibration(resampled) -# BandMath, with input parameters passed as kwargs +# BandMath, with input parameters as kwargs ndvi = pyotb.BandMath(calib, exp="ndvi(im1b1, im1b4)") -# Pythonic slicing using lazy computation (no memory used) +# Pythonic slicing roi = ndvi[20:586, 9:572] -# Pipeline execution -# The actual computation happens here ! +# Pipeline execution. The actual computation happens here! roi.write("output.tif", "float") ``` - -pyotb's objects also enable easy interoperability with -[numpy](https://numpy.org/) and [rasterio](https://rasterio.readthedocs.io/): - -```python -# Numpy and RasterIO style attributes -print(roi.shape, roi.dtype, roi.transform) -print(roi.metadata) - -# Other useful information -print(roi.get_infos()) -print(roi.get_statistics()) - -array = roi.to_numpy() -array, profile = roi.to_rasterio() -``` - -## Contributing - -Contributions are welcome on [Github](https://github.com/orfeotoolbox/pyotb) or the source repository hosted on the Orfeo ToolBox [GitLab](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb). -- GitLab From 1e2330cd78d2a39542d248508f0c4c19cdc73586 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 9 Nov 2023 14:57:48 +0100 Subject: [PATCH 396/399] DOC: add metadata, get_statistics(), get_info() --- doc/features.md | 90 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/doc/features.md b/doc/features.md index 6af1355..3a55c03 100644 --- a/doc/features.md +++ b/doc/features.md @@ -92,4 +92,94 @@ attribute: ```python inp.transform # (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0) +``` + +### Metadata + +Images metadata can be retrieved with the `metadata` attribute: + +```python +print(inp.metadata) +``` + +Gives: + +``` +{ + 'DataType': 1.0, + 'DriverLongName': 'GeoTIFF', + 'DriverShortName': 'GTiff', + 'GeoTransform': (760056.0, 6.0, 0.0, 6946092.0, 0.0, -6.0), + 'LowerLeftCorner': (760056.0, 6944268.0), + 'LowerRightCorner': (761562.0, 6944268.0), + 'AREA_OR_POINT': 'Area', + 'TIFFTAG_SOFTWARE': 'CSinG - 13 SEPTEMBRE 2012', + 'ProjectionRef': 'PROJCS["RGF93 v1 / Lambert-93",\n...', + 'ResolutionFactor': 0, + 'SubDatasetIndex': 0, + 'UpperLeftCorner': (760056.0, 6946092.0), + 'UpperRightCorner': (761562.0, 6946092.0), + 'TileHintX': 251.0, + 'TileHintY': 8.0 +} +``` + +## Information + +The information fetched by the `ReadImageInfo` OTB application is available +through `get_info()`: + +```python +print(inp.get_info()) +``` + +Gives: + +```json lines +{ + 'indexx': 0, + 'indexy': 0, + 'sizex': 251, + 'sizey': 304, + 'spacingx': 6.0, + 'spacingy': -6.0, + 'originx': 760059.0, + 'originy': 6946089.0, + 'estimatedgroundspacingx': 5.978403091430664, + 'estimatedgroundspacingy': 5.996793270111084, + 'numberbands': 4, + 'datatype': 'unsigned_char', + 'ullat': 0.0, + 'ullon': 0.0, + 'urlat': 0.0, + 'urlon': 0.0, + 'lrlat': 0.0, + 'lrlon': 0.0, + 'lllat': 0.0, + 'lllon': 0.0, + 'rgb.r': 0, + 'rgb.g': 1, + 'rgb.b': 2, + 'projectionref': 'PROJCS["RGF93 v1 ..."EPSG","2154"]]', + 'gcp.count': 0 +} +``` + +## Statistics + +Image statistics can be computed on-the-fly using `get_statistics()`: + +```python +print(inp.get_statistics()) +``` + +Gives: + +```json lines +{ + 'out.mean': [79.5505, 109.225, 115.456, 249.349], + 'out.min': [33, 64, 91, 47], + 'out.max': [255, 255, 230, 255], + 'out.std': [51.0754, 35.3152, 23.4514, 20.3827] +} ``` \ No newline at end of file -- GitLab From b44e5089ad4a6170725ee07eafa3d11e58368d88 Mon Sep 17 00:00:00 2001 From: Remi Cresson <remi.cresson@inrae.fr> Date: Thu, 9 Nov 2023 14:58:17 +0100 Subject: [PATCH 397/399] DOC: move contribute section in index --- doc/index.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/doc/index.md b/doc/index.md index d76ef33..4ecfaf4 100644 --- a/doc/index.md +++ b/doc/index.md @@ -1,4 +1,4 @@ -# Pyotb: Orfeo Toolbox for Python +# pyotb: Orfeo Toolbox for Python pyotb is a Python extension of Orfeo Toolbox. It has been built on top of the existing Python API of OTB, in order @@ -9,7 +9,7 @@ to make OTB more Python friendly. ## Get started - [Installation](installation.md) -- [How to use pyotb](quickstart.md) +- [Quick start](quickstart.md) - [Useful features](features.md) - [Functions](functions.md) - [Interaction with Python libraries (numpy, rasterio, tensorflow)](interaction.md) @@ -27,8 +27,17 @@ to make OTB more Python friendly. - [Managing loggers](managing_loggers.md) - [Troubleshooting & limitations](troubleshooting.md) - ## API - See the API reference. If you have any doubts or questions, feel free to ask -on github or gitlab! \ No newline at end of file +on github or gitlab! + +## Contribute + +Contributions are welcome ! +Open a PR/MR, or file an issue if you spot a bug or have any suggestion: + +- [Github](https://github.com/orfeotoolbox/pyotb) +- [Orfeo ToolBox GitLab instance](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb). + +Thank you! \ No newline at end of file -- GitLab From 47bc22af9c25ff76ff21df6961f60d5748b648ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr> Date: Thu, 23 Nov 2023 14:58:06 +0000 Subject: [PATCH 398/399] Update RELEASE_NOTES.txt --- RELEASE_NOTES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index 9b1bad1..d7f6094 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -1,5 +1,5 @@ --------------------------------------------------------------------- -2.00 (Oct XX, 2023) - Changes since version 1.5.4 +2.00 (Nov 23, 2023) - Changes since version 1.5.4 - Major refactoring (see troubleshooting/migration) - Pythonic extended filenames (can use dict, etc) -- GitLab From 8ca3c1072ad7373abda257668952e6290d6a9c61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr> Date: Thu, 23 Nov 2023 15:09:34 +0000 Subject: [PATCH 399/399] Update __init__.py --- pyotb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyotb/__init__.py b/pyotb/__init__.py index 5e74f37..5b455ab 100644 --- a/pyotb/__init__.py +++ b/pyotb/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """This module provides convenient python wrapping of otbApplications.""" -__version__ = "2.0.0.dev8" +__version__ = "2.0.0" from .install import install_otb from .helpers import logger, set_logger_level -- GitLab