From 0cd0b356175f619f022cee112cc7835d85c32b0d Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 5 Jan 2023 16:15:53 +0100
Subject: [PATCH 001/399] ENH: refac module init
---
pyotb/__init__.py | 26 +++++++++++++++++----
pyotb/apps.py | 58 ++++++++++++++---------------------------------
2 files changed, 39 insertions(+), 45 deletions(-)
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index 187b64a..e3bc23a 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -1,8 +1,26 @@
# -*- coding: utf-8 -*-
"""This module provides convenient python wrapping of otbApplications."""
-__version__ = "1.5.4"
+__version__ = "1.6.0"
+
+from .helpers import find_otb, logger, set_logger_level
+
+otb = find_otb()
from .apps import *
-from .core import App, Output, Input, get_nbchannels, get_pixel_type
-from .functions import * # pylint: disable=redefined-builtin
-from .helpers import logger, set_logger_level
+
+from .core import (
+ App,
+ Input,
+ Output,
+ get_nbchannels,
+ get_pixel_type
+)
+
+from .functions import ( # pylint: disable=redefined-builtin
+ all,
+ any,
+ where,
+ clip,
+ run_tf_function,
+ define_processing_area
+)
diff --git a/pyotb/apps.py b/pyotb/apps.py
index e5287bb..30e3363 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -4,9 +4,9 @@ import os
import sys
from pathlib import Path
-from .helpers import logger, find_otb
-
-otb = find_otb()
+import otbApplication as otb
+from .core import App
+from .helpers import logger
def get_available_applications(as_subprocess=False):
@@ -55,27 +55,12 @@ def get_available_applications(as_subprocess=False):
if not app_list:
app_list = otb.Registry.GetAvailableApplications()
if not app_list:
- logger.warning("Unable to load applications. Set env variable OTB_APPLICATION_PATH then try again")
- return ()
+ raise SystemExit("Unable to load applications. Set env variable OTB_APPLICATION_PATH and try again.")
logger.info("Successfully loaded %s OTB applications", len(app_list))
return app_list
-AVAILABLE_APPLICATIONS = get_available_applications(as_subprocess=True)
-
-# First core.py call (within __init__ scope)
-from .core import App # pylint: disable=wrong-import-position
-
-# This is to enable aliases of Apps, i.e. using apps like `pyotb.AppName(...)` instead of `pyotb.App("AppName", ...)`
-_CODE_TEMPLATE = """
-class {name}(App):
- """ """
- def __init__(self, *args, **kwargs):
- super().__init__('{name}', *args, **kwargs)
-"""
-
-
class OTBTFApp(App):
"""Helper for OTBTF."""
@staticmethod
@@ -111,30 +96,21 @@ class OTBTFApp(App):
super().__init__(app_name, *args, **kwargs)
+AVAILABLE_APPLICATIONS = get_available_applications(as_subprocess=True)
+
+# This is to enable aliases of Apps, i.e. using apps like `pyotb.AppName(...)` instead of `pyotb.App("AppName", ...)`
+_CODE_TEMPLATE = """
+class {name}(App):
+ """ """
+ def __init__(self, *args, **kwargs):
+ super().__init__('{name}', *args, **kwargs)
+"""
+
for _app in AVAILABLE_APPLICATIONS:
# Customize the behavior for some OTBTF applications. The user doesn't need to set the env variable
# `OTB_TF_NSOURCES`, it is handled in pyotb
- if _app == 'TensorflowModelServe':
- class TensorflowModelServe(OTBTFApp):
- """Serve a Tensorflow model using OTBTF."""
- def __init__(self, *args, n_sources=None, **kwargs):
- """Constructor for a TensorflowModelServe object."""
- super().__init__('TensorflowModelServe', *args, n_sources=n_sources, **kwargs)
-
- elif _app == 'PatchesExtraction':
- class PatchesExtraction(OTBTFApp):
- """Extract patches using OTBTF."""
- def __init__(self, *args, n_sources=None, **kwargs):
- """Constructor for a PatchesExtraction object."""
- super().__init__('PatchesExtraction', *args, n_sources=n_sources, **kwargs)
-
- elif _app == 'TensorflowModelTrain':
- class TensorflowModelTrain(OTBTFApp):
- """Train a Tensorflow model using OTBTF."""
- def __init__(self, *args, n_sources=None, **kwargs):
- """Constructor for a TensorflowModelTrain object."""
- super().__init__('TensorflowModelTrain', *args, n_sources=n_sources, **kwargs)
-
+ if _app in ("PatchesExtraction", "TensorflowModelTrain", "TensorflowModelServe"):
+ exec(_CODE_TEMPLATE.format(name=_app).replace("(App)", "(OTBTFApp)")) # pylint: disable=exec-used
# Default behavior for any OTB application
else:
- exec(_CODE_TEMPLATE.format(name=_app)) # pylint: disable=exec-used
+ exec(_CODE_TEMPLATE.format(name=_app)) # pylint: disable=exec-used
\ No newline at end of file
--
GitLab
From b4208119179e99d12c3c88b390515cee8620cc2d Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 5 Jan 2023 19:18:31 +0100
Subject: [PATCH 002/399] ENH: big refac for 1.6
---
pyotb/core.py | 1163 ++++++++++++++++++++++++-------------------
pyotb/functions.py | 18 +-
tests/test_numpy.py | 2 +-
3 files changed, 660 insertions(+), 523 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 5b9a895..7992890 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -8,11 +8,58 @@ import otbApplication as otb
from .helpers import logger
-class otbObject:
- """Base class that gathers common operations for any OTB in-memory raster."""
+class OTBObject:
+ """Base class that gathers common operations for any OTB application."""
_name = ""
- app = None
- output_param = ""
+
+ def __init__(self, appname, *args, frozen=False, quiet=False, image_dic=None, **kwargs):
+ """Common constructor for OTB applications. Handles in-memory connection between apps.
+
+ Args:
+ appname: name of the app, e.g. 'BandMath'
+ *args: used for passing application parameters. Can be :
+ - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
+ (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
+ - string, App or Output, useful when the user wants to specify the input "in"
+ - list, useful when the user wants to specify the input list 'il'
+ frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___
+ quiet: whether to print logs of the OTB app
+ **kwargs: used for passing application parameters.
+ e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif'
+
+ """
+ self.parameters = {}
+ self.appname = appname
+ self.quiet = quiet
+ self.image_dic = image_dic
+ if quiet:
+ self.app = otb.Registry.CreateApplicationWithoutLogger(appname)
+ else:
+ self.app = otb.Registry.CreateApplication(appname)
+ self.parameters_keys = tuple(self.app.GetParametersKeys())
+ self.out_param_types = dict(get_out_param_types(self))
+ self.out_param_keys = tuple(self.out_param_types.keys())
+ self.exports_dict = {}
+ if args or kwargs:
+ self.set_parameters(*args, **kwargs)
+ self.frozen = frozen
+ if not frozen:
+ self.execute()
+
+ @property
+ def key_input(self):
+ """Get the name of first input parameter, raster > vector > file."""
+ return self.key_input_image or key_input(self, "vector") or key_input(self, "file")
+
+ @property
+ def key_input_image(self):
+ """Get the name of first output image parameter"""
+ return key_input(self, "raster")
+
+ @property
+ def key_output_image(self):
+ """Get the name of first output image parameter"""
+ return key_output(self, "raster")
@property
def name(self):
@@ -22,17 +69,25 @@ class otbObject:
user's defined name or appname
"""
- return self._name or self.app.GetName()
+ return self._name or self.appname
@name.setter
- def name(self, val):
+ def name(self, name):
"""Set custom name.
Args:
- val: new name
+ name: new name
"""
- self._name = val
+ if isinstance(name, str):
+ self._name = name
+ else:
+ raise TypeError(f"{self.name}: bad type ({type(name)}) for application name, only str is allowed")
+
+ @property
+ def outputs(self):
+ """List of application outputs."""
+ return [getattr(self, key) for key in self.out_param_keys if key in self.parameters]
@property
def dtype(self):
@@ -43,7 +98,7 @@ class otbObject:
"""
try:
- enum = self.app.GetParameterOutputImagePixelType(self.output_param)
+ enum = self.app.GetParameterOutputImagePixelType(self.key_output_image)
return self.app.ConvertPixelTypeToNumpy(enum)
except RuntimeError:
return None
@@ -56,12 +111,114 @@ class otbObject:
shape: (height, width, bands)
"""
- width, height = self.app.GetImageSize(self.output_param)
- bands = self.app.GetImageNbBands(self.output_param)
+ width, height = self.app.GetImageSize(self.key_output_image)
+ bands = self.app.GetImageNbBands(self.key_output_image)
return (height, width, bands)
- def write(self, *args, filename_extension="", pixel_type=None, **kwargs):
- """Trigger execution, set output pixel type and write the output.
+ @property
+ def transform(self):
+ """Get image affine transform, rasterio style (see https://www.perrygeo.com/python-affine-transforms.html)
+
+ Returns:
+ transform: (X spacing, X offset, X origin, Y offset, Y spacing, Y origin)
+ """
+ spacing_x, spacing_y = self.app.GetImageSpacing(self.key_output_image)
+ origin_x, origin_y = self.app.GetImageOrigin(self.key_output_image)
+ # Shift image origin since OTB is giving coordinates of pixel center instead of corners
+ origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2
+ return (spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y)
+
+ def set_parameters(self, *args, **kwargs):
+ """Set some parameters of the app.
+
+ When useful, e.g. for images list, this function appends the parameters
+ instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths
+
+ Args:
+ *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
+ (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
+ - string, App or Output, useful when the user implicitly wants to set the param "in"
+ - list, useful when the user implicitly wants to set the param "il"
+ **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif'
+
+ Raises:
+ Exception: when the setting of a parameter failed
+
+ """
+ parameters = kwargs
+ parameters.update(self.__parse_args(args))
+ # Going through all arguments
+ for key, obj in parameters.items():
+ if key not in self.parameters_keys:
+ raise KeyError(f'{self.name}: unknown parameter name "{key}"')
+ # When the parameter expects a list, if needed, change the value to list
+ if is_key_list(self, key) and not isinstance(obj, (list, tuple)):
+ obj = [obj]
+ logger.info('%s: argument for parameter "%s" was converted to list', self.name, key)
+ try:
+ # This is when we actually call self.app.SetParameter*
+ self.__set_param(key, obj)
+ except (RuntimeError, TypeError, ValueError, KeyError) as e:
+ raise Exception(
+ f"{self.name}: something went wrong before execution "
+ f"(while setting parameter '{key}' to '{obj}')"
+ ) from e
+ # Update _parameters using values from OtbApplication object
+ otb_params = self.app.GetParameters().items()
+ otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params}
+ # Save parameters keys, and values as object attributes
+ self.parameters.update({**parameters, **otb_params})
+
+ def execute(self):
+ """Execute and write to disk if any output parameter has been set during init."""
+ logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters)
+ try:
+ self.app.Execute()
+ except (RuntimeError, FileNotFoundError) as e:
+ raise Exception(f"{self.name}: error during during app execution") from e
+ self.frozen = False
+ logger.debug("%s: execution ended", self.name)
+ if any(key in self.parameters for key in self.out_param_keys):
+ self.flush()
+ self.save_objects()
+
+ def flush(self):
+ """Flush data to disk, this is when WriteOutput is actually called.
+
+ Args:
+ parameters: key value pairs like {parameter: filename}
+ dtypes: optional dict to pass output data types (for rasters)
+
+ """
+ try:
+ logger.debug("%s: flushing data to disk", self.name)
+ self.app.WriteOutput()
+ except RuntimeError:
+ logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name)
+ self.app.ExecuteAndWriteOutput()
+
+ def save_objects(self):
+ """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`.
+
+ This is useful when the key contains reserved characters such as a point eg "io.out"
+ """
+ for key in self.parameters_keys:
+ if key in dir(OTBObject):
+ continue # skip forbidden attribute since it is already used by the class
+ value = self.parameters.get(key) # basic parameters
+ if value is None:
+ try:
+ value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results)
+ except RuntimeError:
+ continue # this is when there is no value for key
+ # Convert output param path to Output object
+ if key in self.out_param_keys:
+ value = Output(self, key, value)
+ # Save attribute
+ setattr(self, key, value)
+
+ def write(self, *args, filename_extension="", pixel_type=None, preserve_dtype=False, **kwargs):
+ """Set output pixel type and write the output raster files.
Args:
*args: Can be : - dictionary containing key-arguments enumeration. Useful when a key contains
@@ -75,7 +232,9 @@ class otbObject:
outputs, all outputs are written with this unique type.
Valid pixel types are uint8, uint16, uint32, int16, int32, float, double,
cint16, cint32, cfloat, cdouble. (Default value = None)
+ preserve_dtype: propagate main input pixel type to outputs, in case pixel_type is None
**kwargs: keyword arguments e.g. out='output.tif'
+
"""
# Gather all input arguments in kwargs dict
for arg in args:
@@ -83,73 +242,151 @@ class otbObject:
kwargs.update(arg)
elif isinstance(arg, str) and kwargs:
logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg)
- elif isinstance(arg, str):
- kwargs.update({self.output_param: arg})
-
- dtypes = {}
- if isinstance(pixel_type, dict):
- dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()}
- elif pixel_type is not None:
- typ = parse_pixel_type(pixel_type)
- if isinstance(self, App):
- dtypes = {key: typ for key in self.output_parameters_keys}
- elif isinstance(self, otbObject):
- dtypes = {self.output_param: typ}
-
+ elif isinstance(arg, str) and self.key_output_image:
+ kwargs.update({self.key_output_image: arg})
+ # Append filename extension to filenames
if filename_extension:
- logger.debug('%s: using extended filename for outputs: %s', self.name, filename_extension)
- if not filename_extension.startswith('?'):
+ logger.debug("%s: using extended filename for outputs: %s", self.name, filename_extension)
+ if not filename_extension.startswith("?"):
filename_extension = "?" + filename_extension
+ for key, value in kwargs.items():
+ if self.out_param_types[key] == 'raster' and '?' not in value:
+ kwargs[key] = value + filename_extension
+ # Manage output pixel types
+ dtypes = {}
+ if pixel_type:
+ if isinstance(pixel_type, str):
+ type_name = self.app.ConvertPixelTypeToNumpy(parse_pixel_type(pixel_type))
+ logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name)
+ for key in kwargs:
+ if self.out_param_types.get(key) == "raster":
+ dtypes[key] = parse_pixel_type(pixel_type)
+ elif isinstance(pixel_type, dict):
+ dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()}
+ elif preserve_dtype:
+ self.propagate_dtype() # all outputs will have the same type as the main input raster
+ # Apply parameters
+ for key, output_filename in kwargs.items():
+ if key in dtypes:
+ self.propagate_dtype(key, dtypes[key])
+ self.set_parameters({key: output_filename})
- # Case output parameter was set during App init
- if not kwargs:
- if self.output_param in self.parameters:
- if dtypes:
- self.app.SetParameterOutputImagePixelType(self.output_param, dtypes[self.output_param])
- if filename_extension:
- new_val = self.parameters[self.output_param] + filename_extension
- self.app.SetParameterString(self.output_param, new_val)
- else:
- raise ValueError(f'{self.app.GetName()}: Output parameter is missing.')
+ self.flush()
+ self.save_objects()
- # Parse kwargs
- for key, output_filename in kwargs.items():
- # Stop process if a bad parameter is given
- if key not in self.app.GetParametersKeys():
- raise KeyError(f'{self.app.GetName()}: Unknown parameter key "{key}"')
- # Check if extended filename was not provided twice
- if '?' in output_filename and filename_extension:
- logger.warning('%s: extended filename was provided twice. Using the one found in path.', self.name)
- elif filename_extension:
- output_filename += filename_extension
-
- logger.debug('%s: "%s" parameter is %s', self.name, key, output_filename)
- self.app.SetParameterString(key, output_filename)
+ def propagate_dtype(self, target_key=None, dtype=None):
+ """Propagate a pixel type from main input to every outputs, or to a target output key only.
- if key in dtypes:
- self.app.SetParameterOutputImagePixelType(key, dtypes[key])
+ With multiple inputs (if dtype is not provided), the type of the first input is considered.
+ With multiple outputs (if target_key is not provided), all outputs will be converted to the same pixel type.
- logger.debug('%s: flushing data to disk', self.name)
- try:
- self.app.WriteOutput()
- except RuntimeError:
- logger.debug('%s: failed to simply write output, executing once again then writing', self.name)
- self.app.ExecuteAndWriteOutput()
+ Args:
+ target_key: output param key to change pixel type
+ dtype: data type to use
+
+ """
+ if not dtype:
+ param = self.parameters.get(self.key_input_image)
+ if not param:
+ logger.warning("%s: could not propagate pixel type from inputs to output", self.name)
+ return
+ if isinstance(param, (list, tuple)):
+ param = param[0] # first image in "il"
+ try:
+ dtype = get_pixel_type(param)
+ except (TypeError, RuntimeError):
+ logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param)
+ return
+
+ if target_key:
+ keys = [target_key]
+ else:
+ keys = [k for k in self.out_param_keys if self.out_param_types[k] == "raster"]
+ for key in keys:
+ # Set output pixel type
+ self.app.SetParameterOutputImagePixelType(key, dtype)
- def to_numpy(self, preserve_dtype=True, copy=False):
+ def read_values_at_coords(self, col, row, bands=None):
+ """Get pixel value(s) at a given YX coordinates.
+
+ Args:
+ col: index along X / longitude axis
+ row: index along Y / latitude axis
+ bands: band number, list or slice to fetch values from
+
+ Returns:
+ single numerical value or a list of values for each band
+
+ """
+ channels = []
+ app = OTBObject("PixelValue", self, coordx=col, coordy=row, frozen=False, quiet=True)
+ if bands is not None:
+ if isinstance(bands, int):
+ if bands < 0:
+ bands = self.shape[2] + bands
+ channels = [bands]
+ elif isinstance(bands, slice):
+ channels = self.__channels_list_from_slice(bands)
+ elif not isinstance(bands, list):
+ raise TypeError(f"{self.name}: type '{bands}' cannot be interpreted as a valid slicing")
+ if channels:
+ app.app.Execute()
+ app.set_parameters({"cl": [f"Channel{n+1}" for n in channels]})
+ app.execute()
+ data = literal_eval(app.app.GetParameterString("value"))
+ if len(channels) == 1:
+ return data[0]
+ return data
+
+ def summarize(self):
+ """Serialize an object and its pipeline into a dictionary.
+
+ Returns:
+ nested dictionary summarizing the pipeline
+
+ """
+ params = self.parameters
+ for k, p in params.items():
+ # In the following, we replace each parameter which is an OTBObject, with its summary.
+ if isinstance(p, OTBObject): # single parameter
+ params[k] = p.summarize()
+ elif isinstance(p, list): # parameter list
+ params[k] = [pi.summarize() if isinstance(pi, OTBObject) else pi for pi in p]
+
+ return {"name": self.name, "parameters": params}
+
+ def export(self, key=None):
+ """Export a specific output image as numpy array and store it in object's exports_dict.
+
+ Args:
+ key: parameter key to export, if None then the default one will be used
+
+ Returns:
+ the exported numpy array
+
+ """
+ if key is None:
+ key = key_output(self, "raster")
+ if key not in self.exports_dict:
+ self.exports_dict[key] = self.app.ExportImage(key)
+ return self.exports_dict[key]
+
+ def to_numpy(self, key=None, preserve_dtype=True, copy=False):
"""Export a pyotb object to numpy array.
Args:
+ key: the output parameter name to export as numpy array
preserve_dtype: when set to True, the numpy array is created with the same pixel type as
- the otbObject first output. Default is True.
+ the OTBObject first output. Default is True.
copy: whether to copy the output array, default is False
required to True if preserve_dtype is False and the source app reference is lost
Returns:
- a numpy array
+ a numpy array
"""
- array = self.app.ExportImage(self.output_param)['array']
+ data = self.export(key)
+ array = data["array"]
if preserve_dtype:
return array.astype(self.dtype)
if copy:
@@ -166,51 +403,117 @@ class otbObject:
"""
array = self.to_numpy(preserve_dtype=True, copy=False)
array = np.moveaxis(array, 2, 0)
- proj = self.app.GetImageProjection(self.output_param)
- spacing_x, spacing_y = self.app.GetImageSpacing(self.output_param)
- origin_x, origin_y = self.app.GetImageOrigin(self.output_param)
- # Shift image origin since OTB is giving coordinates of pixel center instead of corners
- origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2
+ proj = self.app.GetImageProjection(self.key_output_image)
profile = {
'crs': proj, 'dtype': array.dtype,
'count': array.shape[0], 'height': array.shape[1], 'width': array.shape[2],
- 'transform': (spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y) # here we force pixel rotation to 0 !
+ 'transform': self.transform
}
return array, profile
- # Special methods
- def __getitem__(self, key):
- """Override the default __getitem__ behaviour.
+ def xy_to_rowcol(self, x, y):
+ """Find (row, col) index using (x, y) projected coordinates, expect image CRS
- This function enables 2 things :
- - access attributes like that : object['any_attribute']
- - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3]
- selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]]
- selecting 1000x1000 subset : object[:1000, :1000]
+ Args:
+ x: longitude or projected X
+ y: latitude or projected Y
+
+ Returns:
+ pixel index: (row, col)
+ """
+ spacing_x, _, origin_x, _, spacing_y, origin_y = self.transform
+ col = int((x - origin_x) / spacing_x)
+ row = int((origin_y - y) / spacing_y)
+ return (row, col)
+
+ # Private functions
+ def __parse_args(self, args):
+ """Gather all input arguments in kwargs dict.
Args:
- key: attribute key
+ args: the list of arguments passed to set_parameters()
Returns:
- attribute or Slicer
+ a dictionary with the right keyword depending on the object
+
"""
- # Accessing string attributes
- if isinstance(key, str):
- return self.__dict__.get(key)
- # Slicing
- if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)):
- raise ValueError(f'"{key}"cannot be interpreted as valid slicing. Slicing should be 2D or 3D.')
- if isinstance(key, tuple) and len(key) == 2:
- # Adding a 3rd dimension
- key = key + (slice(None, None, None),)
- (rows, cols, channels) = key
- return Slicer(self, rows, cols, channels)
+ kwargs = {}
+ for arg in args:
+ if isinstance(arg, dict):
+ kwargs.update(arg)
+ elif isinstance(arg, (str, OTBObject)):
+ kwargs.update({self.key_input: arg})
+ elif isinstance(arg, list) and is_key_list(self, self.key_input):
+ kwargs.update({self.key_input: arg})
+ return kwargs
+
+ def __set_param(self, key, obj):
+ """Set one parameter, decide which otb.Application method to use depending on target object."""
+ if obj is None or (isinstance(obj, (list, tuple)) and not obj):
+ self.app.ClearValue(key)
+ return
+ if key not in self.parameters_keys:
+ raise Exception(
+ f"{self.name}: parameter '{key}' was not recognized. " f"Available keys are {self.parameters_keys}"
+ )
+ # Single-parameter cases
+ if isinstance(obj, OTBObject):
+ self.app.ConnectImage(key, obj.app, obj.key_output_image)
+ elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB
+ self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0])
+ elif key == "ram": # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200
+ self.app.SetParameterInt("ram", int(obj))
+ elif not isinstance(obj, list): # any other parameters (str, int...)
+ self.app.SetParameterValue(key, obj)
+ # Images list
+ elif is_key_images_list(self, key):
+ # To enable possible in-memory connections, we go through the list and set the parameters one by one
+ for inp in obj:
+ if isinstance(inp, OTBObject):
+ self.app.ConnectImage(key, inp.app, inp.key_output_image)
+ elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB
+ self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0])
+ else: # here `input` should be an image filepath
+ # Append `input` to the list, do not overwrite any previously set element of the image list
+ self.app.AddParameterStringList(key, inp)
+ # List of any other types (str, int...)
+ else:
+ self.app.SetParameterValue(key, obj)
+
+ def __channels_list_from_slice(self, bands):
+ """Get list of channels to read values at, from a slice."""
+ channels = None
+ start, stop, step = bands.start, bands.stop, bands.step
+ if step is None:
+ step = 1
+ if start is not None and stop is not None:
+ channels = list(range(start, stop, step))
+ elif start is not None and stop is None:
+ channels = list(range(start, self.shape[2], step))
+ elif start is None and stop is not None:
+ channels = list(range(0, stop, step))
+ elif start is None and stop is None:
+ channels = list(range(0, self.shape[2], step))
+ return channels
+
+ def __hash__(self):
+ """Override the default behaviour of the hash function.
+
+ Returns:
+ self hash
+
+ """
+ return id(self)
+
+ def __str__(self):
+ """Return a nice string representation with object id."""
+ return f"<pyotb.App {self.appname} object id {id(self)}>"
def __getattr__(self, name):
"""This method is called when the default attribute access fails.
We choose to access the attribute `name` of self.app.
- Thus, any method of otbApplication can be used transparently on otbObject objects,
+ Thus, any method of otbApplication can be used transparently on OTBObject objects,
e.g. SetParameterOutputImagePixelType() or ExportImage() work
Args:
@@ -227,7 +530,45 @@ class otbObject:
res = getattr(self.app, name)
return res
except AttributeError as e:
- raise AttributeError(f'{self.name}: could not find attribute `{name}`') from e
+ raise AttributeError(f"{self.name}: could not find attribute `{name}`") from e
+
+ def __getitem__(self, key):
+ """Override the default __getitem__ behaviour.
+
+ This function enables 2 things :
+ - access attributes like that : object['any_attribute']
+ - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3]
+ selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]]
+ selecting 1000x1000 subset : object[:1000, :1000]
+ - access pixel value(s) at a specified row, col index
+
+ Args:
+ key: attribute key
+
+ Returns:
+ attribute, pixel values or Slicer
+
+ """
+ # Accessing string attributes
+ if isinstance(key, str):
+ return self.__dict__.get(key)
+ # Accessing pixel value(s) using Y/X coordinates
+ if isinstance(key, tuple) and len(key) >= 2:
+ row, col = key[0], key[1]
+ if isinstance(row, int) and isinstance(col, int):
+ if row < 0 or col < 0:
+ raise ValueError(f"{self.name}: can't read pixel value at negative coordinates ({row}, {col})")
+ channels = None
+ if len(key) == 3:
+ channels = key[2]
+ return self.read_values_at_coords(row, col, channels)
+ # Slicing
+ if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)):
+ raise ValueError(f'"{key}"cannot be interpreted as valid slicing. Slicing should be 2D or 3D.')
+ if isinstance(key, tuple) and len(key) == 2:
+ # Adding a 3rd dimension
+ key = key + (slice(None, None, None),)
+ return Slicer(self, *key)
def __add__(self, other):
"""Overrides the default addition and flavours it with BandMathX.
@@ -236,12 +577,12 @@ class otbObject:
other: the other member of the operation
Returns:
- self + other
+ self + other
"""
if isinstance(other, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return Operation('+', self, other)
+ return Operation("+", self, other)
def __sub__(self, other):
"""Overrides the default subtraction and flavours it with BandMathX.
@@ -250,12 +591,12 @@ class otbObject:
other: the other member of the operation
Returns:
- self - other
+ self - other
"""
if isinstance(other, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return Operation('-', self, other)
+ return Operation("-", self, other)
def __mul__(self, other):
"""Overrides the default subtraction and flavours it with BandMathX.
@@ -264,12 +605,12 @@ class otbObject:
other: the other member of the operation
Returns:
- self * other
+ self * other
"""
if isinstance(other, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return Operation('*', self, other)
+ return Operation("*", self, other)
def __truediv__(self, other):
"""Overrides the default subtraction and flavours it with BandMathX.
@@ -278,12 +619,12 @@ class otbObject:
other: the other member of the operation
Returns:
- self / other
+ self / other
"""
if isinstance(other, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return Operation('/', self, other)
+ return Operation("/", self, other)
def __radd__(self, other):
"""Overrides the default reverse addition and flavours it with BandMathX.
@@ -292,12 +633,12 @@ class otbObject:
other: the other member of the operation
Returns:
- other + self
+ other + self
"""
if isinstance(other, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return Operation('+', other, self)
+ return Operation("+", other, self)
def __rsub__(self, other):
"""Overrides the default subtraction and flavours it with BandMathX.
@@ -306,12 +647,12 @@ class otbObject:
other: the other member of the operation
Returns:
- other - self
+ other - self
"""
if isinstance(other, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return Operation('-', other, self)
+ return Operation("-", other, self)
def __rmul__(self, other):
"""Overrides the default multiplication and flavours it with BandMathX.
@@ -320,12 +661,12 @@ class otbObject:
other: the other member of the operation
Returns:
- other * self
+ other * self
"""
if isinstance(other, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return Operation('*', other, self)
+ return Operation("*", other, self)
def __rtruediv__(self, other):
"""Overrides the default division and flavours it with BandMathX.
@@ -334,12 +675,12 @@ class otbObject:
other: the other member of the operation
Returns:
- other / self
+ other / self
"""
if isinstance(other, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return Operation('/', other, self)
+ return Operation("/", other, self)
def __abs__(self):
"""Overrides the default abs operator and flavours it with BandMathX.
@@ -348,7 +689,7 @@ class otbObject:
abs(self)
"""
- return Operation('abs', self)
+ return Operation("abs", self)
def __ge__(self, other):
"""Overrides the default greater or equal and flavours it with BandMathX.
@@ -357,12 +698,12 @@ class otbObject:
other: the other member of the operation
Returns:
- self >= other
+ self >= other
"""
if isinstance(other, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return logicalOperation('>=', self, other)
+ return LogicalOperation(">=", self, other)
def __le__(self, other):
"""Overrides the default less or equal and flavours it with BandMathX.
@@ -371,12 +712,12 @@ class otbObject:
other: the other member of the operation
Returns:
- self <= other
+ self <= other
"""
if isinstance(other, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return logicalOperation('<=', self, other)
+ return LogicalOperation("<=", self, other)
def __gt__(self, other):
"""Overrides the default greater operator and flavours it with BandMathX.
@@ -385,12 +726,12 @@ class otbObject:
other: the other member of the operation
Returns:
- self > other
+ self > other
"""
if isinstance(other, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return logicalOperation('>', self, other)
+ return LogicalOperation(">", self, other)
def __lt__(self, other):
"""Overrides the default less operator and flavours it with BandMathX.
@@ -399,12 +740,12 @@ class otbObject:
other: the other member of the operation
Returns:
- self < other
+ self < other
"""
if isinstance(other, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return logicalOperation('<', self, other)
+ return LogicalOperation("<", self, other)
def __eq__(self, other):
"""Overrides the default eq operator and flavours it with BandMathX.
@@ -413,12 +754,12 @@ class otbObject:
other: the other member of the operation
Returns:
- self == other
+ self == other
"""
if isinstance(other, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return logicalOperation('==', self, other)
+ return LogicalOperation("==", self, other)
def __ne__(self, other):
"""Overrides the default different operator and flavours it with BandMathX.
@@ -427,12 +768,12 @@ class otbObject:
other: the other member of the operation
Returns:
- self != other
+ self != other
"""
if isinstance(other, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return logicalOperation('!=', self, other)
+ return LogicalOperation("!=", self, other)
def __or__(self, other):
"""Overrides the default or operator and flavours it with BandMathX.
@@ -441,12 +782,12 @@ class otbObject:
other: the other member of the operation
Returns:
- self || other
+ self || other
"""
if isinstance(other, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return logicalOperation('||', self, other)
+ return LogicalOperation("||", self, other)
def __and__(self, other):
"""Overrides the default and operator and flavours it with BandMathX.
@@ -455,25 +796,16 @@ class otbObject:
other: the other member of the operation
Returns:
- self && other
+ self && other
"""
if isinstance(other, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return logicalOperation('&&', self, other)
+ return LogicalOperation("&&", self, other)
# TODO: other operations ?
# e.g. __pow__... cf https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types
- def __hash__(self):
- """Override the default behaviour of the hash function.
-
- Returns:
- self hash
-
- """
- return id(self)
-
def __array__(self):
"""This is called when running np.asarray(pyotb_object).
@@ -499,158 +831,43 @@ class otbObject:
a pyotb object
"""
- if method == '__call__':
+ if method == "__call__":
# Converting potential pyotb inputs to arrays
arrays = []
image_dic = None
for inp in inputs:
if isinstance(inp, (float, int, np.ndarray, np.generic)):
arrays.append(inp)
- elif isinstance(inp, otbObject):
- image_dic = inp.app.ExportImage(self.output_param)
- array = image_dic['array']
+ elif isinstance(inp, OTBObject):
+ if not inp.exports_dict:
+ inp.export()
+ image_dic = inp.exports_dict[inp.key_output_image]
+ array = image_dic["array"]
arrays.append(array)
else:
- print(type(self))
+ logger.debug(type(self))
return NotImplemented
# Performing the numpy operation
result_array = ufunc(*arrays, **kwargs)
result_dic = image_dic
- result_dic['array'] = result_array
+ result_dic["array"] = result_array
- # Importing back to OTB
- app = App('ExtractROI', frozen=True, image_dic=result_dic) # pass the result_dic just to keep reference
+ # Importing back to OTB, pass the result_dic just to keep reference
+ app = OTBObject("ExtractROI", image_dic=result_dic, frozen=True, quiet=True)
if result_array.shape[2] == 1:
- app.ImportImage('in', result_dic)
+ app.ImportImage("in", result_dic)
else:
- app.ImportVectorImage('in', result_dic)
+ app.ImportVectorImage("in", result_dic)
app.execute()
return app
return NotImplemented
- def summarize(self):
- """Return a nested dictionary summarizing the otbObject.
- Returns:
- Nested dictionary summarizing the otbObject
+class App(OTBObject):
- """
- params = self.parameters
- for k, p in params.items():
- # In the following, we replace each parameter which is an otbObject, with its summary.
- if isinstance(p, otbObject): # single parameter
- params[k] = p.summarize()
- elif isinstance(p, list): # parameter list
- params[k] = [pi.summarize() if isinstance(pi, otbObject) else pi for pi in p]
-
- return {"name": self.name, "parameters": params}
-
-
-class App(otbObject):
- """Class of an OTB app."""
- def __init__(self, appname, *args, frozen=False, quiet=False,
- preserve_dtype=False, image_dic=None, **kwargs):
- """Enables to init an OTB application as a oneliner. Handles in-memory connection between apps.
-
- Args:
- appname: name of the app, e.g. 'Smoothing'
- *args: used for passing application parameters. Can be :
- - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
- (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
- - string, App or Output, useful when the user wants to specify the input "in"
- - list, useful when the user wants to specify the input list 'il'
- frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___
- quiet: whether to print logs of the OTB app
- preserve_dtype: propagate the pixel type from inputs to output. If several inputs, the type of an
- arbitrary input is considered. If several outputs, all will have the same type.
- image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as
- the result of app.ExportImage(). Use it when the app takes a numpy array as input.
- See this related issue for why it is necessary to keep reference of object:
- https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824
- **kwargs: used for passing application parameters.
- e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif'
-
- """
- self.appname = appname
- self.frozen = frozen
- self.quiet = quiet
- self.preserve_dtype = preserve_dtype
- self.image_dic = image_dic
- if self.quiet:
- self.app = otb.Registry.CreateApplicationWithoutLogger(appname)
- else:
- self.app = otb.Registry.CreateApplication(appname)
- self.description = self.app.GetDocLongDescription()
- self.output_parameters_keys = self.__get_output_parameters_keys()
- if self.output_parameters_keys:
- self.output_param = self.output_parameters_keys[0]
-
- self.parameters = {}
- if (args or kwargs):
- self.set_parameters(*args, **kwargs)
- if not self.frozen:
- self.execute()
-
- def set_parameters(self, *args, **kwargs):
- """Set some parameters of the app.
-
- When useful, e.g. for images list, this function appends the parameters
- instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths
-
- Args:
- *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
- (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
- - string, App or Output, useful when the user implicitly wants to set the param "in"
- - list, useful when the user implicitly wants to set the param "il"
- **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif'
-
- Raises:
- Exception: when the setting of a parameter failed
-
- """
- parameters = kwargs
- parameters.update(self.__parse_args(args))
- # Going through all arguments
- for param, obj in parameters.items():
- if param not in self.app.GetParametersKeys():
- raise Exception(f"{self.name}: parameter '{param}' was not recognized. "
- f"Available keys are {self.app.GetParametersKeys()}")
- # When the parameter expects a list, if needed, change the value to list
- if self.__is_key_list(param) and not isinstance(obj, (list, tuple)):
- parameters[param] = [obj]
- obj = [obj]
- logger.warning('%s: argument for parameter "%s" was converted to list', self.name, param)
- try:
- # This is when we actually call self.app.SetParameter*
- self.__set_param(param, obj)
- except (RuntimeError, TypeError, ValueError, KeyError) as e:
- raise Exception(f"{self.name}: something went wrong before execution "
- f"(while setting parameter '{param}' to '{obj}')") from e
- # Update _parameters using values from OtbApplication object
- otb_params = self.app.GetParameters().items()
- otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params}
- self.parameters.update({**parameters, **otb_params})
- # Update output images pixel types
- if self.preserve_dtype:
- self.__propagate_pixel_type()
-
- def execute(self):
- """Execute and write to disk if any output parameter has been set during init."""
- logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters)
- try:
- self.app.Execute()
- except (RuntimeError, FileNotFoundError) as e:
- raise Exception(f'{self.name}: error during during app execution') from e
- self.frozen = False
- logger.debug("%s: execution ended", self.name)
- if self.__has_output_param_key():
- logger.debug('%s: flushing data to disk', self.name)
- self.app.WriteOutput()
- self.__save_objects()
-
- def find_output(self):
+ def find_outputs(self):
"""Find output files on disk using path found in parameters.
Returns:
@@ -659,8 +876,7 @@ class App(otbObject):
"""
files = []
missing = []
- outputs = [p for p in self.output_parameters_keys if p in self.parameters]
- for param in outputs:
+ for param in self.outputs:
filename = self.parameters[param]
# Remove filename extension
if '?' in filename:
@@ -677,144 +893,11 @@ class App(otbObject):
return files
- # Private functions
- def __get_output_parameters_keys(self):
- """Get raster output parameter keys.
-
- Returns:
- output parameters keys
- """
- return [param for param in self.app.GetParametersKeys()
- if self.app.GetParameterType(param) == otb.ParameterType_OutputImage]
- def __has_output_param_key(self):
- """Check if App has any output parameter key."""
- if not self.output_param:
- return True # apps like ReadImageInfo with no filetype output param still needs to WriteOutput
- types = (otb.ParameterType_OutputFilename, otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData)
- outfile_params = [param for param in self.app.GetParametersKeys() if self.app.GetParameterType(param) in types]
- return any(key in self.parameters for key in outfile_params)
-
- @staticmethod
- def __parse_args(args):
- """Gather all input arguments in kwargs dict.
-
- Returns:
- a dictionary with the right keyword depending on the object
-
- """
- kwargs = {}
- for arg in args:
- if isinstance(arg, dict):
- kwargs.update(arg)
- elif isinstance(arg, (str, otbObject)):
- kwargs.update({'in': arg})
- elif isinstance(arg, list):
- kwargs.update({'il': arg})
- return kwargs
-
- def __set_param(self, param, obj):
- """Set one parameter, decide which otb.Application method to use depending on target object."""
- if obj is not None:
- # Single-parameter cases
- if isinstance(obj, otbObject):
- self.app.ConnectImage(param, obj.app, obj.output_param)
- elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB
- outparamkey = [param for param in obj.GetParametersKeys()
- if obj.GetParameterType(param) == otb.ParameterType_OutputImage][0]
- self.app.ConnectImage(param, obj, outparamkey)
- elif param == 'ram': # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200
- self.app.SetParameterInt('ram', int(obj))
- elif not isinstance(obj, list): # any other parameters (str, int...)
- self.app.SetParameterValue(param, obj)
- # Images list
- elif self.__is_key_images_list(param):
- # To enable possible in-memory connections, we go through the list and set the parameters one by one
- for inp in obj:
- if isinstance(inp, otbObject):
- self.app.ConnectImage(param, inp.app, inp.output_param)
- elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB
- outparamkey = [param for param in inp.GetParametersKeys() if
- inp.GetParameterType(param) == otb.ParameterType_OutputImage][0]
- self.app.ConnectImage(param, inp, outparamkey)
- else: # here `input` should be an image filepath
- # Append `input` to the list, do not overwrite any previously set element of the image list
- self.app.AddParameterStringList(param, inp)
- # List of any other types (str, int...)
- else:
- self.app.SetParameterValue(param, obj)
-
- def __propagate_pixel_type(self):
- """Propagate the pixel type from inputs to output.
-
- For several inputs, or with an image list, the type of the first input is considered.
- If several outputs, all outputs will have the same type.
-
- """
- pixel_type = None
- for key, param in self.parameters.items():
- if self.__is_key_input_image(key):
- if not param:
- continue
- if isinstance(param, list):
- param = param[0] # first image in "il"
- try:
- pixel_type = get_pixel_type(param)
- type_name = self.app.ConvertPixelTypeToNumpy(pixel_type)
- logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name)
- for out_key in self.output_parameters_keys:
- self.app.SetParameterOutputImagePixelType(out_key, pixel_type)
- return
- except TypeError:
- pass
-
- logger.warning("%s: could not propagate pixel type from inputs to output, no valid input found", self.name)
-
- def __save_objects(self):
- """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`.
-
- This is useful when the key contains reserved characters such as a point eg "io.out"
- """
- for key in self.app.GetParametersKeys():
- if key == 'parameters': # skip forbidden attribute since it is already used by the App class
- continue
- value = None
- if key in self.output_parameters_keys: # raster outputs
- value = Output(self, key)
- elif key in self.parameters: # user or default app parameters
- value = self.parameters[key]
- else: # any other app attribute (e.g. ReadImageInfo results)
- try:
- value = self.app.GetParameterValue(key)
- except RuntimeError:
- pass # this is when there is no value for key
- if value is not None:
- setattr(self, key, value)
-
- def __is_key_input_image(self, key):
- """Check if a key of the App is an input parameter image list."""
- return self.app.GetParameterType(key) in (otb.ParameterType_InputImage, otb.ParameterType_InputImageList)
-
- def __is_key_list(self, key):
- """Check if a key of the App is an input parameter list."""
- return self.app.GetParameterType(key) in (otb.ParameterType_InputImageList, otb.ParameterType_StringList,
- otb.ParameterType_InputFilenameList, otb.ParameterType_ListView,
- otb.ParameterType_InputVectorDataList)
-
- def __is_key_images_list(self, key):
- """Check if a key of the App is an input parameter image list."""
- return self.app.GetParameterType(key) in (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList)
-
- # Special methods
- def __str__(self):
- """Return a nice string representation with object id."""
- return f'<pyotb.App {self.appname} object id {id(self)}>'
-
-
-class Slicer(App):
+class Slicer(OTBObject):
"""Slicer objects i.e. when we call something like raster[:, :, 2] from Python."""
- def __init__(self, x, rows, cols, channels):
+ def __init__(self, obj, rows, cols, channels):
"""Create a slicer object, that can be used directly for writing or inside a BandMath.
It contains :
@@ -822,22 +905,20 @@ class Slicer(App):
- in case the user only wants to extract one band, an expression such as "im1b#"
Args:
- x: input
+ obj: input
rows: slice along Y / Latitude axis
cols: slice along X / Longitude axis
channels: channels, can be slicing, list or int
"""
- # Initialize the app that will be used for writing the slicer
- self.name = 'Slicer'
-
- self.output_parameter_key = 'out'
- parameters = {'in': x, 'mode': 'extent'}
- super().__init__('ExtractROI', parameters, preserve_dtype=True, frozen=True)
+ super().__init__("ExtractROI", {"in": obj, "mode": "extent"}, quiet=True, frozen=True)
+ self.name = "Slicer"
+ self.rows, self.cols = rows, cols
+ parameters = {}
# Channel slicing
if channels != slice(None, None, None):
# Trigger source app execution if needed
- nb_channels = get_nbchannels(x)
+ nb_channels = get_nbchannels(obj)
self.app.Execute() # this is needed by ExtractROI for setting the `cl` parameter
# if needed, converting int to list
if isinstance(channels, int):
@@ -853,90 +934,39 @@ class Slicer(App):
elif isinstance(channels, tuple):
channels = list(channels)
elif not isinstance(channels, list):
- raise ValueError(f'Invalid type for channels, should be int, slice or list of bands. : {channels}')
+ raise ValueError(f"Invalid type for channels, should be int, slice or list of bands. : {channels}")
# Change the potential negative index values to reverse index
channels = [c if c >= 0 else nb_channels + c for c in channels]
- parameters.update({'cl': [f'Channel{i + 1}' for i in channels]})
+ parameters.update({"cl": [f"Channel{i + 1}" for i in channels]})
# Spatial slicing
spatial_slicing = False
- # TODO: handle PixelValue app so that accessing value is possible, e.g. raster[120, 200, 0]
# TODO TBD: handle the step value in the slice so that NN undersampling is possible ? e.g. raster[::2, ::2]
if rows.start is not None:
- parameters.update({'mode.extent.uly': rows.start})
+ parameters.update({"mode.extent.uly": rows.start})
spatial_slicing = True
if rows.stop is not None and rows.stop != -1:
- parameters.update(
- {'mode.extent.lry': rows.stop - 1}) # subtract 1 to be compliant with python convention
+ parameters.update({"mode.extent.lry": rows.stop - 1}) # subtract 1 to respect python convention
spatial_slicing = True
if cols.start is not None:
- parameters.update({'mode.extent.ulx': cols.start})
+ parameters.update({"mode.extent.ulx": cols.start})
spatial_slicing = True
if cols.stop is not None and cols.stop != -1:
- parameters.update(
- {'mode.extent.lrx': cols.stop - 1}) # subtract 1 to be compliant with python convention
+ parameters.update({"mode.extent.lrx": cols.stop - 1}) # subtract 1 to respect python convention
spatial_slicing = True
+
# Execute app
- self.set_parameters(**parameters)
+ self.set_parameters(parameters)
self.execute()
-
# These are some attributes when the user simply wants to extract *one* band to be used in an Operation
if not spatial_slicing and isinstance(channels, list) and len(channels) == 1:
self.one_band_sliced = channels[0] + 1 # OTB convention: channels start at 1
- self.input = x
-
-
-class Input(App):
- """Class for transforming a filepath to pyOTB object."""
-
- def __init__(self, filepath):
- """Constructor for an Input object.
-
- Args:
- filepath: raster file path
-
- """
- self.filepath = filepath
- super().__init__('ExtractROI', {'in': self.filepath}, preserve_dtype=True)
-
- def __str__(self):
- """Return a nice string representation with input file path."""
- return f'<pyotb.Input object from {self.filepath}>'
+ self.input = obj
+ self.propagate_dtype()
-class Output(otbObject):
- """Class for output of an app."""
-
- def __init__(self, app, output_parameter_key):
- """Constructor for an Output object.
-
- Args:
- app: The pyotb App
- output_parameter_key: Output parameter key
-
- """
- # Keeping the OTB app and the pyotb app
- self.pyotb_app, self.app = app, app.app
- self.parameters = self.pyotb_app.parameters
- self.output_param = output_parameter_key
- self.name = f'Output {output_parameter_key} from {self.app.GetName()}'
-
- def summarize(self):
- """Return the summary of the pipeline that generates the Output object.
-
- Returns:
- Nested dictionary summarizing the pipeline that generates the Output object.
-
- """
- return self.pyotb_app.summarize()
-
- def __str__(self):
- """Return a nice string representation with object id."""
- return f'<pyotb.Output {self.app.GetName()} object, id {id(self)}>'
-
-
-class Operation(App):
+class Operation(OTBObject):
"""Class for arithmetic/math operations done in Python.
Example:
@@ -976,9 +1006,7 @@ class Operation(App):
self.nb_channels = {}
self.fake_exp_bands = []
self.logical_fake_exp_bands = []
-
self.create_fake_exp(operator, inputs, nb_bands=nb_bands)
-
# Transforming images to the adequate im#, e.g. `input1` to "im1"
# creating a dictionary that is like {str(input1): 'im1', 'image2.tif': 'im2', ...}.
# NB: the keys of the dictionary are strings-only, instead of 'complex' objects, to enable easy serialization
@@ -988,20 +1016,18 @@ class Operation(App):
for inp in self.inputs:
if not isinstance(inp, (int, float)):
if str(inp) not in self.im_dic:
- self.im_dic[str(inp)] = f'im{self.im_count}'
+ self.im_dic[str(inp)] = f"im{self.im_count}"
mapping_str_to_input[str(inp)] = inp
self.im_count += 1
- # getting unique image inputs, in the order im1, im2, im3 ...
+ # Getting unique image inputs, in the order im1, im2, im3 ...
self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)]
- self.output_param = 'out'
-
# Computing the BandMath or BandMathX app
self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands)
+ # Init app
self.name = f'Operation exp="{self.exp}"'
-
- appname = 'BandMath' if len(self.exp_bands) == 1 else 'BandMathX'
- super().__init__(appname, il=self.unique_inputs, exp=self.exp)
+ appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX"
+ super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True)
def create_fake_exp(self, operator, inputs, nb_bands=None):
"""Create a 'fake' expression.
@@ -1019,18 +1045,18 @@ class Operation(App):
logger.debug("%s, %s", operator, inputs)
# this is when we use the ternary operator with `pyotb.where` function. The output nb of bands is already known
- if operator == '?' and nb_bands:
+ if operator == "?" and nb_bands:
pass
# For any other operations, the output number of bands is the same as inputs
else:
- if any(isinstance(inp, Slicer) and hasattr(inp, 'one_band_sliced') for inp in inputs):
+ if any(isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced") for inp in inputs):
nb_bands = 1
else:
nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))]
# check that all inputs have the same nb of bands
if len(nb_bands_list) > 1:
if not all(x == nb_bands_list[0] for x in nb_bands_list):
- raise Exception('All images do not have the same number of bands')
+ raise Exception("All images do not have the same number of bands")
nb_bands = nb_bands_list[0]
# Create a list of fake expressions, each item of the list corresponding to one band
@@ -1046,12 +1072,14 @@ class Operation(App):
cond_band = 1
else:
cond_band = band
- fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp(inp, cond_band,
- keep_logical=True)
+ fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp(
+ inp, cond_band, keep_logical=True
+ )
# any other input
else:
- fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp(inp, band,
- keep_logical=False)
+ fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp(
+ inp, band, keep_logical=False
+ )
fake_exps.append(fake_exp)
# Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates)
if i == 0 and corresponding_inputs and nb_channels:
@@ -1060,13 +1088,13 @@ class Operation(App):
# Generating the fake expression of the whole operation
if len(inputs) == 1: # this is only for 'abs'
- fake_exp = f'({operator}({fake_exps[0]}))'
+ fake_exp = f"({operator}({fake_exps[0]}))"
elif len(inputs) == 2:
# We create here the "fake" expression. For example, for a BandMathX expression such as '2 * im1 + im2',
# the false expression stores the expression 2 * str(input1) + str(input2)
- fake_exp = f'({fake_exps[0]} {operator} {fake_exps[1]})'
- elif len(inputs) == 3 and operator == '?': # this is only for ternary expression
- fake_exp = f'({fake_exps[0]} ? {fake_exps[1]} : {fake_exps[2]})'
+ fake_exp = f"({fake_exps[0]} {operator} {fake_exps[1]})"
+ elif len(inputs) == 3 and operator == "?": # this is only for ternary expression
+ fake_exp = f"({fake_exps[0]} ? {fake_exps[1]} : {fake_exps[2]})"
self.fake_exp_bands.append(fake_exp)
@@ -1091,7 +1119,7 @@ class Operation(App):
exp_bands.append(one_band_exp)
# Form the final expression (e.g. 'im1b1 + 1; im1b2 + 1')
- exp = ';'.join(exp_bands)
+ exp = ";".join(exp_bands)
return exp_bands, exp
@@ -1116,8 +1144,8 @@ class Operation(App):
"""
# Special case for one-band slicer
- if isinstance(x, Slicer) and hasattr(x, 'one_band_sliced'):
- if keep_logical and isinstance(x.input, logicalOperation):
+ if isinstance(x, Slicer) and hasattr(x, "one_band_sliced"):
+ if keep_logical and isinstance(x.input, LogicalOperation):
fake_exp = x.input.logical_fake_exp_bands[x.one_band_sliced - 1]
inputs = x.input.inputs
nb_channels = x.input.nb_channels
@@ -1128,11 +1156,11 @@ class Operation(App):
nb_channels = x.input.nb_channels
else:
# Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1')
- fake_exp = str(x.input) + f'b{x.one_band_sliced}'
+ fake_exp = str(x.input) + f"b{x.one_band_sliced}"
inputs = [x.input]
nb_channels = {x.input: 1}
- # For logicalOperation, we save almost the same attributes as an Operation
- elif keep_logical and isinstance(x, logicalOperation):
+ # For LogicalOperation, we save almost the same attributes as an Operation
+ elif keep_logical and isinstance(x, LogicalOperation):
fake_exp = x.logical_fake_exp_bands[band - 1]
inputs = x.inputs
nb_channels = x.nb_channels
@@ -1150,25 +1178,24 @@ class Operation(App):
nb_channels = {x: get_nbchannels(x)}
inputs = [x]
# Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1')
- fake_exp = str(x) + f'b{band}'
+ fake_exp = str(x) + f"b{band}"
return fake_exp, inputs, nb_channels
def __str__(self):
- """Return a nice string representation with object id."""
- return f'<pyotb.Operation `{self.operator}` object, id {id(self)}>'
+ """Return a nice string representation with operator and object id."""
+ return f"<pyotb.Operation `{self.operator}` object, id {id(self)}>"
-class logicalOperation(Operation):
+class LogicalOperation(Operation):
"""A specialization of Operation class for boolean logical operations i.e. >, <, >=, <=, ==, !=, `&` and `|`.
The only difference is that not only the BandMath expression is saved (e.g. "im1b1 > 0 ? 1 : 0"), but also the
logical expression (e.g. "im1b1 > 0")
"""
-
def __init__(self, operator, *inputs, nb_bands=None):
- """Constructor for a logicalOperation object.
+ """Constructor for a LogicalOperation object.
Args:
operator: string operator (one of >, <, >=, <=, ==, !=, &, |)
@@ -1192,39 +1219,89 @@ class logicalOperation(Operation):
"""
# For any other operations, the output number of bands is the same as inputs
- if any(isinstance(inp, Slicer) and hasattr(inp, 'one_band_sliced') for inp in inputs):
+ if any(isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced") for inp in inputs):
nb_bands = 1
else:
nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))]
# check that all inputs have the same nb of bands
if len(nb_bands_list) > 1:
if not all(x == nb_bands_list[0] for x in nb_bands_list):
- raise Exception('All images do not have the same number of bands')
+ raise Exception("All images do not have the same number of bands")
nb_bands = nb_bands_list[0]
# Create a list of fake exp, each item of the list corresponding to one band
for i, band in enumerate(range(1, nb_bands + 1)):
fake_exps = []
for inp in inputs:
- fake_exp, corresponding_inputs, nb_channels = super().create_one_input_fake_exp(inp, band,
- keep_logical=True)
+ fake_exp, corresp_inputs, nb_channels = super().create_one_input_fake_exp(inp, band, keep_logical=True)
fake_exps.append(fake_exp)
# Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates)
- if i == 0 and corresponding_inputs and nb_channels:
- self.inputs.extend(corresponding_inputs)
+ if i == 0 and corresp_inputs and nb_channels:
+ self.inputs.extend(corresp_inputs)
self.nb_channels.update(nb_channels)
# We create here the "fake" expression. For example, for a BandMathX expression such as 'im1 > im2',
# the logical fake expression stores the expression "str(input1) > str(input2)"
- logical_fake_exp = f'({fake_exps[0]} {operator} {fake_exps[1]})'
+ logical_fake_exp = f"({fake_exps[0]} {operator} {fake_exps[1]})"
# We keep the logical expression, useful if later combined with other logical operations
self.logical_fake_exp_bands.append(logical_fake_exp)
# We create a valid BandMath expression, e.g. "str(input1) > str(input2) ? 1 : 0"
- fake_exp = f'({logical_fake_exp} ? 1 : 0)'
+ fake_exp = f"({logical_fake_exp} ? 1 : 0)"
self.fake_exp_bands.append(fake_exp)
+class FileIO:
+ """Base class of an IO file object."""
+ # TODO: check file exists, create missing directories, ..?
+
+
+class Input(OTBObject, FileIO):
+ """Class for transforming a filepath to pyOTB object."""
+
+ def __init__(self, filepath):
+ """Default constructor.
+
+ Args:
+ filepath: the path of an input image
+
+ """
+ super().__init__("ExtractROI", {"in": str(filepath)})
+ self._name = f"Input from {filepath}"
+ self.filepath = Path(filepath)
+ self.propagate_dtype()
+
+ def __str__(self):
+ """Return a nice string representation with file path."""
+ return f"<pyotb.Input object from {self.filepath}>"
+
+
+class Output(FileIO):
+ """Object that behave like a pointer to a specific application output file."""
+
+ def __init__(self, source_app, param_key, filepath=None):
+ """Constructor for an Output object.
+
+ Args:
+ source_app: The pyotb App to store reference from
+ param_key: Output parameter key of the target app
+ filepath: path of the output file (if not in memory)
+
+ """
+ self.source_app = source_app
+ self.param_key = param_key
+ self.filepath = None
+ if filepath:
+ if '?' in filepath:
+ filepath = filepath.split('?')[0]
+ self.filepath = Path(filepath)
+ self.name = f"Output {param_key} from {self.source_app.name}"
+
+ def __str__(self):
+ """Return a nice string representation with source app name and object id."""
+ return f"<pyotb.Output {self.source_app.name} object, id {id(self)}>"
+
+
def get_nbchannels(inp):
"""Get the nb of bands of input image.
@@ -1235,7 +1312,7 @@ def get_nbchannels(inp):
number of bands in image
"""
- if isinstance(inp, otbObject):
+ if isinstance(inp, OTBObject):
nb_channels = inp.shape[-1]
else:
# Executing the app, without printing its log
@@ -1272,8 +1349,8 @@ def get_pixel_type(inp):
'float': 'float', 'double': 'double'}
pixel_type = datatype_to_pixeltype[datatype]
pixel_type = getattr(otb, f'ImagePixelType_{pixel_type}')
- elif isinstance(inp, (otbObject)):
- pixel_type = inp.GetParameterOutputImagePixelType(inp.output_param)
+ elif isinstance(inp, (OTBObject)):
+ pixel_type = inp.GetParameterOutputImagePixelType(inp.key_output_image)
else:
raise TypeError(f'Could not get the pixel type. Not supported type: {inp}')
@@ -1295,3 +1372,63 @@ def parse_pixel_type(pixel_type):
if isinstance(pixel_type, int):
return pixel_type
raise ValueError(f'Bad pixel type specification ({pixel_type})')
+
+
+def is_key_list(pyotb_app, key):
+ """Check if a key of the App is an input parameter list."""
+ return pyotb_app.app.GetParameterType(key) in (
+ otb.ParameterType_InputImageList,
+ otb.ParameterType_StringList,
+ otb.ParameterType_InputFilenameList,
+ otb.ParameterType_ListView,
+ otb.ParameterType_InputVectorDataList,
+ )
+
+
+def is_key_images_list(pyotb_app, key):
+ """Check if a key of the App is an input parameter image list."""
+ return pyotb_app.app.GetParameterType(key) in (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList)
+
+
+def get_out_param_types(pyotb_app):
+ """Get output parameter data type (raster, vector, file)."""
+ outfile_types = {
+ otb.ParameterType_OutputImage: "raster",
+ otb.ParameterType_OutputVectorData: "vector",
+ otb.ParameterType_OutputFilename: "file",
+ }
+ for k in pyotb_app.parameters_keys:
+ t = pyotb_app.app.GetParameterType(k)
+ if t in outfile_types:
+ yield k, outfile_types[t]
+
+
+def get_out_images_param_keys(app):
+ """Return every output parameter keys of an OTB app."""
+ return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage]
+
+
+def key_input(pyotb_app, file_type):
+ """Get the first input param key for a specific file type."""
+ types = {
+ "raster": (otb.ParameterType_InputImage, otb.ParameterType_InputImageList),
+ "vector": (otb.ParameterType_InputVectorData, otb.ParameterType_InputVectorDataList),
+ "file": (otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList)
+ }
+ for key in pyotb_app.parameters_keys:
+ if pyotb_app.app.GetParameterType(key) in types[file_type]:
+ return key
+ return None
+
+
+def key_output(pyotb_app, file_type):
+ """Get the first output param key for a specific file type."""
+ types = {
+ "raster": otb.ParameterType_OutputImage,
+ "vector": otb.ParameterType_OutputVectorData,
+ "file": otb.ParameterType_OutputFilename
+ }
+ for key in pyotb_app.parameters_keys:
+ if pyotb_app.app.GetParameterType(key) == types[file_type]:
+ return key
+ return None
\ No newline at end of file
diff --git a/pyotb/functions.py b/pyotb/functions.py
index eee401a..942a760 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -7,7 +7,7 @@ import textwrap
import uuid
from collections import Counter
-from .core import (otbObject, App, Input, Operation, logicalOperation, get_nbchannels)
+from .core import (OTBObject, App, Input, Operation, LogicalOperation, get_nbchannels)
from .helpers import logger
@@ -111,25 +111,25 @@ def all(*inputs): # pylint: disable=redefined-builtin
# Checking that all bands of the single image are True
if len(inputs) == 1:
inp = inputs[0]
- if isinstance(inp, logicalOperation):
+ if isinstance(inp, LogicalOperation):
res = inp[:, :, 0]
else:
res = (inp[:, :, 0] != 0)
for band in range(1, inp.shape[-1]):
- if isinstance(inp, logicalOperation):
+ if isinstance(inp, LogicalOperation):
res = res & inp[:, :, band]
else:
res = res & (inp[:, :, band] != 0)
# Checking that all images are True
else:
- if isinstance(inputs[0], logicalOperation):
+ if isinstance(inputs[0], LogicalOperation):
res = inputs[0]
else:
res = (inputs[0] != 0)
for inp in inputs[1:]:
- if isinstance(inp, logicalOperation):
+ if isinstance(inp, LogicalOperation):
res = res & inp
else:
res = res & (inp != 0)
@@ -166,25 +166,25 @@ def any(*inputs): # pylint: disable=redefined-builtin
# Checking that at least one band of the image is True
if len(inputs) == 1:
inp = inputs[0]
- if isinstance(inp, logicalOperation):
+ if isinstance(inp, LogicalOperation):
res = inp[:, :, 0]
else:
res = (inp[:, :, 0] != 0)
for band in range(1, inp.shape[-1]):
- if isinstance(inp, logicalOperation):
+ if isinstance(inp, LogicalOperation):
res = res | inp[:, :, band]
else:
res = res | (inp[:, :, band] != 0)
# Checking that at least one image is True
else:
- if isinstance(inputs[0], logicalOperation):
+ if isinstance(inputs[0], LogicalOperation):
res = inputs[0]
else:
res = (inputs[0] != 0)
for inp in inputs[1:]:
- if isinstance(inp, logicalOperation):
+ if isinstance(inp, LogicalOperation):
res = res | inp
else:
res = res | (inp != 0)
diff --git a/tests/test_numpy.py b/tests/test_numpy.py
index 5b1dd04..8780473 100644
--- a/tests/test_numpy.py
+++ b/tests/test_numpy.py
@@ -31,7 +31,7 @@ def test_convert_to_array():
def test_add_noise_array():
white_noise = np.random.normal(0, 50, size=INPUT.shape)
noisy_image = INPUT + white_noise
- assert isinstance(noisy_image, pyotb.otbObject)
+ assert isinstance(noisy_image, pyotb.core.OTBObject)
assert noisy_image.shape == INPUT.shape
--
GitLab
From 4123e0377d6327eafd02e2cdfa694e6b80f4232e Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 5 Jan 2023 19:33:02 +0100
Subject: [PATCH 003/399] ENH: move App class def to apps.py
---
pyotb/__init__.py | 1 -
pyotb/apps.py | 31 ++++++++++++++++++++++++++++++-
pyotb/core.py | 33 ++-------------------------------
pyotb/functions.py | 8 ++++----
4 files changed, 36 insertions(+), 37 deletions(-)
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index e3bc23a..6164870 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -9,7 +9,6 @@ otb = find_otb()
from .apps import *
from .core import (
- App,
Input,
Output,
get_nbchannels,
diff --git a/pyotb/apps.py b/pyotb/apps.py
index 30e3363..6d0ff91 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -5,8 +5,8 @@ import sys
from pathlib import Path
import otbApplication as otb
-from .core import App
from .helpers import logger
+from .core import OTBObject
def get_available_applications(as_subprocess=False):
@@ -61,6 +61,35 @@ def get_available_applications(as_subprocess=False):
return app_list
+class App(OTBObject):
+
+ def find_outputs(self):
+ """Find output files on disk using path found in parameters.
+
+ Returns:
+ list of files found on disk
+
+ """
+ files = []
+ missing = []
+ for param in self.outputs:
+ filename = self.parameters[param]
+ # Remove filename extension
+ if '?' in filename:
+ filename = filename.split('?')[0]
+ path = Path(filename)
+ if path.exists():
+ files.append(str(path.absolute()))
+ else:
+ missing.append(str(path.absolute()))
+ if missing:
+ missing = tuple(missing)
+ for filename in missing:
+ logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
+
+ return files
+
+
class OTBTFApp(App):
"""Helper for OTBTF."""
@staticmethod
diff --git a/pyotb/core.py b/pyotb/core.py
index 7992890..4df41df 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -865,35 +865,6 @@ class OTBObject:
return NotImplemented
-class App(OTBObject):
-
- def find_outputs(self):
- """Find output files on disk using path found in parameters.
-
- Returns:
- list of files found on disk
-
- """
- files = []
- missing = []
- for param in self.outputs:
- filename = self.parameters[param]
- # Remove filename extension
- if '?' in filename:
- filename = filename.split('?')[0]
- path = Path(filename)
- if path.exists():
- files.append(str(path.absolute()))
- else:
- missing.append(str(path.absolute()))
- if missing:
- missing = tuple(missing)
- for filename in missing:
- logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
-
- return files
-
-
class Slicer(OTBObject):
"""Slicer objects i.e. when we call something like raster[:, :, 2] from Python."""
@@ -1317,7 +1288,7 @@ def get_nbchannels(inp):
else:
# Executing the app, without printing its log
try:
- info = App("ReadImageInfo", inp, quiet=True)
+ info = OTBObject("ReadImageInfo", inp, quiet=True)
nb_channels = info.GetParameterInt("numberbands")
except Exception as e: # this happens when we pass a str that is not a filepath
raise TypeError(f'Could not get the number of channels of `{inp}`. Not a filepath or wrong filepath') from e
@@ -1338,7 +1309,7 @@ def get_pixel_type(inp):
if isinstance(inp, str):
# Executing the app, without printing its log
try:
- info = App("ReadImageInfo", inp, quiet=True)
+ info = OTBObject("ReadImageInfo", inp, quiet=True)
except Exception as info_err: # this happens when we pass a str that is not a filepath
raise TypeError(f'Could not get the pixel type of `{inp}`. Not a filepath or wrong filepath') from info_err
datatype = info.GetParameterString("datatype") # which is such as short, float...
diff --git a/pyotb/functions.py b/pyotb/functions.py
index 942a760..0ead43d 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -7,7 +7,7 @@ import textwrap
import uuid
from collections import Counter
-from .core import (OTBObject, App, Input, Operation, LogicalOperation, get_nbchannels)
+from .core import (OTBObject, Input, Operation, LogicalOperation, get_nbchannels)
from .helpers import logger
@@ -412,7 +412,7 @@ def define_processing_area(*args, window_rule='intersection', pixel_size_rule='m
'mode.extent.ulx': ulx, 'mode.extent.uly': lry, # bug in OTB <= 7.3 :
'mode.extent.lrx': lrx, 'mode.extent.lry': uly, # ULY/LRY are inverted
}
- new_input = App('ExtractROI', params)
+ new_input = OTBObject('ExtractROI', params)
# TODO: OTB 7.4 fixes this bug, how to handle different versions of OTB?
new_inputs.append(new_input)
# Potentially update the reference inputs for later resampling
@@ -453,7 +453,7 @@ def define_processing_area(*args, window_rule='intersection', pixel_size_rule='m
new_inputs = []
for inp in inputs:
if metadatas[inp]['GeoTransform'][1] != pixel_size:
- superimposed = App('Superimpose', inr=reference_input, inm=inp, interpolator=interpolator)
+ superimposed = OTBObject('Superimpose', inr=reference_input, inm=inp, interpolator=interpolator)
new_inputs.append(superimposed)
else:
new_inputs.append(inp)
@@ -478,7 +478,7 @@ def define_processing_area(*args, window_rule='intersection', pixel_size_rule='m
new_inputs = []
for inp in inputs:
if image_sizes[inp] != most_common_image_size:
- superimposed = App('Superimpose', inr=same_size_images[0], inm=inp, interpolator=interpolator)
+ superimposed = OTBObject('Superimpose', inr=same_size_images[0], inm=inp, interpolator=interpolator)
new_inputs.append(superimposed)
else:
new_inputs.append(inp)
--
GitLab
From 6b53f8ae364dea699820a375b360369056b36ec2 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 5 Jan 2023 19:56:34 +0100
Subject: [PATCH 004/399] ENH: update flush() docstring
---
pyotb/core.py | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 4df41df..fabc38b 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -183,13 +183,7 @@ class OTBObject:
self.save_objects()
def flush(self):
- """Flush data to disk, this is when WriteOutput is actually called.
-
- Args:
- parameters: key value pairs like {parameter: filename}
- dtypes: optional dict to pass output data types (for rasters)
-
- """
+ """Flush data to disk, this is when WriteOutput is actually called."""
try:
logger.debug("%s: flushing data to disk", self.name)
self.app.WriteOutput()
--
GitLab
From e29539ed4f2f4bcf46dae1760a7669adba684908 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 5 Jan 2023 20:47:08 +0100
Subject: [PATCH 005/399] STYLE: linting
---
pyotb/apps.py | 4 +++-
pyotb/core.py | 13 +++++++++----
pyotb/functions.py | 2 +-
3 files changed, 13 insertions(+), 6 deletions(-)
diff --git a/pyotb/apps.py b/pyotb/apps.py
index 6d0ff91..0302ebc 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -62,6 +62,7 @@ def get_available_applications(as_subprocess=False):
class App(OTBObject):
+ """Base class for UI related functions, will be subclassed using app name as class name, see CODE_TEMPLATE."""
def find_outputs(self):
"""Find output files on disk using path found in parameters.
@@ -120,6 +121,7 @@ class OTBTFApp(App):
n_sources: number of sources. Default is None (resolves the number of sources based on the
content of the dict passed in args, where some 'source' str is found)
**kwargs: kwargs
+
"""
self.set_nb_sources(*args, n_sources=n_sources)
super().__init__(app_name, *args, **kwargs)
@@ -142,4 +144,4 @@ for _app in AVAILABLE_APPLICATIONS:
exec(_CODE_TEMPLATE.format(name=_app).replace("(App)", "(OTBTFApp)")) # pylint: disable=exec-used
# Default behavior for any OTB application
else:
- exec(_CODE_TEMPLATE.format(name=_app)) # pylint: disable=exec-used
\ No newline at end of file
+ exec(_CODE_TEMPLATE.format(name=_app)) # pylint: disable=exec-used
diff --git a/pyotb/core.py b/pyotb/core.py
index fabc38b..7204767 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -24,6 +24,11 @@ class OTBObject:
- list, useful when the user wants to specify the input list 'il'
frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___
quiet: whether to print logs of the OTB app
+ image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as
+ the result of app.ExportImage(). Use it when the app takes a numpy array as input.
+ See this related issue for why it is necessary to keep reference of object:
+ https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824
+
**kwargs: used for passing application parameters.
e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif'
@@ -53,12 +58,12 @@ class OTBObject:
@property
def key_input_image(self):
- """Get the name of first output image parameter"""
+ """Get the name of first input image parameter."""
return key_input(self, "raster")
@property
def key_output_image(self):
- """Get the name of first output image parameter"""
+ """Get the name of first output image parameter."""
return key_output(self, "raster")
@property
@@ -117,7 +122,7 @@ class OTBObject:
@property
def transform(self):
- """Get image affine transform, rasterio style (see https://www.perrygeo.com/python-affine-transforms.html)
+ """Get image affine transform, rasterio style (see https://www.perrygeo.com/python-affine-transforms.html).
Returns:
transform: (X spacing, X offset, X origin, Y offset, Y spacing, Y origin)
@@ -406,7 +411,7 @@ class OTBObject:
return array, profile
def xy_to_rowcol(self, x, y):
- """Find (row, col) index using (x, y) projected coordinates, expect image CRS
+ """Find (row, col) index using (x, y) projected coordinates - image CRS is exepcted.
Args:
x: longitude or projected X
diff --git a/pyotb/functions.py b/pyotb/functions.py
index 0ead43d..669039a 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -352,7 +352,7 @@ def define_processing_area(*args, window_rule='intersection', pixel_size_rule='m
for inp in inputs:
if isinstance(inp, str): # this is for filepaths
metadata = Input(inp).GetImageMetaData('out')
- elif isinstance(inp, otbObject):
+ elif isinstance(inp, OTBObject):
metadata = inp.GetImageMetaData(inp.output_param)
else:
raise TypeError(f"Wrong input : {inp}")
--
GitLab
From 9988d74385b1636e049f9b93cb4ed2287d257f88 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 5 Jan 2023 20:00:06 +0000
Subject: [PATCH 006/399] STYLE: more linting
---
pyotb/core.py | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 7204767..81e0409 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
"""This module is the core of pyotb."""
from pathlib import Path
+from ast import literal_eval
import numpy as np
import otbApplication as otb
@@ -55,7 +56,7 @@ class OTBObject:
def key_input(self):
"""Get the name of first input parameter, raster > vector > file."""
return self.key_input_image or key_input(self, "vector") or key_input(self, "file")
-
+
@property
def key_input_image(self):
"""Get the name of first input image parameter."""
@@ -92,7 +93,7 @@ class OTBObject:
@property
def outputs(self):
"""List of application outputs."""
- return [getattr(self, key) for key in self.out_param_keys if key in self.parameters]
+ return [getattr(self, key) for key in self.out_param_keys if key in self.parameters]
@property
def dtype(self):
@@ -411,7 +412,7 @@ class OTBObject:
return array, profile
def xy_to_rowcol(self, x, y):
- """Find (row, col) index using (x, y) projected coordinates - image CRS is exepcted.
+ """Find (row, col) index using (x, y) projected coordinates - image CRS is expected.
Args:
x: longitude or projected X
@@ -1357,7 +1358,9 @@ def is_key_list(pyotb_app, key):
def is_key_images_list(pyotb_app, key):
"""Check if a key of the App is an input parameter image list."""
- return pyotb_app.app.GetParameterType(key) in (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList)
+ return pyotb_app.app.GetParameterType(key) in (
+ otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList
+ )
def get_out_param_types(pyotb_app):
@@ -1401,4 +1404,4 @@ def key_output(pyotb_app, file_type):
for key in pyotb_app.parameters_keys:
if pyotb_app.app.GetParameterType(key) == types[file_type]:
return key
- return None
\ No newline at end of file
+ return None
--
GitLab
From a682b9efdb40093d5ea9d63ccca2a245f5692dea Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 6 Jan 2023 18:06:03 +0100
Subject: [PATCH 007/399] FIX: make sure to call execute *after*
propagate_dtype
---
pyotb/core.py | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 81e0409..56a9786 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -926,15 +926,15 @@ class Slicer(OTBObject):
if cols.stop is not None and cols.stop != -1:
parameters.update({"mode.extent.lrx": cols.stop - 1}) # subtract 1 to respect python convention
spatial_slicing = True
-
- # Execute app
- self.set_parameters(parameters)
- self.execute()
# These are some attributes when the user simply wants to extract *one* band to be used in an Operation
if not spatial_slicing and isinstance(channels, list) and len(channels) == 1:
self.one_band_sliced = channels[0] + 1 # OTB convention: channels start at 1
self.input = obj
+
+ # Execute app
+ self.set_parameters(parameters)
self.propagate_dtype()
+ self.execute()
class Operation(OTBObject):
@@ -1237,10 +1237,11 @@ class Input(OTBObject, FileIO):
filepath: the path of an input image
"""
- super().__init__("ExtractROI", {"in": str(filepath)})
+ super().__init__("ExtractROI", {"in": str(filepath)}, frozen=True)
self._name = f"Input from {filepath}"
self.filepath = Path(filepath)
self.propagate_dtype()
+ self.execute()
def __str__(self):
"""Return a nice string representation with file path."""
--
GitLab
From 4ab1fc77df0e90ad9e56a3ec64d5c713a4f8e2de Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 9 Jan 2023 10:48:10 +0100
Subject: [PATCH 008/399] STYLE: exports_dict -> exports_dic to align with
image_dic
---
pyotb/core.py | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 56a9786..a6aaa7a 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -38,6 +38,7 @@ class OTBObject:
self.appname = appname
self.quiet = quiet
self.image_dic = image_dic
+ self.exports_dic = {}
if quiet:
self.app = otb.Registry.CreateApplicationWithoutLogger(appname)
else:
@@ -45,7 +46,6 @@ class OTBObject:
self.parameters_keys = tuple(self.app.GetParametersKeys())
self.out_param_types = dict(get_out_param_types(self))
self.out_param_keys = tuple(self.out_param_types.keys())
- self.exports_dict = {}
if args or kwargs:
self.set_parameters(*args, **kwargs)
self.frozen = frozen
@@ -356,7 +356,7 @@ class OTBObject:
return {"name": self.name, "parameters": params}
def export(self, key=None):
- """Export a specific output image as numpy array and store it in object's exports_dict.
+ """Export a specific output image as numpy array and store it in object's exports_dic.
Args:
key: parameter key to export, if None then the default one will be used
@@ -367,9 +367,9 @@ class OTBObject:
"""
if key is None:
key = key_output(self, "raster")
- if key not in self.exports_dict:
- self.exports_dict[key] = self.app.ExportImage(key)
- return self.exports_dict[key]
+ if key not in self.exports_dic:
+ self.exports_dic[key] = self.app.ExportImage(key)
+ return self.exports_dic[key]
def to_numpy(self, key=None, preserve_dtype=True, copy=False):
"""Export a pyotb object to numpy array.
@@ -839,9 +839,9 @@ class OTBObject:
if isinstance(inp, (float, int, np.ndarray, np.generic)):
arrays.append(inp)
elif isinstance(inp, OTBObject):
- if not inp.exports_dict:
+ if not inp.exports_dic:
inp.export()
- image_dic = inp.exports_dict[inp.key_output_image]
+ image_dic = inp.exports_dic[inp.key_output_image]
array = image_dic["array"]
arrays.append(array)
else:
--
GitLab
From f8e72333096da9e2c8b84971ae7ec895f75080d3 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 9 Jan 2023 15:04:44 +0100
Subject: [PATCH 009/399] ENH: add preserve_dtype to np export function
---
pyotb/core.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index a6aaa7a..795e4e7 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -355,7 +355,7 @@ class OTBObject:
return {"name": self.name, "parameters": params}
- def export(self, key=None):
+ def export(self, key=None, preserve_dtype=True):
"""Export a specific output image as numpy array and store it in object's exports_dic.
Args:
@@ -366,9 +366,11 @@ class OTBObject:
"""
if key is None:
- key = key_output(self, "raster")
+ key = self.key_output_image
if key not in self.exports_dic:
self.exports_dic[key] = self.app.ExportImage(key)
+ if preserve_dtype:
+ self.exports_dic[key]["array"] = self.exports_dic[key]["array"].astype(self.dtype)
return self.exports_dic[key]
def to_numpy(self, key=None, preserve_dtype=True, copy=False):
@@ -376,7 +378,7 @@ class OTBObject:
Args:
key: the output parameter name to export as numpy array
- preserve_dtype: when set to True, the numpy array is created with the same pixel type as
+ preserve_dtype: when set to True, the numpy array is converted to the same pixel type as
the OTBObject first output. Default is True.
copy: whether to copy the output array, default is False
required to True if preserve_dtype is False and the source app reference is lost
@@ -385,10 +387,8 @@ class OTBObject:
a numpy array
"""
- data = self.export(key)
+ data = self.export(key, preserve_dtype)
array = data["array"]
- if preserve_dtype:
- return array.astype(self.dtype)
if copy:
return array.copy()
return array
--
GitLab
From ba47461d86508615e588f2c8a66c61466f48721e Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 9 Jan 2023 15:05:56 +0100
Subject: [PATCH 010/399] CI: enhance existing tests + add new ones
---
tests/test_core.py | 63 ++++++++++++++++++++++++++++++++++++------
tests/test_numpy.py | 11 +++++++-
tests/test_pipeline.py | 2 +-
3 files changed, 66 insertions(+), 10 deletions(-)
diff --git a/tests/test_core.py b/tests/test_core.py
index abefc80..7cbff55 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -8,7 +8,26 @@ FILEPATH = os.environ["TEST_INPUT_IMAGE"]
INPUT = pyotb.Input(FILEPATH)
-# Basic tests
+# Test ExtractROI app's parameters were set for the Input object
+def test_parameters():
+ assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304)
+
+
+# Test OTB objects properties
+def test_name():
+ assert INPUT.name == "Input from tests/image.tif"
+ INPUT.name = "Test input"
+ assert INPUT.name == "Test input"
+
+
+def test_key_input():
+ assert INPUT.key_input == INPUT.key_input_image == "in"
+
+
+def test_key_output():
+ assert INPUT.key_output_image == "out"
+
+
def test_dtype():
assert INPUT.dtype == "uint8"
@@ -17,9 +36,14 @@ def test_shape():
assert INPUT.shape == (304, 251, 4)
+def test_transform():
+ assert INPUT.transform == (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0)
+
+
def test_slicer_shape():
extract = INPUT[:50, :60, :3]
assert extract.shape == (50, 60, 3)
+ assert extract.parameters["cl"] == ("Channel1", "Channel2", "Channel3")
def test_slicer_preserve_dtype():
@@ -27,10 +51,18 @@ def test_slicer_preserve_dtype():
assert extract.dtype == "uint8"
-# More complex tests
+# Arithmetic tests
def test_operation():
op = INPUT / 255 * 128
- assert op.exp == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)"
+ assert (
+ op.exp
+ == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)"
+ )
+ assert op.dtype == "float32"
+
+
+def test_func_abs_expression():
+ assert abs(INPUT).exp == "(abs(im1b1));(abs(im1b2));(abs(im1b3));(abs(im1b4))"
def test_sum_bands():
@@ -43,7 +75,10 @@ def test_binary_mask_where():
# Create binary mask based on several possible values
values = [1, 2, 3, 4]
res = pyotb.where(pyotb.any(INPUT[:, :, 0] == value for value in values), 255, 0)
- assert res.exp == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)"
+ assert (
+ res.exp
+ == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)"
+ )
# Apps
@@ -64,9 +99,11 @@ def test_app_computeimagestats_sliced():
assert slicer_stats["out.min"] == "[180]"
-# NDVI
+# BandMath NDVI == RadiometricIndices NDVI ?
def test_ndvi_comparison():
- ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (INPUT[:, :, -1] + INPUT[:, :, 0])
+ ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (
+ INPUT[:, :, -1] + INPUT[:, :, 0]
+ )
ndvi_indices = pyotb.RadiometricIndices(
{"in": INPUT, "list": "Vegetation:NDVI", "channels.red": 1, "channels.nir": 4}
)
@@ -77,7 +114,9 @@ def test_ndvi_comparison():
ndvi_indices.write("/tmp/ndvi_indices.tif", pixel_type="float")
assert Path("/tmp/ndvi_indices.tif").exists()
- compared = pyotb.CompareImages({"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"})
+ compared = pyotb.CompareImages(
+ {"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"}
+ )
assert compared.count == 0
assert compared.mse == 0
@@ -85,4 +124,12 @@ def test_ndvi_comparison():
assert thresholded_indices.exp == "((im1b1 >= 0.3) ? 1 : 0)"
thresholded_bandmath = pyotb.where(ndvi_bandmath >= 0.3, 1, 0)
- assert thresholded_bandmath.exp == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)"
+ assert (
+ thresholded_bandmath.exp
+ == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)"
+ )
+
+
+# XY => RowCol
+def test_xy_to_rowcol():
+ assert INPUT.xy_to_rowcol(760100, 6946210) == (19, 7)
diff --git a/tests/test_numpy.py b/tests/test_numpy.py
index 8780473..4240cb9 100644
--- a/tests/test_numpy.py
+++ b/tests/test_numpy.py
@@ -7,6 +7,14 @@ FILEPATH = os.environ["TEST_INPUT_IMAGE"]
INPUT = pyotb.Input(FILEPATH)
+def test_export():
+ INPUT.export()
+ assert "out" in INPUT.exports_dic
+ array = INPUT.exports_dic["out"]["array"]
+ assert isinstance(array, np.ndarray)
+ assert array.dtype == "uint8"
+
+
def test_to_numpy():
array = INPUT.to_numpy()
assert array.dtype == np.uint8
@@ -42,8 +50,9 @@ def test_to_rasterio():
assert profile["transform"] == (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0)
# CRS test requires GDAL python bindings
- try:
+ try:
from osgeo import osr
+
crs = osr.SpatialReference()
crs.ImportFromEPSG(2154)
dest_crs = osr.SpatialReference()
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index c2d8f0e..3c88153 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -17,7 +17,7 @@ OTBAPPS_BLOCKS = [
]
PYOTB_BLOCKS = [
- lambda inp: 1 + abs(inp) * 2,
+ lambda inp: 1 / (1 + abs(inp) * 2),
lambda inp: inp[:80, 10:60, :],
]
PIPELINES_LENGTH = [1, 2, 3]
--
GitLab
From 5dc70553851882edd1916ef1776d54475e915395 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr>
Date: Mon, 9 Jan 2023 14:15:19 +0000
Subject: [PATCH 011/399] Apply 7 suggestion(s) to 2 file(s)
---
pyotb/apps.py | 6 ++----
pyotb/core.py | 14 ++++++--------
2 files changed, 8 insertions(+), 12 deletions(-)
diff --git a/pyotb/apps.py b/pyotb/apps.py
index 0302ebc..2ef6a90 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -79,10 +79,8 @@ class App(OTBObject):
if '?' in filename:
filename = filename.split('?')[0]
path = Path(filename)
- if path.exists():
- files.append(str(path.absolute()))
- else:
- missing.append(str(path.absolute()))
+ dest = files if path.exists() else missing
+ dest.append(str(path.absolute()))
if missing:
missing = tuple(missing)
for filename in missing:
diff --git a/pyotb/core.py b/pyotb/core.py
index 795e4e7..4de1440 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -441,9 +441,7 @@ class OTBObject:
for arg in args:
if isinstance(arg, dict):
kwargs.update(arg)
- elif isinstance(arg, (str, OTBObject)):
- kwargs.update({self.key_input: arg})
- elif isinstance(arg, list) and is_key_list(self, self.key_input):
+ elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.key_input):
kwargs.update({self.key_input: arg})
return kwargs
@@ -453,7 +451,7 @@ class OTBObject:
self.app.ClearValue(key)
return
if key not in self.parameters_keys:
- raise Exception(
+ raise KeyError(
f"{self.name}: parameter '{key}' was not recognized. " f"Available keys are {self.parameters_keys}"
)
# Single-parameter cases
@@ -1027,7 +1025,7 @@ class Operation(OTBObject):
# check that all inputs have the same nb of bands
if len(nb_bands_list) > 1:
if not all(x == nb_bands_list[0] for x in nb_bands_list):
- raise Exception("All images do not have the same number of bands")
+ raise ValueError("All images do not have the same number of bands")
nb_bands = nb_bands_list[0]
# Create a list of fake expressions, each item of the list corresponding to one band
@@ -1127,7 +1125,7 @@ class Operation(OTBObject):
nb_channels = x.input.nb_channels
else:
# Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1')
- fake_exp = str(x.input) + f"b{x.one_band_sliced}"
+ fake_exp = f"{x.input}b{x.one_band_sliced}"
inputs = [x.input]
nb_channels = {x.input: 1}
# For LogicalOperation, we save almost the same attributes as an Operation
@@ -1149,7 +1147,7 @@ class Operation(OTBObject):
nb_channels = {x: get_nbchannels(x)}
inputs = [x]
# Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1')
- fake_exp = str(x) + f"b{band}"
+ fake_exp = f"{x}b{band}"
return fake_exp, inputs, nb_channels
@@ -1197,7 +1195,7 @@ class LogicalOperation(Operation):
# check that all inputs have the same nb of bands
if len(nb_bands_list) > 1:
if not all(x == nb_bands_list[0] for x in nb_bands_list):
- raise Exception("All images do not have the same number of bands")
+ raise ValueError("All images do not have the same number of bands")
nb_bands = nb_bands_list[0]
# Create a list of fake exp, each item of the list corresponding to one band
--
GitLab
From 33681e964aa0ea59229e5f5e3e9556c555253ba0 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 9 Jan 2023 15:31:23 +0100
Subject: [PATCH 012/399] CI: temporarily mark ndvi_test as expected failure
---
tests/test_core.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/tests/test_core.py b/tests/test_core.py
index 7cbff55..5539245 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -3,6 +3,8 @@ import pyotb
from ast import literal_eval
from pathlib import Path
+import pytest
+
FILEPATH = os.environ["TEST_INPUT_IMAGE"]
INPUT = pyotb.Input(FILEPATH)
@@ -100,6 +102,7 @@ def test_app_computeimagestats_sliced():
# BandMath NDVI == RadiometricIndices NDVI ?
+@pytest.mark.xfail(reason="Regression in OTB 8.2, waiting for Rémi's patch to be merged.")
def test_ndvi_comparison():
ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (
INPUT[:, :, -1] + INPUT[:, :, 0]
--
GitLab
From a1eae249bc0a50732825b00b3d8b90ab0486a6b2 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 10 Jan 2023 15:56:25 +0100
Subject: [PATCH 013/399] ENH: better Input and Output classes, check path
before exec
---
pyotb/core.py | 27 +++++++++++++++------------
1 file changed, 15 insertions(+), 12 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 4de1440..4cd7ae0 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -172,8 +172,9 @@ class OTBObject:
# Update _parameters using values from OtbApplication object
otb_params = self.app.GetParameters().items()
otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params}
- # Save parameters keys, and values as object attributes
+ # Update param dict and save values as object attributes
self.parameters.update({**parameters, **otb_params})
+ self.save_objects()
def execute(self):
"""Execute and write to disk if any output parameter has been set during init."""
@@ -272,7 +273,6 @@ class OTBObject:
self.set_parameters({key: output_filename})
self.flush()
- self.save_objects()
def propagate_dtype(self, target_key=None, dtype=None):
"""Propagate a pixel type from main input to every outputs, or to a target output key only.
@@ -1220,12 +1220,7 @@ class LogicalOperation(Operation):
self.fake_exp_bands.append(fake_exp)
-class FileIO:
- """Base class of an IO file object."""
- # TODO: check file exists, create missing directories, ..?
-
-
-class Input(OTBObject, FileIO):
+class Input(OTBObject):
"""Class for transforming a filepath to pyOTB object."""
def __init__(self, filepath):
@@ -1235,9 +1230,11 @@ class Input(OTBObject, FileIO):
filepath: the path of an input image
"""
+ self.filepath = Path(filepath)
+ if not self.filepath.exists():
+ raise FileNotFoundError(filepath)
super().__init__("ExtractROI", {"in": str(filepath)}, frozen=True)
self._name = f"Input from {filepath}"
- self.filepath = Path(filepath)
self.propagate_dtype()
self.execute()
@@ -1246,10 +1243,10 @@ class Input(OTBObject, FileIO):
return f"<pyotb.Input object from {self.filepath}>"
-class Output(FileIO):
+class Output:
"""Object that behave like a pointer to a specific application output file."""
- def __init__(self, source_app, param_key, filepath=None):
+ def __init__(self, source_app, param_key, filepath=None, mkdir=True):
"""Constructor for an Output object.
Args:
@@ -1261,12 +1258,18 @@ class Output(FileIO):
self.source_app = source_app
self.param_key = param_key
self.filepath = None
- if filepath:
+ if filepath is not None:
if '?' in filepath:
filepath = filepath.split('?')[0]
self.filepath = Path(filepath)
+ if mkdir:
+ self.make_parent_dirs()
self.name = f"Output {param_key} from {self.source_app.name}"
+ def make_parent_dirs(self):
+ if not self.filepath.parent.exists():
+ self.filepath.parent.mkdir(parents=True)
+
def __str__(self):
"""Return a nice string representation with source app name and object id."""
return f"<pyotb.Output {self.source_app.name} object, id {id(self)}>"
--
GitLab
From 959deab9fdab2acbcdefd96e24c6f51e47f852c7 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 10 Jan 2023 15:56:43 +0100
Subject: [PATCH 014/399] ENH: more tests for the core module
---
tests/test_core.py | 76 ++++++++++++++++++++++++----------------------
1 file changed, 40 insertions(+), 36 deletions(-)
diff --git a/tests/test_core.py b/tests/test_core.py
index 5539245..7d231bb 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -10,12 +10,29 @@ FILEPATH = os.environ["TEST_INPUT_IMAGE"]
INPUT = pyotb.Input(FILEPATH)
-# Test ExtractROI app's parameters were set for the Input object
+# Input settings
def test_parameters():
assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304)
-# Test OTB objects properties
+# Catch errors before exec
+def test_missing_input_file():
+ with pytest.raises(FileNotFoundError):
+ pyotb.Input("missing_file.tif")
+
+
+def test_wrong_key():
+ with pytest.raises(KeyError):
+ app = pyotb.BandMath(INPUT, expression="im1b1")
+
+
+# Create dir before write
+def test_write():
+ INPUT.write("/tmp/missing_dir/test_write.tif")
+ assert INPUT.out.filepath.exists()
+
+
+# OTBObject's properties
def test_name():
assert INPUT.name == "Input from tests/image.tif"
INPUT.name = "Test input"
@@ -53,13 +70,10 @@ def test_slicer_preserve_dtype():
assert extract.dtype == "uint8"
-# Arithmetic tests
+# Arithmetics
def test_operation():
op = INPUT / 255 * 128
- assert (
- op.exp
- == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)"
- )
+ assert op.exp == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)"
assert op.dtype == "float32"
@@ -68,7 +82,6 @@ def test_func_abs_expression():
def test_sum_bands():
- # Sum of bands
summed = sum(INPUT[:, :, b] for b in range(INPUT.shape[-1]))
assert summed.exp == "((((0 + im1b1) + im1b2) + im1b3) + im1b4)"
@@ -77,17 +90,13 @@ def test_binary_mask_where():
# Create binary mask based on several possible values
values = [1, 2, 3, 4]
res = pyotb.where(pyotb.any(INPUT[:, :, 0] == value for value in values), 255, 0)
- assert (
- res.exp
- == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)"
- )
+ assert res.exp == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)"
# Apps
def test_app_readimageinfo():
info = pyotb.ReadImageInfo(INPUT, quiet=True)
- assert info.sizex == 251
- assert info.sizey == 304
+ assert (info.sizex, info.sizey) == (251, 304)
assert info["numberbands"] == info.numberbands == 4
@@ -101,38 +110,33 @@ def test_app_computeimagestats_sliced():
assert slicer_stats["out.min"] == "[180]"
+def test_read_values_at_coords():
+ assert INPUT[0, 0, 0] == 180
+ assert INPUT[10, 20, :] == [196, 188, 172, 255]
+
+
+# XY => RowCol
+def test_xy_to_rowcol():
+ assert INPUT.xy_to_rowcol(760100, 6946210) == (19, 7)
+
+
# BandMath NDVI == RadiometricIndices NDVI ?
@pytest.mark.xfail(reason="Regression in OTB 8.2, waiting for Rémi's patch to be merged.")
def test_ndvi_comparison():
- ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (
- INPUT[:, :, -1] + INPUT[:, :, 0]
- )
- ndvi_indices = pyotb.RadiometricIndices(
- {"in": INPUT, "list": "Vegetation:NDVI", "channels.red": 1, "channels.nir": 4}
- )
+ ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (INPUT[:, :, -1] + INPUT[:, :, 0])
+ ndvi_indices = pyotb.RadiometricIndices(INPUT, {"list": "Vegetation:NDVI", "channels.red": 1, "channels.nir": 4})
assert ndvi_bandmath.exp == "((im1b4 - im1b1) / (im1b4 + im1b1))"
ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", pixel_type="float")
- assert Path("/tmp/ndvi_bandmath.tif").exists()
+ assert ndvi_bandmath.out.filepath.exists()
ndvi_indices.write("/tmp/ndvi_indices.tif", pixel_type="float")
- assert Path("/tmp/ndvi_indices.tif").exists()
+ assert ndvi_indices.out.filepath.exists()
- compared = pyotb.CompareImages(
- {"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"}
- )
- assert compared.count == 0
- assert compared.mse == 0
+ compared = pyotb.CompareImages({"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"})
+ assert (compared.count, compared.mse) == (0, 0)
thresholded_indices = pyotb.where(ndvi_indices >= 0.3, 1, 0)
assert thresholded_indices.exp == "((im1b1 >= 0.3) ? 1 : 0)"
thresholded_bandmath = pyotb.where(ndvi_bandmath >= 0.3, 1, 0)
- assert (
- thresholded_bandmath.exp
- == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)"
- )
-
-
-# XY => RowCol
-def test_xy_to_rowcol():
- assert INPUT.xy_to_rowcol(760100, 6946210) == (19, 7)
+ assert thresholded_bandmath.exp == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)"
--
GitLab
From 07bda7ce7606834a7a98fbd3c74977ecf1cf424f Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 10 Jan 2023 16:24:06 +0100
Subject: [PATCH 015/399] CI: linter and test order
---
pyotb/core.py | 8 +++++++-
tests/test_core.py | 16 ++++++++--------
2 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 4cd7ae0..94b26d8 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1253,12 +1253,13 @@ class Output:
source_app: The pyotb App to store reference from
param_key: Output parameter key of the target app
filepath: path of the output file (if not in memory)
+ mkdir: create missing parent directories
"""
self.source_app = source_app
self.param_key = param_key
self.filepath = None
- if filepath is not None:
+ if filepath:
if '?' in filepath:
filepath = filepath.split('?')[0]
self.filepath = Path(filepath)
@@ -1266,7 +1267,12 @@ class Output:
self.make_parent_dirs()
self.name = f"Output {param_key} from {self.source_app.name}"
+ def exists(self):
+ """Check file exist."""
+ return self.filepath.exists()
+
def make_parent_dirs(self):
+ """Create missing parent directories."""
if not self.filepath.parent.exists():
self.filepath.parent.mkdir(parents=True)
diff --git a/tests/test_core.py b/tests/test_core.py
index 7d231bb..128e75f 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -23,13 +23,7 @@ def test_missing_input_file():
def test_wrong_key():
with pytest.raises(KeyError):
- app = pyotb.BandMath(INPUT, expression="im1b1")
-
-
-# Create dir before write
-def test_write():
- INPUT.write("/tmp/missing_dir/test_write.tif")
- assert INPUT.out.filepath.exists()
+ pyotb.BandMath(INPUT, expression="im1b1")
# OTBObject's properties
@@ -70,7 +64,7 @@ def test_slicer_preserve_dtype():
assert extract.dtype == "uint8"
-# Arithmetics
+# Arithmetic
def test_operation():
op = INPUT / 255 * 128
assert op.exp == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)"
@@ -120,6 +114,12 @@ def test_xy_to_rowcol():
assert INPUT.xy_to_rowcol(760100, 6946210) == (19, 7)
+# Create dir before write
+def test_write():
+ INPUT.write("/tmp/missing_dir/test_write.tif")
+ assert INPUT.out.filepath.exists()
+
+
# BandMath NDVI == RadiometricIndices NDVI ?
@pytest.mark.xfail(reason="Regression in OTB 8.2, waiting for Rémi's patch to be merged.")
def test_ndvi_comparison():
--
GitLab
From edc7c6047ca225c1977e9986b3cbafc31f33d034 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 10 Jan 2023 16:31:02 +0100
Subject: [PATCH 016/399] ENH: use new Output helper functions
---
pyotb/apps.py | 24 ++++++++----------------
1 file changed, 8 insertions(+), 16 deletions(-)
diff --git a/pyotb/apps.py b/pyotb/apps.py
index 2ef6a90..72b3942 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -73,20 +73,13 @@ class App(OTBObject):
"""
files = []
missing = []
- for param in self.outputs:
- filename = self.parameters[param]
- # Remove filename extension
- if '?' in filename:
- filename = filename.split('?')[0]
- path = Path(filename)
- dest = files if path.exists() else missing
- dest.append(str(path.absolute()))
- if missing:
- missing = tuple(missing)
- for filename in missing:
- logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
-
- return files
+ for out in self.outputs:
+ dest = files if out.exists() else missing
+ dest.append(str(out.filepath.absolute()))
+ for filename in missing:
+ logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
+
+ return tuple(files)
class OTBTFApp(App):
@@ -136,8 +129,7 @@ class {name}(App):
"""
for _app in AVAILABLE_APPLICATIONS:
- # Customize the behavior for some OTBTF applications. The user doesn't need to set the env variable
- # `OTB_TF_NSOURCES`, it is handled in pyotb
+ # Customize the behavior for some OTBTF applications. `OTB_TF_NSOURCES` is now handled by pyotb
if _app in ("PatchesExtraction", "TensorflowModelTrain", "TensorflowModelServe"):
exec(_CODE_TEMPLATE.format(name=_app).replace("(App)", "(OTBTFApp)")) # pylint: disable=exec-used
# Default behavior for any OTB application
--
GitLab
From e93a9db047a2f8934e482aeccbccb48167b41cad Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 10 Jan 2023 17:11:24 +0100
Subject: [PATCH 017/399] ENH: move UI code to App def
---
pyotb/apps.py | 34 ++++++++++++++++++
pyotb/core.py | 88 ++++++++++++++++------------------------------
tests/test_core.py | 4 ---
3 files changed, 64 insertions(+), 62 deletions(-)
diff --git a/pyotb/apps.py b/pyotb/apps.py
index 72b3942..bb15f1c 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -63,6 +63,40 @@ def get_available_applications(as_subprocess=False):
class App(OTBObject):
"""Base class for UI related functions, will be subclassed using app name as class name, see CODE_TEMPLATE."""
+ _name = ""
+
+ def __init__(self, *args, **kwargs):
+ """Default App constructor, adds UI specific attributes and functions."""
+ super().__init__(*args, **kwargs)
+ self.description = self.app.GetDocLongDescription()
+
+ @property
+ def name(self):
+ """Application name that will be printed in logs.
+
+ Returns:
+ user's defined name or appname
+
+ """
+ return self._name or self.appname
+
+ @name.setter
+ def name(self, name):
+ """Set custom name.
+
+ Args:
+ name: new name
+
+ """
+ if isinstance(name, str):
+ self._name = name
+ else:
+ raise TypeError(f"{self.name}: bad type ({type(name)}) for application name, only str is allowed")
+
+ @property
+ def outputs(self):
+ """List of application outputs."""
+ return [getattr(self, key) for key in self.out_param_keys if key in self.parameters]
def find_outputs(self):
"""Find output files on disk using path found in parameters.
diff --git a/pyotb/core.py b/pyotb/core.py
index 94b26d8..0943996 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -11,7 +11,6 @@ from .helpers import logger
class OTBObject:
"""Base class that gathers common operations for any OTB application."""
- _name = ""
def __init__(self, appname, *args, frozen=False, quiet=False, image_dic=None, **kwargs):
"""Common constructor for OTB applications. Handles in-memory connection between apps.
@@ -35,7 +34,7 @@ class OTBObject:
"""
self.parameters = {}
- self.appname = appname
+ self.name = self.appname = appname
self.quiet = quiet
self.image_dic = image_dic
self.exports_dic = {}
@@ -46,6 +45,7 @@ class OTBObject:
self.parameters_keys = tuple(self.app.GetParametersKeys())
self.out_param_types = dict(get_out_param_types(self))
self.out_param_keys = tuple(self.out_param_types.keys())
+
if args or kwargs:
self.set_parameters(*args, **kwargs)
self.frozen = frozen
@@ -67,34 +67,6 @@ class OTBObject:
"""Get the name of first output image parameter."""
return key_output(self, "raster")
- @property
- def name(self):
- """Application name that will be printed in logs.
-
- Returns:
- user's defined name or appname
-
- """
- return self._name or self.appname
-
- @name.setter
- def name(self, name):
- """Set custom name.
-
- Args:
- name: new name
-
- """
- if isinstance(name, str):
- self._name = name
- else:
- raise TypeError(f"{self.name}: bad type ({type(name)}) for application name, only str is allowed")
-
- @property
- def outputs(self):
- """List of application outputs."""
- return [getattr(self, key) for key in self.out_param_keys if key in self.parameters]
-
@property
def dtype(self):
"""Expose the pixel type of an output image using numpy convention.
@@ -176,6 +148,26 @@ class OTBObject:
self.parameters.update({**parameters, **otb_params})
self.save_objects()
+ def save_objects(self):
+ """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`.
+
+ This is useful when the key contains reserved characters such as a point eg "io.out"
+ """
+ for key in self.parameters_keys:
+ if key in dir(self.__class__):
+ continue # skip forbidden attribute since it is already used by the class
+ value = self.parameters.get(key) # basic parameters
+ if value is None:
+ try:
+ value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results)
+ except RuntimeError:
+ continue # this is when there is no value for key
+ # Convert output param path to Output object
+ if key in self.out_param_keys:
+ value = Output(self, key, value)
+ # Save attribute
+ setattr(self, key, value)
+
def execute(self):
"""Execute and write to disk if any output parameter has been set during init."""
logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters)
@@ -185,9 +177,9 @@ class OTBObject:
raise Exception(f"{self.name}: error during during app execution") from e
self.frozen = False
logger.debug("%s: execution ended", self.name)
+ self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics
if any(key in self.parameters for key in self.out_param_keys):
- self.flush()
- self.save_objects()
+ self.flush() # auto flush if any output param was provided during app init
def flush(self):
"""Flush data to disk, this is when WriteOutput is actually called."""
@@ -198,26 +190,6 @@ class OTBObject:
logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name)
self.app.ExecuteAndWriteOutput()
- def save_objects(self):
- """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`.
-
- This is useful when the key contains reserved characters such as a point eg "io.out"
- """
- for key in self.parameters_keys:
- if key in dir(OTBObject):
- continue # skip forbidden attribute since it is already used by the class
- value = self.parameters.get(key) # basic parameters
- if value is None:
- try:
- value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results)
- except RuntimeError:
- continue # this is when there is no value for key
- # Convert output param path to Output object
- if key in self.out_param_keys:
- value = Output(self, key, value)
- # Save attribute
- setattr(self, key, value)
-
def write(self, *args, filename_extension="", pixel_type=None, preserve_dtype=False, **kwargs):
"""Set output pixel type and write the output raster files.
@@ -328,7 +300,7 @@ class OTBObject:
elif isinstance(bands, slice):
channels = self.__channels_list_from_slice(bands)
elif not isinstance(bands, list):
- raise TypeError(f"{self.name}: type '{bands}' cannot be interpreted as a valid slicing")
+ raise TypeError(f"{self.name}: type '{type(bands)}' cannot be interpreted as a valid slicing")
if channels:
app.app.Execute()
app.set_parameters({"cl": [f"Channel{n+1}" for n in channels]})
@@ -422,9 +394,8 @@ class OTBObject:
pixel index: (row, col)
"""
spacing_x, _, origin_x, _, spacing_y, origin_y = self.transform
- col = int((x - origin_x) / spacing_x)
- row = int((origin_y - y) / spacing_y)
- return (row, col)
+ row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x
+ return (int(row), int(col))
# Private functions
def __parse_args(self, args):
@@ -494,6 +465,7 @@ class OTBObject:
channels = list(range(0, self.shape[2], step))
return channels
+ # Special functions
def __hash__(self):
"""Override the default behaviour of the hash function.
@@ -994,9 +966,9 @@ class Operation(OTBObject):
# Computing the BandMath or BandMathX app
self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands)
# Init app
- self.name = f'Operation exp="{self.exp}"'
appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX"
super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True)
+ self.name = f'Operation exp="{self.exp}"'
def create_fake_exp(self, operator, inputs, nb_bands=None):
"""Create a 'fake' expression.
@@ -1234,7 +1206,7 @@ class Input(OTBObject):
if not self.filepath.exists():
raise FileNotFoundError(filepath)
super().__init__("ExtractROI", {"in": str(filepath)}, frozen=True)
- self._name = f"Input from {filepath}"
+ self.name = f"Input from {filepath}"
self.propagate_dtype()
self.execute()
diff --git a/tests/test_core.py b/tests/test_core.py
index 128e75f..bd59d70 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -27,10 +27,6 @@ def test_wrong_key():
# OTBObject's properties
-def test_name():
- assert INPUT.name == "Input from tests/image.tif"
- INPUT.name = "Test input"
- assert INPUT.name == "Test input"
def test_key_input():
--
GitLab
From d0125cda7de9e1aed0dfc43adbcfc121a12b6500 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 10 Jan 2023 17:39:25 +0100
Subject: [PATCH 018/399] FIX: tests for xy_to_rowcol and read_values_at_coords
---
pyotb/core.py | 6 +++---
tests/test_core.py | 8 ++++++--
2 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 0943996..e3b9254 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -278,12 +278,12 @@ class OTBObject:
# Set output pixel type
self.app.SetParameterOutputImagePixelType(key, dtype)
- def read_values_at_coords(self, col, row, bands=None):
+ def read_values_at_coords(self, row, col, bands=None):
"""Get pixel value(s) at a given YX coordinates.
Args:
- col: index along X / longitude axis
row: index along Y / latitude axis
+ col: index along X / longitude axis
bands: band number, list or slice to fetch values from
Returns:
@@ -395,7 +395,7 @@ class OTBObject:
"""
spacing_x, _, origin_x, _, spacing_y, origin_y = self.transform
row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x
- return (int(row), int(col))
+ return (abs(int(row)), int(col))
# Private functions
def __parse_args(self, args):
diff --git a/tests/test_core.py b/tests/test_core.py
index bd59d70..10d8ac5 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -102,12 +102,16 @@ def test_app_computeimagestats_sliced():
def test_read_values_at_coords():
assert INPUT[0, 0, 0] == 180
- assert INPUT[10, 20, :] == [196, 188, 172, 255]
+ assert INPUT[10, 20, :] == [207, 192, 172, 255]
# XY => RowCol
def test_xy_to_rowcol():
- assert INPUT.xy_to_rowcol(760100, 6946210) == (19, 7)
+ assert INPUT.xy_to_rowcol(760101, 6945977) == (19, 7)
+
+
+def test_pixel_coords_numpy_equals_otb():
+ assert INPUT[19,7] == list(INPUT.to_numpy()[19,7])
# Create dir before write
--
GitLab
From 663963109aeefe9cca01c6cb289bc078b950e6af Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 10 Jan 2023 18:04:54 +0100
Subject: [PATCH 019/399] ENH: prevent access image properties of app without
raster output
---
pyotb/core.py | 17 ++++++++++-------
tests/test_core.py | 8 ++++++--
2 files changed, 16 insertions(+), 9 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index e3b9254..5d2ed01 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -55,7 +55,7 @@ class OTBObject:
@property
def key_input(self):
"""Get the name of first input parameter, raster > vector > file."""
- return self.key_input_image or key_input(self, "vector") or key_input(self, "file")
+ return key_input(self, "raster") or key_input(self, "vector") or key_input(self, "file")
@property
def key_input_image(self):
@@ -75,11 +75,10 @@ class OTBObject:
dtype: pixel type of the output image
"""
- try:
- enum = self.app.GetParameterOutputImagePixelType(self.key_output_image)
- return self.app.ConvertPixelTypeToNumpy(enum)
- except RuntimeError:
- return None
+ if not self.key_output_image:
+ raise TypeError(f"{self.name}: this application has no raster output")
+ enum = self.app.GetParameterOutputImagePixelType(self.key_output_image)
+ return self.app.ConvertPixelTypeToNumpy(enum)
@property
def shape(self):
@@ -89,6 +88,8 @@ class OTBObject:
shape: (height, width, bands)
"""
+ if not self.key_output_image:
+ raise TypeError(f"{self.name}: this application has no raster output")
width, height = self.app.GetImageSize(self.key_output_image)
bands = self.app.GetImageNbBands(self.key_output_image)
return (height, width, bands)
@@ -100,6 +101,8 @@ class OTBObject:
Returns:
transform: (X spacing, X offset, X origin, Y offset, Y spacing, Y origin)
"""
+ if not self.key_output_image:
+ raise TypeError(f"{self.name}: this application has no raster output")
spacing_x, spacing_y = self.app.GetImageSpacing(self.key_output_image)
origin_x, origin_y = self.app.GetImageOrigin(self.key_output_image)
# Shift image origin since OTB is giving coordinates of pixel center instead of corners
@@ -1303,7 +1306,7 @@ def get_pixel_type(inp):
elif isinstance(inp, (OTBObject)):
pixel_type = inp.GetParameterOutputImagePixelType(inp.key_output_image)
else:
- raise TypeError(f'Could not get the pixel type. Not supported type: {inp}')
+ raise TypeError(f'Could not get the pixel type of {type(inp)} object {inp}')
return pixel_type
diff --git a/tests/test_core.py b/tests/test_core.py
index 10d8ac5..adf74f3 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -27,8 +27,6 @@ def test_wrong_key():
# OTBObject's properties
-
-
def test_key_input():
assert INPUT.key_input == INPUT.key_input_image == "in"
@@ -49,6 +47,12 @@ def test_transform():
assert INPUT.transform == (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0)
+def test_nonraster_property():
+ with pytest.raises(TypeError):
+ pyotb.ReadImageInfo(INPUT).dtype
+
+
+# Slicer
def test_slicer_shape():
extract = INPUT[:50, :60, :3]
assert extract.shape == (50, 60, 3)
--
GitLab
From 0801bdef187174bd1c4c239bc1e610352a81f08e Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 10 Jan 2023 18:23:33 +0100
Subject: [PATCH 020/399] ENH: use function channels_list_from_slice in Slicer
---
pyotb/core.py | 41 ++++++++++++++++++-----------------------
1 file changed, 18 insertions(+), 23 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 5d2ed01..a913512 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -301,7 +301,7 @@ class OTBObject:
bands = self.shape[2] + bands
channels = [bands]
elif isinstance(bands, slice):
- channels = self.__channels_list_from_slice(bands)
+ channels = self.channels_list_from_slice(bands)
elif not isinstance(bands, list):
raise TypeError(f"{self.name}: type '{type(bands)}' cannot be interpreted as a valid slicing")
if channels:
@@ -313,6 +313,22 @@ class OTBObject:
return data[0]
return data
+ def channels_list_from_slice(self, bands):
+ """Get list of channels to read values at, from a slice."""
+ channels = None
+ start, stop, step = bands.start, bands.stop, bands.step
+ if step is None:
+ step = 1
+ if start is not None and stop is not None:
+ channels = list(range(start, stop, step))
+ elif start is not None and stop is None:
+ channels = list(range(start, self.shape[2], step))
+ elif start is None and stop is not None:
+ channels = list(range(0, stop, step))
+ elif start is None and stop is None:
+ channels = list(range(0, self.shape[2], step))
+ return channels
+
def summarize(self):
"""Serialize an object and its pipeline into a dictionary.
@@ -452,22 +468,6 @@ class OTBObject:
else:
self.app.SetParameterValue(key, obj)
- def __channels_list_from_slice(self, bands):
- """Get list of channels to read values at, from a slice."""
- channels = None
- start, stop, step = bands.start, bands.stop, bands.step
- if step is None:
- step = 1
- if start is not None and stop is not None:
- channels = list(range(start, stop, step))
- elif start is not None and stop is None:
- channels = list(range(start, self.shape[2], step))
- elif start is None and stop is not None:
- channels = list(range(0, stop, step))
- elif start is None and stop is None:
- channels = list(range(0, self.shape[2], step))
- return channels
-
# Special functions
def __hash__(self):
"""Override the default behaviour of the hash function.
@@ -869,12 +869,7 @@ class Slicer(OTBObject):
channels = [channels]
# if needed, converting slice to list
elif isinstance(channels, slice):
- channels_start = channels.start if channels.start is not None else 0
- channels_start = channels_start if channels_start >= 0 else nb_channels + channels_start
- channels_end = channels.stop if channels.stop is not None else nb_channels
- channels_end = channels_end if channels_end >= 0 else nb_channels + channels_end
- channels_step = channels.step if channels.step is not None else 1
- channels = range(channels_start, channels_end, channels_step)
+ channels = self.channels_list_from_slice(channels)
elif isinstance(channels, tuple):
channels = list(channels)
elif not isinstance(channels, list):
--
GitLab
From aa810bec48a616b89d31a64099cfcae5b795137f Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 10 Jan 2023 18:42:35 +0100
Subject: [PATCH 021/399] FIX: add missing negative band index to
channels_list_from_slice
---
pyotb/core.py | 22 +++++++++++-----------
tests/test_core.py | 4 ++++
2 files changed, 15 insertions(+), 11 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index a913512..ff7b6ef 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -315,19 +315,19 @@ class OTBObject:
def channels_list_from_slice(self, bands):
"""Get list of channels to read values at, from a slice."""
- channels = None
+ channels, nb_channels = None, self.shape[2]
start, stop, step = bands.start, bands.stop, bands.step
- if step is None:
- step = 1
+ start = nb_channels + start if isinstance(start, int) and start < 0 else start
+ stop = nb_channels + stop if isinstance(stop, int) and stop < 0 else stop
+ step = 1 if step is None else step
if start is not None and stop is not None:
- channels = list(range(start, stop, step))
- elif start is not None and stop is None:
- channels = list(range(start, self.shape[2], step))
- elif start is None and stop is not None:
- channels = list(range(0, stop, step))
- elif start is None and stop is None:
- channels = list(range(0, self.shape[2], step))
- return channels
+ return list(range(start, stop, step))
+ if start is not None and stop is None:
+ return list(range(start, nb_channels, step))
+ if start is None and stop is not None:
+ return list(range(0, stop, step))
+ if start is None and stop is None:
+ return list(range(0, nb_channels, step))
def summarize(self):
"""Serialize an object and its pipeline into a dictionary.
diff --git a/tests/test_core.py b/tests/test_core.py
index adf74f3..2d99bd2 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -64,6 +64,10 @@ def test_slicer_preserve_dtype():
assert extract.dtype == "uint8"
+def test_slicer_negative_band_index():
+ assert INPUT[:50, :60, :-2].shape == (50, 60, 2)
+
+
# Arithmetic
def test_operation():
op = INPUT / 255 * 128
--
GitLab
From 7bb9aad4c13aa895285bfc362826986dcfa63023 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 10 Jan 2023 18:50:46 +0100
Subject: [PATCH 022/399] STYLE: linting
---
pyotb/core.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index ff7b6ef..65a05f6 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -315,7 +315,7 @@ class OTBObject:
def channels_list_from_slice(self, bands):
"""Get list of channels to read values at, from a slice."""
- channels, nb_channels = None, self.shape[2]
+ nb_channels = self.shape[2]
start, stop, step = bands.start, bands.stop, bands.step
start = nb_channels + start if isinstance(start, int) and start < 0 else start
stop = nb_channels + stop if isinstance(stop, int) and stop < 0 else stop
@@ -328,6 +328,7 @@ class OTBObject:
return list(range(0, stop, step))
if start is None and stop is None:
return list(range(0, nb_channels, step))
+ raise ValueError(f"{self.name}: '{bands}' cannot be interpreted as valid slicing.")
def summarize(self):
"""Serialize an object and its pipeline into a dictionary.
--
GitLab
From 5d4be4d77a3bef9c2d4c1a0953e6d402fa64d21e Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 11 Jan 2023 11:58:17 +0100
Subject: [PATCH 023/399] ENH: autoflush during app init if any output param
was provided
---
pyotb/core.py | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 65a05f6..e9e2018 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -35,6 +35,7 @@ class OTBObject:
"""
self.parameters = {}
self.name = self.appname = appname
+ self.frozen = frozen
self.quiet = quiet
self.image_dic = image_dic
self.exports_dic = {}
@@ -45,12 +46,12 @@ class OTBObject:
self.parameters_keys = tuple(self.app.GetParametersKeys())
self.out_param_types = dict(get_out_param_types(self))
self.out_param_keys = tuple(self.out_param_types.keys())
-
if args or kwargs:
self.set_parameters(*args, **kwargs)
- self.frozen = frozen
- if not frozen:
+ if not self.frozen:
self.execute()
+ if any(key in self.parameters for key in self.out_param_keys):
+ self.flush() # auto flush if any output param was provided during app init
@property
def key_input(self):
@@ -181,8 +182,6 @@ class OTBObject:
self.frozen = False
logger.debug("%s: execution ended", self.name)
self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics
- if any(key in self.parameters for key in self.out_param_keys):
- self.flush() # auto flush if any output param was provided during app init
def flush(self):
"""Flush data to disk, this is when WriteOutput is actually called."""
@@ -246,7 +245,6 @@ class OTBObject:
if key in dtypes:
self.propagate_dtype(key, dtypes[key])
self.set_parameters({key: output_filename})
-
self.flush()
def propagate_dtype(self, target_key=None, dtype=None):
--
GitLab
From af002a4502234208b0b11d613804b3ca39c323de Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 11 Jan 2023 12:17:05 +0100
Subject: [PATCH 024/399] STYLE: whitespaces and comments
---
pyotb/core.py | 42 ++++++++++++++----------------------------
1 file changed, 14 insertions(+), 28 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index e9e2018..cf13497 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -219,6 +219,7 @@ class OTBObject:
logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg)
elif isinstance(arg, str) and self.key_output_image:
kwargs.update({self.key_output_image: arg})
+
# Append filename extension to filenames
if filename_extension:
logger.debug("%s: using extended filename for outputs: %s", self.name, filename_extension)
@@ -227,6 +228,7 @@ class OTBObject:
for key, value in kwargs.items():
if self.out_param_types[key] == 'raster' and '?' not in value:
kwargs[key] = value + filename_extension
+
# Manage output pixel types
dtypes = {}
if pixel_type:
@@ -240,7 +242,8 @@ class OTBObject:
dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()}
elif preserve_dtype:
self.propagate_dtype() # all outputs will have the same type as the main input raster
- # Apply parameters
+
+ # Set parameters and flush to disk
for key, output_filename in kwargs.items():
if key in dtypes:
self.propagate_dtype(key, dtypes[key])
@@ -270,13 +273,11 @@ class OTBObject:
except (TypeError, RuntimeError):
logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param)
return
-
if target_key:
keys = [target_key]
else:
keys = [k for k in self.out_param_keys if self.out_param_types[k] == "raster"]
for key in keys:
- # Set output pixel type
self.app.SetParameterOutputImagePixelType(key, dtype)
def read_values_at_coords(self, row, col, bands=None):
@@ -292,7 +293,7 @@ class OTBObject:
"""
channels = []
- app = OTBObject("PixelValue", self, coordx=col, coordy=row, frozen=False, quiet=True)
+ app = OTBObject("PixelValue", self, coordx=col, coordy=row, frozen=True, quiet=True)
if bands is not None:
if isinstance(bands, int):
if bands < 0:
@@ -342,7 +343,6 @@ class OTBObject:
params[k] = p.summarize()
elif isinstance(p, list): # parameter list
params[k] = [pi.summarize() if isinstance(pi, OTBObject) else pi for pi in p]
-
return {"name": self.name, "parameters": params}
def export(self, key=None, preserve_dtype=True):
@@ -441,7 +441,7 @@ class OTBObject:
return
if key not in self.parameters_keys:
raise KeyError(
- f"{self.name}: parameter '{key}' was not recognized. " f"Available keys are {self.parameters_keys}"
+ f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}"
)
# Single-parameter cases
if isinstance(obj, OTBObject):
@@ -819,12 +819,10 @@ class OTBObject:
else:
logger.debug(type(self))
return NotImplemented
-
# Performing the numpy operation
result_array = ufunc(*arrays, **kwargs)
result_dic = image_dic
result_dic["array"] = result_array
-
# Importing back to OTB, pass the result_dic just to keep reference
app = OTBObject("ExtractROI", image_dic=result_dic, frozen=True, quiet=True)
if result_array.shape[2] == 1:
@@ -833,7 +831,6 @@ class OTBObject:
app.ImportVectorImage("in", result_dic)
app.execute()
return app
-
return NotImplemented
@@ -858,6 +855,7 @@ class Slicer(OTBObject):
self.name = "Slicer"
self.rows, self.cols = rows, cols
parameters = {}
+
# Channel slicing
if channels != slice(None, None, None):
# Trigger source app execution if needed
@@ -873,7 +871,6 @@ class Slicer(OTBObject):
channels = list(channels)
elif not isinstance(channels, list):
raise ValueError(f"Invalid type for channels, should be int, slice or list of bands. : {channels}")
-
# Change the potential negative index values to reverse index
channels = [c if c >= 0 else nb_channels + c for c in channels]
parameters.update({"cl": [f"Channel{i + 1}" for i in channels]})
@@ -957,12 +954,10 @@ class Operation(OTBObject):
self.im_dic[str(inp)] = f"im{self.im_count}"
mapping_str_to_input[str(inp)] = inp
self.im_count += 1
-
# Getting unique image inputs, in the order im1, im2, im3 ...
self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)]
- # Computing the BandMath or BandMathX app
self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands)
- # Init app
+ # Execute app
appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX"
super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True)
self.name = f'Operation exp="{self.exp}"'
@@ -980,9 +975,8 @@ class Operation(OTBObject):
"""
self.inputs.clear()
self.nb_channels.clear()
-
logger.debug("%s, %s", operator, inputs)
- # this is when we use the ternary operator with `pyotb.where` function. The output nb of bands is already known
+ # This is when we use the ternary operator with `pyotb.where` function. The output nb of bands is already known
if operator == "?" and nb_bands:
pass
# For any other operations, the output number of bands is the same as inputs
@@ -1002,10 +996,10 @@ class Operation(OTBObject):
for i, band in enumerate(range(1, nb_bands + 1)):
fake_exps = []
for k, inp in enumerate(inputs):
- # Generating the fake expression of the current input
+ # Generating the fake expression of the current input,
# this is a special case for the condition of the ternary operator `cond ? x : y`
if len(inputs) == 3 and k == 0:
- # when cond is monoband whereas the result is multiband, we expand the cond to multiband
+ # When cond is monoband whereas the result is multiband, we expand the cond to multiband
if nb_bands != inp.shape[2]:
cond_band = 1
else:
@@ -1013,8 +1007,8 @@ class Operation(OTBObject):
fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp(
inp, cond_band, keep_logical=True
)
- # any other input
else:
+ # Any other input
fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp(
inp, band, keep_logical=False
)
@@ -1033,7 +1027,6 @@ class Operation(OTBObject):
fake_exp = f"({fake_exps[0]} {operator} {fake_exps[1]})"
elif len(inputs) == 3 and operator == "?": # this is only for ternary expression
fake_exp = f"({fake_exps[0]} ? {fake_exps[1]} : {fake_exps[2]})"
-
self.fake_exp_bands.append(fake_exp)
def get_real_exp(self, fake_exp_bands):
@@ -1052,13 +1045,11 @@ class Operation(OTBObject):
for one_band_fake_exp in fake_exp_bands:
one_band_exp = one_band_fake_exp
for inp in self.inputs:
- # replace the name of in-memory object (e.g. '<pyotb.App object>b1' by 'im1b1')
+ # Replace the name of in-memory object (e.g. '<pyotb.App object>b1' by 'im1b1')
one_band_exp = one_band_exp.replace(str(inp), self.im_dic[str(inp)])
exp_bands.append(one_band_exp)
-
# Form the final expression (e.g. 'im1b1 + 1; im1b2 + 1')
exp = ";".join(exp_bands)
-
return exp_bands, exp
@staticmethod
@@ -1088,7 +1079,7 @@ class Operation(OTBObject):
inputs = x.input.inputs
nb_channels = x.input.nb_channels
elif isinstance(x.input, Operation):
- # keep only one band of the expression
+ # Keep only one band of the expression
fake_exp = x.input.fake_exp_bands[x.one_band_sliced - 1]
inputs = x.input.inputs
nb_channels = x.input.nb_channels
@@ -1117,7 +1108,6 @@ class Operation(OTBObject):
inputs = [x]
# Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1')
fake_exp = f"{x}b{band}"
-
return fake_exp, inputs, nb_channels
def __str__(self):
@@ -1177,11 +1167,9 @@ class LogicalOperation(Operation):
if i == 0 and corresp_inputs and nb_channels:
self.inputs.extend(corresp_inputs)
self.nb_channels.update(nb_channels)
-
# We create here the "fake" expression. For example, for a BandMathX expression such as 'im1 > im2',
# the logical fake expression stores the expression "str(input1) > str(input2)"
logical_fake_exp = f"({fake_exps[0]} {operator} {fake_exps[1]})"
-
# We keep the logical expression, useful if later combined with other logical operations
self.logical_fake_exp_bands.append(logical_fake_exp)
# We create a valid BandMath expression, e.g. "str(input1) > str(input2) ? 1 : 0"
@@ -1284,7 +1272,6 @@ def get_pixel_type(inp):
"""
if isinstance(inp, str):
- # Executing the app, without printing its log
try:
info = OTBObject("ReadImageInfo", inp, quiet=True)
except Exception as info_err: # this happens when we pass a str that is not a filepath
@@ -1301,7 +1288,6 @@ def get_pixel_type(inp):
pixel_type = inp.GetParameterOutputImagePixelType(inp.key_output_image)
else:
raise TypeError(f'Could not get the pixel type of {type(inp)} object {inp}')
-
return pixel_type
--
GitLab
From 712014ed5b691e413a4ff2c7e89985fce501b2f7 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 11 Jan 2023 12:32:42 +0100
Subject: [PATCH 025/399] STYLE: whitespaces
---
pyotb/apps.py | 14 ++++----------
1 file changed, 4 insertions(+), 10 deletions(-)
diff --git a/pyotb/apps.py b/pyotb/apps.py
index bb15f1c..2a85fc6 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -5,8 +5,8 @@ import sys
from pathlib import Path
import otbApplication as otb
-from .helpers import logger
from .core import OTBObject
+from .helpers import logger
def get_available_applications(as_subprocess=False):
@@ -32,7 +32,6 @@ def get_available_applications(as_subprocess=False):
env["OTB_LOGGER_LEVEL"] = "CRITICAL" # in order to suppress warnings while listing applications
pycmd = "import otbApplication; print(otbApplication.Registry.GetAvailableApplications())"
cmd_args = [sys.executable, "-c", pycmd]
-
try:
import subprocess # pylint: disable=import-outside-toplevel
params = {"env": env, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE}
@@ -48,7 +47,6 @@ def get_available_applications(as_subprocess=False):
logger.debug("Failed to call subprocess")
except (ValueError, SyntaxError, AssertionError):
logger.debug("Failed to decode output or convert to tuple:\nstdout=%s\nstderr=%s", stdout, stderr)
-
if not app_list:
logger.info("Failed to list applications in an independent process. Falling back to local python import")
# Find applications using the normal way
@@ -56,7 +54,6 @@ def get_available_applications(as_subprocess=False):
app_list = otb.Registry.GetAvailableApplications()
if not app_list:
raise SystemExit("Unable to load applications. Set env variable OTB_APPLICATION_PATH and try again.")
-
logger.info("Successfully loaded %s OTB applications", len(app_list))
return app_list
@@ -88,10 +85,9 @@ class App(OTBObject):
name: new name
"""
- if isinstance(name, str):
- self._name = name
- else:
+ if not isinstance(name, str):
raise TypeError(f"{self.name}: bad type ({type(name)}) for application name, only str is allowed")
+ self._name = name
@property
def outputs(self):
@@ -105,14 +101,12 @@ class App(OTBObject):
list of files found on disk
"""
- files = []
- missing = []
+ files, missing = [], []
for out in self.outputs:
dest = files if out.exists() else missing
dest.append(str(out.filepath.absolute()))
for filename in missing:
logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
-
return tuple(files)
--
GitLab
From 7e2a2c1a89569175e8914082cdc7e4cd6d4fc0bc Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 11 Jan 2023 14:56:44 +0100
Subject: [PATCH 026/399] WIP: proposition to reduce code complexity, simplify
app naming
---
pyotb/apps.py | 22 ----------------------
pyotb/core.py | 18 ++++++++----------
2 files changed, 8 insertions(+), 32 deletions(-)
diff --git a/pyotb/apps.py b/pyotb/apps.py
index 2a85fc6..b3f9fd7 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -67,28 +67,6 @@ class App(OTBObject):
super().__init__(*args, **kwargs)
self.description = self.app.GetDocLongDescription()
- @property
- def name(self):
- """Application name that will be printed in logs.
-
- Returns:
- user's defined name or appname
-
- """
- return self._name or self.appname
-
- @name.setter
- def name(self, name):
- """Set custom name.
-
- Args:
- name: new name
-
- """
- if not isinstance(name, str):
- raise TypeError(f"{self.name}: bad type ({type(name)}) for application name, only str is allowed")
- self._name = name
-
@property
def outputs(self):
"""List of application outputs."""
diff --git a/pyotb/core.py b/pyotb/core.py
index cf13497..380602d 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -12,11 +12,11 @@ from .helpers import logger
class OTBObject:
"""Base class that gathers common operations for any OTB application."""
- def __init__(self, appname, *args, frozen=False, quiet=False, image_dic=None, **kwargs):
+ def __init__(self, name, *args, frozen=False, quiet=False, image_dic=None, **kwargs):
"""Common constructor for OTB applications. Handles in-memory connection between apps.
Args:
- appname: name of the app, e.g. 'BandMath'
+ name: name of the app, e.g. 'BandMath'
*args: used for passing application parameters. Can be :
- dictionary containing key-arguments enumeration. Useful when a key is python-reserved
(e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
@@ -34,15 +34,13 @@ class OTBObject:
"""
self.parameters = {}
- self.name = self.appname = appname
+ self.name = name
self.frozen = frozen
self.quiet = quiet
self.image_dic = image_dic
self.exports_dic = {}
- if quiet:
- self.app = otb.Registry.CreateApplicationWithoutLogger(appname)
- else:
- self.app = otb.Registry.CreateApplication(appname)
+ self.app = otb.Registry.CreateApplicationWithoutLogger(name) if quiet \
+ else otb.Registry.CreateApplication(name)
self.parameters_keys = tuple(self.app.GetParametersKeys())
self.out_param_types = dict(get_out_param_types(self))
self.out_param_keys = tuple(self.out_param_types.keys())
@@ -479,7 +477,7 @@ class OTBObject:
def __str__(self):
"""Return a nice string representation with object id."""
- return f"<pyotb.App {self.appname} object id {id(self)}>"
+ return f"<pyotb.App {self.name} object id {id(self)}>"
def __getattr__(self, name):
"""This method is called when the default attribute access fails.
@@ -958,8 +956,8 @@ class Operation(OTBObject):
self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)]
self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands)
# Execute app
- appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX"
- super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True)
+ name = "BandMath" if len(self.exp_bands) == 1 else "BandMathX"
+ super().__init__(name, il=self.unique_inputs, exp=self.exp, quiet=True)
self.name = f'Operation exp="{self.exp}"'
def create_fake_exp(self, operator, inputs, nb_bands=None):
--
GitLab
From 277579963121dfd0b9bc979a4228836618958809 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 11 Jan 2023 15:34:25 +0100
Subject: [PATCH 027/399] FIX: Input back working with /vsiXX/ resources
---
pyotb/core.py | 30 +++++++++++++++---------------
1 file changed, 15 insertions(+), 15 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 380602d..532c8f3 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -39,8 +39,8 @@ class OTBObject:
self.quiet = quiet
self.image_dic = image_dic
self.exports_dic = {}
- self.app = otb.Registry.CreateApplicationWithoutLogger(name) if quiet \
- else otb.Registry.CreateApplication(name)
+ create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
+ self.app = create(name)
self.parameters_keys = tuple(self.app.GetParametersKeys())
self.out_param_types = dict(get_out_param_types(self))
self.out_param_keys = tuple(self.out_param_types.keys())
@@ -68,7 +68,7 @@ class OTBObject:
@property
def dtype(self):
- """Expose the pixel type of an output image using numpy convention.
+ """Expose the pixel type of output image using numpy convention.
Returns:
dtype: pixel type of the output image
@@ -91,7 +91,7 @@ class OTBObject:
raise TypeError(f"{self.name}: this application has no raster output")
width, height = self.app.GetImageSize(self.key_output_image)
bands = self.app.GetImageNbBands(self.key_output_image)
- return (height, width, bands)
+ return height, width, bands
@property
def transform(self):
@@ -106,7 +106,7 @@ class OTBObject:
origin_x, origin_y = self.app.GetImageOrigin(self.key_output_image)
# Shift image origin since OTB is giving coordinates of pixel center instead of corners
origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2
- return (spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y)
+ return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y
def set_parameters(self, *args, **kwargs):
"""Set some parameters of the app.
@@ -344,10 +344,12 @@ class OTBObject:
return {"name": self.name, "parameters": params}
def export(self, key=None, preserve_dtype=True):
- """Export a specific output image as numpy array and store it in object's exports_dic.
+ """Export a specific output image as numpy array and store it in object exports_dic.
Args:
key: parameter key to export, if None then the default one will be used
+ preserve_dtype: when set to True, the numpy array is converted to the same pixel type as
+ the OTBObject first output. Default is True
Returns:
the exported numpy array
@@ -367,7 +369,7 @@ class OTBObject:
Args:
key: the output parameter name to export as numpy array
preserve_dtype: when set to True, the numpy array is converted to the same pixel type as
- the OTBObject first output. Default is True.
+ the OTBObject first output. Default is True
copy: whether to copy the output array, default is False
required to True if preserve_dtype is False and the source app reference is lost
@@ -1178,24 +1180,22 @@ class LogicalOperation(Operation):
class Input(OTBObject):
"""Class for transforming a filepath to pyOTB object."""
- def __init__(self, filepath):
+ def __init__(self, path):
"""Default constructor.
Args:
- filepath: the path of an input image
+ path: Anything supported by GDAL (local file on the filesystem, remote resource e.g. /vsicurl/.., etc.)
"""
- self.filepath = Path(filepath)
- if not self.filepath.exists():
- raise FileNotFoundError(filepath)
- super().__init__("ExtractROI", {"in": str(filepath)}, frozen=True)
- self.name = f"Input from {filepath}"
+ self.path = path
+ super().__init__("ExtractROI", {"in": path}, frozen=True)
+ self.name = f"Input from {path}"
self.propagate_dtype()
self.execute()
def __str__(self):
"""Return a nice string representation with file path."""
- return f"<pyotb.Input object from {self.filepath}>"
+ return f"<pyotb.Input object from {self.path}>"
class Output:
--
GitLab
From 68ef3d21b3752ee3bf472b92d8adab2cb5cac80b Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 11 Jan 2023 15:34:45 +0100
Subject: [PATCH 028/399] FIX: remove file missing test
---
tests/test_core.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/tests/test_core.py b/tests/test_core.py
index 2d99bd2..f053b2f 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -17,7 +17,7 @@ def test_parameters():
# Catch errors before exec
def test_missing_input_file():
- with pytest.raises(FileNotFoundError):
+ with pytest.raises(RuntimeError):
pyotb.Input("missing_file.tif")
@@ -129,7 +129,6 @@ def test_write():
# BandMath NDVI == RadiometricIndices NDVI ?
-@pytest.mark.xfail(reason="Regression in OTB 8.2, waiting for Rémi's patch to be merged.")
def test_ndvi_comparison():
ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (INPUT[:, :, -1] + INPUT[:, :, 0])
ndvi_indices = pyotb.RadiometricIndices(INPUT, {"list": "Vegetation:NDVI", "channels.red": 1, "channels.nir": 4})
--
GitLab
From 45e32e849579391020da1e9ef05d496f4a216071 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 11 Jan 2023 15:35:14 +0100
Subject: [PATCH 029/399] ENH: test_serialization can work with any input image
path
---
tests/test_serialization.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/tests/test_serialization.py b/tests/test_serialization.py
index 2e0ce7f..e64f2f1 100644
--- a/tests/test_serialization.py
+++ b/tests/test_serialization.py
@@ -12,7 +12,7 @@ def test_pipeline_simple():
summary = app3.summarize()
reference = {'name': 'ManageNoData', 'parameters': {'in': {
'name': 'OrthoRectification', 'parameters': {'io.in': {
- 'name': 'BandMath', 'parameters': {'il': ('tests/image.tif',), 'exp': 'im1b1'}},
+ 'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}},
'map': 'utm',
'outputs.isotropic': True}},
'mode': 'buildmask'}}
@@ -28,12 +28,12 @@ def test_pipeline_diamond():
summary = app4.summarize()
reference = {'name': 'BandMathX', 'parameters': {'il': [
{'name': 'OrthoRectification', 'parameters': {'io.in': {
- 'name': 'BandMath', 'parameters': {'il': ('tests/image.tif',), 'exp': 'im1b1'}},
+ 'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}},
'map': 'utm',
'outputs.isotropic': True}},
{'name': 'ManageNoData', 'parameters': {'in': {
'name': 'OrthoRectification', 'parameters': {
- 'io.in': {'name': 'BandMath', 'parameters': {'il': ('tests/image.tif',), 'exp': 'im1b1'}},
+ 'io.in': {'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}},
'map': 'utm',
'outputs.isotropic': True}},
'mode': 'buildmask'}}
--
GitLab
From 93eb844b398c46142a4d75756c699d79916eb8c6 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 11 Jan 2023 15:47:31 +0100
Subject: [PATCH 030/399] FIX: remove file missing test
---
tests/test_core.py | 6 ------
1 file changed, 6 deletions(-)
diff --git a/tests/test_core.py b/tests/test_core.py
index f053b2f..2eadef7 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -15,12 +15,6 @@ def test_parameters():
assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304)
-# Catch errors before exec
-def test_missing_input_file():
- with pytest.raises(RuntimeError):
- pyotb.Input("missing_file.tif")
-
-
def test_wrong_key():
with pytest.raises(KeyError):
pyotb.BandMath(INPUT, expression="im1b1")
--
GitLab
From 210b3f70fb9732d0b22b0d222c5b2b26031c005e Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 11 Jan 2023 17:20:06 +0100
Subject: [PATCH 031/399] REFAC: simplify input/output keys accessors
---
pyotb/apps.py | 8 ++--
pyotb/core.py | 105 ++++++++++++++++++++++----------------------------
2 files changed, 50 insertions(+), 63 deletions(-)
diff --git a/pyotb/apps.py b/pyotb/apps.py
index b3f9fd7..5e30d0e 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -68,9 +68,9 @@ class App(OTBObject):
self.description = self.app.GetDocLongDescription()
@property
- def outputs(self):
- """List of application outputs."""
- return [getattr(self, key) for key in self.out_param_keys if key in self.parameters]
+ def used_outputs(self):
+ """List of used application outputs."""
+ return [getattr(self, key) for key in self.out_param_types if key in self.parameters]
def find_outputs(self):
"""Find output files on disk using path found in parameters.
@@ -80,7 +80,7 @@ class App(OTBObject):
"""
files, missing = [], []
- for out in self.outputs:
+ for out in self.used_outputs:
dest = files if out.exists() else missing
dest.append(str(out.filepath.absolute()))
for filename in missing:
diff --git a/pyotb/core.py b/pyotb/core.py
index 532c8f3..bbd681d 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -42,29 +42,45 @@ class OTBObject:
create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
self.app = create(name)
self.parameters_keys = tuple(self.app.GetParametersKeys())
- self.out_param_types = dict(get_out_param_types(self))
- self.out_param_keys = tuple(self.out_param_types.keys())
+ # Output parameters types
+ self.all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys}
+ self.out_param_types = {k: v for k, v in self.all_param_types.items()
+ if v in (otb.ParameterType_OutputImage,
+ otb.ParameterType_OutputVectorData,
+ otb.ParameterType_OutputFilename)}
if args or kwargs:
self.set_parameters(*args, **kwargs)
if not self.frozen:
self.execute()
- if any(key in self.parameters for key in self.out_param_keys):
+ if any(key in self.parameters for key in self.parameters_keys):
self.flush() # auto flush if any output param was provided during app init
+ def get_first_key(self, param_types):
+ """Get the first output param key for specific file types."""
+ for key, param_type in sorted(self.all_param_types.items()):
+ if param_type in param_types:
+ return key
+ return None
+
@property
def key_input(self):
"""Get the name of first input parameter, raster > vector > file."""
- return key_input(self, "raster") or key_input(self, "vector") or key_input(self, "file")
+ return self.get_first_key(param_types=[otb.ParameterType_InputImage,
+ otb.ParameterType_InputImageList]) \
+ or self.get_first_key(param_types=[otb.ParameterType_InputVectorData,
+ otb.ParameterType_InputVectorDataList]) \
+ or self.get_first_key(param_types=[otb.ParameterType_InputFilename,
+ otb.ParameterType_InputFilenameList])
@property
def key_input_image(self):
"""Get the name of first input image parameter."""
- return key_input(self, "raster")
+ return self.get_first_key(param_types=[otb.ParameterType_InputImage, otb.ParameterType_InputImageList])
@property
def key_output_image(self):
"""Get the name of first output image parameter."""
- return key_output(self, "raster")
+ return self.get_first_key(param_types=[otb.ParameterType_OutputImage])
@property
def dtype(self):
@@ -165,7 +181,7 @@ class OTBObject:
except RuntimeError:
continue # this is when there is no value for key
# Convert output param path to Output object
- if key in self.out_param_keys:
+ if key in self.out_param_types:
value = Output(self, key, value)
# Save attribute
setattr(self, key, value)
@@ -224,7 +240,7 @@ class OTBObject:
if not filename_extension.startswith("?"):
filename_extension = "?" + filename_extension
for key, value in kwargs.items():
- if self.out_param_types[key] == 'raster' and '?' not in value:
+ if self.out_param_types[key] == otb.ParameterType_OutputImage and '?' not in value:
kwargs[key] = value + filename_extension
# Manage output pixel types
@@ -234,7 +250,7 @@ class OTBObject:
type_name = self.app.ConvertPixelTypeToNumpy(parse_pixel_type(pixel_type))
logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name)
for key in kwargs:
- if self.out_param_types.get(key) == "raster":
+ if self.out_param_types[key] == otb.ParameterType_OutputImage:
dtypes[key] = parse_pixel_type(pixel_type)
elif isinstance(pixel_type, dict):
dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()}
@@ -274,7 +290,7 @@ class OTBObject:
if target_key:
keys = [target_key]
else:
- keys = [k for k in self.out_param_keys if self.out_param_types[k] == "raster"]
+ keys = [k for k, v in self.out_param_types.items() if v == otb.ParameterType_OutputImage]
for key in keys:
self.app.SetParameterOutputImagePixelType(key, dtype)
@@ -303,7 +319,7 @@ class OTBObject:
raise TypeError(f"{self.name}: type '{type(bands)}' cannot be interpreted as a valid slicing")
if channels:
app.app.Execute()
- app.set_parameters({"cl": [f"Channel{n+1}" for n in channels]})
+ app.set_parameters({"cl": [f"Channel{n + 1}" for n in channels]})
app.execute()
data = literal_eval(app.app.GetParameterString("value"))
if len(channels) == 1:
@@ -413,7 +429,7 @@ class OTBObject:
"""
spacing_x, _, origin_x, _, spacing_y, origin_y = self.transform
row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x
- return (abs(int(row)), int(col))
+ return abs(int(row)), int(col)
# Private functions
def __parse_args(self, args):
@@ -1122,6 +1138,7 @@ class LogicalOperation(Operation):
logical expression (e.g. "im1b1 > 0")
"""
+
def __init__(self, operator, *inputs, nb_bands=None):
"""Constructor for a LogicalOperation object.
@@ -1273,16 +1290,25 @@ def get_pixel_type(inp):
try:
info = OTBObject("ReadImageInfo", inp, quiet=True)
except Exception as info_err: # this happens when we pass a str that is not a filepath
- raise TypeError(f'Could not get the pixel type of `{inp}`. Not a filepath or wrong filepath') from info_err
+ raise TypeError(f"Could not get the pixel type of `{inp}`. Not a filepath or wrong filepath") from info_err
datatype = info.GetParameterString("datatype") # which is such as short, float...
if not datatype:
- raise Exception(f'Unable to read pixel type of image {inp}')
- datatype_to_pixeltype = {'unsigned_char': 'uint8', 'short': 'int16', 'unsigned_short': 'uint16',
- 'int': 'int32', 'unsigned_int': 'uint32', 'long': 'int32', 'ulong': 'uint32',
- 'float': 'float', 'double': 'double'}
- pixel_type = datatype_to_pixeltype[datatype]
- pixel_type = getattr(otb, f'ImagePixelType_{pixel_type}')
- elif isinstance(inp, (OTBObject)):
+ raise TypeError(f"Unable to read pixel type of image {inp}")
+ datatype_to_pixeltype = {
+ 'unsigned_char': 'uint8',
+ 'short': 'int16',
+ 'unsigned_short': 'uint16',
+ 'int': 'int32',
+ 'unsigned_int': 'uint32',
+ 'long': 'int32',
+ 'ulong': 'uint32',
+ 'float': 'float',
+ 'double': 'double'
+ }
+ if datatype not in datatype_to_pixeltype:
+ raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}")
+ pixel_type = getattr(otb, f'ImagePixelType_{datatype_to_pixeltype[datatype]}')
+ elif isinstance(inp, OTBObject):
pixel_type = inp.GetParameterOutputImagePixelType(inp.key_output_image)
else:
raise TypeError(f'Could not get the pixel type of {type(inp)} object {inp}')
@@ -1324,45 +1350,6 @@ def is_key_images_list(pyotb_app, key):
)
-def get_out_param_types(pyotb_app):
- """Get output parameter data type (raster, vector, file)."""
- outfile_types = {
- otb.ParameterType_OutputImage: "raster",
- otb.ParameterType_OutputVectorData: "vector",
- otb.ParameterType_OutputFilename: "file",
- }
- for k in pyotb_app.parameters_keys:
- t = pyotb_app.app.GetParameterType(k)
- if t in outfile_types:
- yield k, outfile_types[t]
-
-
def get_out_images_param_keys(app):
"""Return every output parameter keys of an OTB app."""
return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage]
-
-
-def key_input(pyotb_app, file_type):
- """Get the first input param key for a specific file type."""
- types = {
- "raster": (otb.ParameterType_InputImage, otb.ParameterType_InputImageList),
- "vector": (otb.ParameterType_InputVectorData, otb.ParameterType_InputVectorDataList),
- "file": (otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList)
- }
- for key in pyotb_app.parameters_keys:
- if pyotb_app.app.GetParameterType(key) in types[file_type]:
- return key
- return None
-
-
-def key_output(pyotb_app, file_type):
- """Get the first output param key for a specific file type."""
- types = {
- "raster": otb.ParameterType_OutputImage,
- "vector": otb.ParameterType_OutputVectorData,
- "file": otb.ParameterType_OutputFilename
- }
- for key in pyotb_app.parameters_keys:
- if pyotb_app.app.GetParameterType(key) == types[file_type]:
- return key
- return None
--
GitLab
From 4f8d363cb84e2e26ab72102c61723c6bb786066f Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 11 Jan 2023 17:05:44 +0000
Subject: [PATCH 032/399] Apply 1 suggestion(s) to 1 file(s)
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index bbd681d..86bfb9c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -52,7 +52,7 @@ class OTBObject:
self.set_parameters(*args, **kwargs)
if not self.frozen:
self.execute()
- if any(key in self.parameters for key in self.parameters_keys):
+ if any(key in self.parameters for key in self.out_param_types):
self.flush() # auto flush if any output param was provided during app init
def get_first_key(self, param_types):
--
GitLab
From 9638ef14dbff9a4afc842bbc831394aff1a0d5ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr>
Date: Fri, 13 Jan 2023 10:40:27 +0000
Subject: [PATCH 033/399] Typing hints in functions
---
pyotb/apps.py | 15 +++---
pyotb/core.py | 123 ++++++++++++++++++++++++---------------------
pyotb/functions.py | 18 ++++---
pyotb/helpers.py | 16 +++---
4 files changed, 92 insertions(+), 80 deletions(-)
diff --git a/pyotb/apps.py b/pyotb/apps.py
index 5e30d0e..a2af7ed 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
"""Search for OTB (set env if necessary), subclass core.App for each available application."""
+from __future__ import annotations
import os
import sys
from pathlib import Path
@@ -9,7 +10,7 @@ from .core import OTBObject
from .helpers import logger
-def get_available_applications(as_subprocess=False):
+def get_available_applications(as_subprocess: bool = False) -> list[str]:
"""Find available OTB applications.
Args:
@@ -68,11 +69,11 @@ class App(OTBObject):
self.description = self.app.GetDocLongDescription()
@property
- def used_outputs(self):
+ def used_outputs(self) -> list[str]:
"""List of used application outputs."""
return [getattr(self, key) for key in self.out_param_types if key in self.parameters]
- def find_outputs(self):
+ def find_outputs(self) -> tuple[str]:
"""Find output files on disk using path found in parameters.
Returns:
@@ -91,7 +92,7 @@ class App(OTBObject):
class OTBTFApp(App):
"""Helper for OTBTF."""
@staticmethod
- def set_nb_sources(*args, n_sources=None):
+ def set_nb_sources(*args, n_sources: int = None):
"""Set the number of sources of TensorflowModelServe. Can be either user-defined or deduced from the args.
Args:
@@ -109,11 +110,11 @@ class OTBTFApp(App):
if n_sources >= 1:
os.environ['OTB_TF_NSOURCES'] = str(n_sources)
- def __init__(self, app_name, *args, n_sources=None, **kwargs):
+ def __init__(self, name: str, *args, n_sources: int = None, **kwargs):
"""Constructor for an OTBTFApp object.
Args:
- app_name: name of the OTBTF app
+ name: name of the OTBTF app
*args: arguments (dict). NB: we don't need kwargs because it cannot contain source#.il
n_sources: number of sources. Default is None (resolves the number of sources based on the
content of the dict passed in args, where some 'source' str is found)
@@ -121,7 +122,7 @@ class OTBTFApp(App):
"""
self.set_nb_sources(*args, n_sources=n_sources)
- super().__init__(app_name, *args, **kwargs)
+ super().__init__(name, *args, **kwargs)
AVAILABLE_APPLICATIONS = get_available_applications(as_subprocess=True)
diff --git a/pyotb/core.py b/pyotb/core.py
index 86bfb9c..88496b6 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1,5 +1,7 @@
# -*- coding: utf-8 -*-
"""This module is the core of pyotb."""
+from __future__ import annotations
+from typing import Any
from pathlib import Path
from ast import literal_eval
@@ -12,7 +14,7 @@ from .helpers import logger
class OTBObject:
"""Base class that gathers common operations for any OTB application."""
- def __init__(self, name, *args, frozen=False, quiet=False, image_dic=None, **kwargs):
+ def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, **kwargs):
"""Common constructor for OTB applications. Handles in-memory connection between apps.
Args:
@@ -55,7 +57,7 @@ class OTBObject:
if any(key in self.parameters for key in self.out_param_types):
self.flush() # auto flush if any output param was provided during app init
- def get_first_key(self, param_types):
+ def get_first_key(self, param_types: list[str]) -> str:
"""Get the first output param key for specific file types."""
for key, param_type in sorted(self.all_param_types.items()):
if param_type in param_types:
@@ -63,7 +65,7 @@ class OTBObject:
return None
@property
- def key_input(self):
+ def key_input(self) -> str:
"""Get the name of first input parameter, raster > vector > file."""
return self.get_first_key(param_types=[otb.ParameterType_InputImage,
otb.ParameterType_InputImageList]) \
@@ -73,17 +75,17 @@ class OTBObject:
otb.ParameterType_InputFilenameList])
@property
- def key_input_image(self):
+ def key_input_image(self) -> str:
"""Get the name of first input image parameter."""
return self.get_first_key(param_types=[otb.ParameterType_InputImage, otb.ParameterType_InputImageList])
@property
- def key_output_image(self):
+ def key_output_image(self) -> str:
"""Get the name of first output image parameter."""
return self.get_first_key(param_types=[otb.ParameterType_OutputImage])
@property
- def dtype(self):
+ def dtype(self) -> np.dtype:
"""Expose the pixel type of output image using numpy convention.
Returns:
@@ -96,7 +98,7 @@ class OTBObject:
return self.app.ConvertPixelTypeToNumpy(enum)
@property
- def shape(self):
+ def shape(self) -> tuple(int):
"""Enables to retrieve the shape of a pyotb object using numpy convention.
Returns:
@@ -110,7 +112,7 @@ class OTBObject:
return height, width, bands
@property
- def transform(self):
+ def transform(self) -> tuple(int):
"""Get image affine transform, rasterio style (see https://www.perrygeo.com/python-affine-transforms.html).
Returns:
@@ -206,7 +208,8 @@ class OTBObject:
logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name)
self.app.ExecuteAndWriteOutput()
- def write(self, *args, filename_extension="", pixel_type=None, preserve_dtype=False, **kwargs):
+ def write(self, *args, filename_extension: str = "", pixel_type: dict[str, str] | str = None,
+ preserve_dtype: bool = False, **kwargs):
"""Set output pixel type and write the output raster files.
Args:
@@ -264,7 +267,7 @@ class OTBObject:
self.set_parameters({key: output_filename})
self.flush()
- def propagate_dtype(self, target_key=None, dtype=None):
+ def propagate_dtype(self, target_key: str = None, dtype: int = None):
"""Propagate a pixel type from main input to every outputs, or to a target output key only.
With multiple inputs (if dtype is not provided), the type of the first input is considered.
@@ -294,7 +297,7 @@ class OTBObject:
for key in keys:
self.app.SetParameterOutputImagePixelType(key, dtype)
- def read_values_at_coords(self, row, col, bands=None):
+ def read_values_at_coords(self, row: int, col: int, bands: int = None) -> list[int | float] | int | float:
"""Get pixel value(s) at a given YX coordinates.
Args:
@@ -326,7 +329,7 @@ class OTBObject:
return data[0]
return data
- def channels_list_from_slice(self, bands):
+ def channels_list_from_slice(self, bands: int) -> list[int]:
"""Get list of channels to read values at, from a slice."""
nb_channels = self.shape[2]
start, stop, step = bands.start, bands.stop, bands.step
@@ -343,7 +346,7 @@ class OTBObject:
return list(range(0, nb_channels, step))
raise ValueError(f"{self.name}: '{bands}' cannot be interpreted as valid slicing.")
- def summarize(self):
+ def summarize(self) -> dict:
"""Serialize an object and its pipeline into a dictionary.
Returns:
@@ -359,7 +362,7 @@ class OTBObject:
params[k] = [pi.summarize() if isinstance(pi, OTBObject) else pi for pi in p]
return {"name": self.name, "parameters": params}
- def export(self, key=None, preserve_dtype=True):
+ def export(self, key: str = None, preserve_dtype: bool = True) -> dict[str, dict[str, np.ndarray]]:
"""Export a specific output image as numpy array and store it in object exports_dic.
Args:
@@ -379,7 +382,7 @@ class OTBObject:
self.exports_dic[key]["array"] = self.exports_dic[key]["array"].astype(self.dtype)
return self.exports_dic[key]
- def to_numpy(self, key=None, preserve_dtype=True, copy=False):
+ def to_numpy(self, key: str = None, preserve_dtype: bool = True, copy: bool = False) -> np.ndarray:
"""Export a pyotb object to numpy array.
Args:
@@ -399,7 +402,7 @@ class OTBObject:
return array.copy()
return array
- def to_rasterio(self):
+ def to_rasterio(self) -> tuple[np.ndarray, dict[str, Any]]:
"""Export image as a numpy array and its metadata compatible with rasterio.
Returns:
@@ -417,7 +420,7 @@ class OTBObject:
}
return array, profile
- def xy_to_rowcol(self, x, y):
+ def xy_to_rowcol(self, x: float, y: float) -> tuple[int, int]:
"""Find (row, col) index using (x, y) projected coordinates - image CRS is expected.
Args:
@@ -432,7 +435,7 @@ class OTBObject:
return abs(int(row)), int(col)
# Private functions
- def __parse_args(self, args):
+ def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]:
"""Gather all input arguments in kwargs dict.
Args:
@@ -450,7 +453,7 @@ class OTBObject:
kwargs.update({self.key_input: arg})
return kwargs
- def __set_param(self, key, obj):
+ def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]):
"""Set one parameter, decide which otb.Application method to use depending on target object."""
if obj is None or (isinstance(obj, (list, tuple)) and not obj):
self.app.ClearValue(key)
@@ -558,7 +561,7 @@ class OTBObject:
key = key + (slice(None, None, None),)
return Slicer(self, *key)
- def __add__(self, other):
+ def __add__(self, other: OTBObject | Output | str | int | float) -> Operation:
"""Overrides the default addition and flavours it with BandMathX.
Args:
@@ -572,7 +575,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("+", self, other)
- def __sub__(self, other):
+ def __sub__(self, other: OTBObject | Output | str | int | float) -> Operation:
"""Overrides the default subtraction and flavours it with BandMathX.
Args:
@@ -586,7 +589,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("-", self, other)
- def __mul__(self, other):
+ def __mul__(self, other: OTBObject | Output | str | int | float) -> Operation:
"""Overrides the default subtraction and flavours it with BandMathX.
Args:
@@ -600,7 +603,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("*", self, other)
- def __truediv__(self, other):
+ def __truediv__(self, other: OTBObject | Output | str | int | float) -> Operation:
"""Overrides the default subtraction and flavours it with BandMathX.
Args:
@@ -614,7 +617,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("/", self, other)
- def __radd__(self, other):
+ def __radd__(self, other: OTBObject | Output | str | int | float) -> Operation:
"""Overrides the default reverse addition and flavours it with BandMathX.
Args:
@@ -628,7 +631,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("+", other, self)
- def __rsub__(self, other):
+ def __rsub__(self, other: OTBObject | Output | str | int | float) -> Operation:
"""Overrides the default subtraction and flavours it with BandMathX.
Args:
@@ -642,7 +645,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("-", other, self)
- def __rmul__(self, other):
+ def __rmul__(self, other: OTBObject | Output | str | int | float) -> Operation:
"""Overrides the default multiplication and flavours it with BandMathX.
Args:
@@ -656,7 +659,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("*", other, self)
- def __rtruediv__(self, other):
+ def __rtruediv__(self, other: OTBObject | Output | str | int | float) -> Operation:
"""Overrides the default division and flavours it with BandMathX.
Args:
@@ -670,7 +673,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("/", other, self)
- def __abs__(self):
+ def __abs__(self) -> Operation:
"""Overrides the default abs operator and flavours it with BandMathX.
Returns:
@@ -679,7 +682,7 @@ class OTBObject:
"""
return Operation("abs", self)
- def __ge__(self, other):
+ def __ge__(self, other: OTBObject | Output | str | int | float) -> Operation:
"""Overrides the default greater or equal and flavours it with BandMathX.
Args:
@@ -693,7 +696,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation(">=", self, other)
- def __le__(self, other):
+ def __le__(self, other: OTBObject | Output | str | int | float) -> Operation:
"""Overrides the default less or equal and flavours it with BandMathX.
Args:
@@ -707,7 +710,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation("<=", self, other)
- def __gt__(self, other):
+ def __gt__(self, other: OTBObject | Output | str | int | float) -> Operation:
"""Overrides the default greater operator and flavours it with BandMathX.
Args:
@@ -721,7 +724,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation(">", self, other)
- def __lt__(self, other):
+ def __lt__(self, other: OTBObject | Output | str | int | float) -> Operation:
"""Overrides the default less operator and flavours it with BandMathX.
Args:
@@ -735,7 +738,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation("<", self, other)
- def __eq__(self, other):
+ def __eq__(self, other: OTBObject | Output | str | int | float) -> Operation:
"""Overrides the default eq operator and flavours it with BandMathX.
Args:
@@ -749,7 +752,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation("==", self, other)
- def __ne__(self, other):
+ def __ne__(self, other: OTBObject | Output | str | int | float) -> Operation:
"""Overrides the default different operator and flavours it with BandMathX.
Args:
@@ -763,7 +766,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation("!=", self, other)
- def __or__(self, other):
+ def __or__(self, other: OTBObject | Output | str | int | float) -> Operation:
"""Overrides the default or operator and flavours it with BandMathX.
Args:
@@ -777,7 +780,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation("||", self, other)
- def __and__(self, other):
+ def __and__(self, other: OTBObject | Output | str | int | float) -> Operation:
"""Overrides the default and operator and flavours it with BandMathX.
Args:
@@ -794,7 +797,7 @@ class OTBObject:
# TODO: other operations ?
# e.g. __pow__... cf https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types
- def __array__(self):
+ def __array__(self) -> np.ndarray:
"""This is called when running np.asarray(pyotb_object).
Returns:
@@ -803,7 +806,7 @@ class OTBObject:
"""
return self.to_numpy()
- def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):
+ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs) -> OTBObject:
"""This is called whenever a numpy function is called on a pyotb object.
Operation is performed in numpy, then imported back to pyotb with the same georeference as input.
@@ -853,7 +856,7 @@ class OTBObject:
class Slicer(OTBObject):
"""Slicer objects i.e. when we call something like raster[:, :, 2] from Python."""
- def __init__(self, obj, rows, cols, channels):
+ def __init__(self, obj: OTBObject | Output | str, rows: int, cols: int, channels: int):
"""Create a slicer object, that can be used directly for writing or inside a BandMath.
It contains :
@@ -938,7 +941,7 @@ class Operation(OTBObject):
"""
- def __init__(self, operator, *inputs, nb_bands=None):
+ def __init__(self, operator: str, *inputs, nb_bands: int = None):
"""Given some inputs and an operator, this function enables to transform this into an OTB application.
Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator.
@@ -978,7 +981,8 @@ class Operation(OTBObject):
super().__init__(name, il=self.unique_inputs, exp=self.exp, quiet=True)
self.name = f'Operation exp="{self.exp}"'
- def create_fake_exp(self, operator, inputs, nb_bands=None):
+ def create_fake_exp(self, operator: str, inputs: list[OTBObject | Output | str | int | float],
+ nb_bands: int = None):
"""Create a 'fake' expression.
E.g for the operation input1 + input2, we create a fake expression that is like "str(input1) + str(input2)"
@@ -1045,7 +1049,7 @@ class Operation(OTBObject):
fake_exp = f"({fake_exps[0]} ? {fake_exps[1]} : {fake_exps[2]})"
self.fake_exp_bands.append(fake_exp)
- def get_real_exp(self, fake_exp_bands):
+ def get_real_exp(self, fake_exp_bands: str) -> tuple(list[str], str):
"""Generates the BandMathX expression.
Args:
@@ -1069,7 +1073,8 @@ class Operation(OTBObject):
return exp_bands, exp
@staticmethod
- def create_one_input_fake_exp(x, band, keep_logical=False):
+ def create_one_input_fake_exp(x: OTBObject | Output | str,
+ band: int, keep_logical: bool = False) -> tuple(str, list[OTBObject], int):
"""This an internal function, only to be used by `create_fake_exp`.
Enable to create a fake expression just for one input and one band.
@@ -1126,7 +1131,7 @@ class Operation(OTBObject):
fake_exp = f"{x}b{band}"
return fake_exp, inputs, nb_channels
- def __str__(self):
+ def __str__(self) -> str:
"""Return a nice string representation with operator and object id."""
return f"<pyotb.Operation `{self.operator}` object, id {id(self)}>"
@@ -1139,7 +1144,7 @@ class LogicalOperation(Operation):
"""
- def __init__(self, operator, *inputs, nb_bands=None):
+ def __init__(self, operator: str, *inputs, nb_bands: int = None):
"""Constructor for a LogicalOperation object.
Args:
@@ -1151,7 +1156,8 @@ class LogicalOperation(Operation):
super().__init__(operator, *inputs, nb_bands=nb_bands)
self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands)
- def create_fake_exp(self, operator, inputs, nb_bands=None):
+ def create_fake_exp(self, operator: str, inputs: list[OTBObject | Output | str | int | float],
+ nb_bands: int = None):
"""Create a 'fake' expression.
E.g for the operation input1 > input2, we create a fake expression that is like
@@ -1197,7 +1203,7 @@ class LogicalOperation(Operation):
class Input(OTBObject):
"""Class for transforming a filepath to pyOTB object."""
- def __init__(self, path):
+ def __init__(self, path: str):
"""Default constructor.
Args:
@@ -1210,7 +1216,7 @@ class Input(OTBObject):
self.propagate_dtype()
self.execute()
- def __str__(self):
+ def __str__(self) -> str:
"""Return a nice string representation with file path."""
return f"<pyotb.Input object from {self.path}>"
@@ -1218,7 +1224,7 @@ class Input(OTBObject):
class Output:
"""Object that behave like a pointer to a specific application output file."""
- def __init__(self, source_app, param_key, filepath=None, mkdir=True):
+ def __init__(self, source_app: OTBObject, param_key: str, filepath: str = None, mkdir: bool = True):
"""Constructor for an Output object.
Args:
@@ -1239,7 +1245,7 @@ class Output:
self.make_parent_dirs()
self.name = f"Output {param_key} from {self.source_app.name}"
- def exists(self):
+ def exists(self) -> bool:
"""Check file exist."""
return self.filepath.exists()
@@ -1248,12 +1254,12 @@ class Output:
if not self.filepath.parent.exists():
self.filepath.parent.mkdir(parents=True)
- def __str__(self):
+ def __str__(self) -> str:
"""Return a nice string representation with source app name and object id."""
return f"<pyotb.Output {self.source_app.name} object, id {id(self)}>"
-def get_nbchannels(inp):
+def get_nbchannels(inp: str | OTBObject) -> int:
"""Get the nb of bands of input image.
Args:
@@ -1275,7 +1281,7 @@ def get_nbchannels(inp):
return nb_channels
-def get_pixel_type(inp):
+def get_pixel_type(inp: str | OTBObject) -> str:
"""Get the encoding of input image pixels.
Args:
@@ -1315,7 +1321,7 @@ def get_pixel_type(inp):
return pixel_type
-def parse_pixel_type(pixel_type):
+def parse_pixel_type(pixel_type: str | int) -> int:
"""Convert one str pixel type to OTB integer enum if necessary.
Args:
@@ -1332,7 +1338,7 @@ def parse_pixel_type(pixel_type):
raise ValueError(f'Bad pixel type specification ({pixel_type})')
-def is_key_list(pyotb_app, key):
+def is_key_list(pyotb_app: OTBObject, key: str) -> bool:
"""Check if a key of the App is an input parameter list."""
return pyotb_app.app.GetParameterType(key) in (
otb.ParameterType_InputImageList,
@@ -1343,13 +1349,14 @@ def is_key_list(pyotb_app, key):
)
-def is_key_images_list(pyotb_app, key):
+def is_key_images_list(pyotb_app: OTBObject, key: str) -> bool:
"""Check if a key of the App is an input parameter image list."""
return pyotb_app.app.GetParameterType(key) in (
- otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList
+ otb.ParameterType_InputImageList,
+ otb.ParameterType_InputFilenameList
)
-def get_out_images_param_keys(app):
+def get_out_images_param_keys(app: OTBObject) -> list[str]:
"""Return every output parameter keys of an OTB app."""
return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage]
diff --git a/pyotb/functions.py b/pyotb/functions.py
index 669039a..aa161e6 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
"""This module provides a set of functions for pyotb."""
+from __future__ import annotations
import inspect
import os
import sys
@@ -7,18 +8,19 @@ import textwrap
import uuid
from collections import Counter
-from .core import (OTBObject, Input, Operation, LogicalOperation, get_nbchannels)
+from .core import OTBObject, Input, Operation, LogicalOperation, get_nbchannels, Output
from .helpers import logger
-def where(cond, x, y):
+def where(cond: OTBObject | Output | str, x: OTBObject | Output | str | int | float,
+ y: OTBObject | Output | str | int | float) -> Operation:
"""Functionally similar to numpy.where. Where cond is True (!=0), returns x. Else returns y.
Args:
cond: condition, must be a raster (filepath, App, Operation...). If cond is monoband whereas x or y are
multiband, cond channels are expanded to match x & y ones.
- x: value if cond is True. Can be float, int, App, filepath, Operation...
- y: value if cond is False. Can be float, int, App, filepath, Operation...
+ x: value if cond is True. Can be: float, int, App, filepath, Operation...
+ y: value if cond is False. Can be: float, int, App, filepath, Operation...
Returns:
an output where pixels are x if cond is True, else y
@@ -61,7 +63,8 @@ def where(cond, x, y):
return operation
-def clip(a, a_min, a_max):
+def clip(a: OTBObject | Output | str, a_min: OTBObject | Output | str | int | float,
+ a_max: OTBObject | Output | str | int | float):
"""Clip values of image in a range of values.
Args:
@@ -321,8 +324,9 @@ def run_tf_function(func):
return wrapper
-def define_processing_area(*args, window_rule='intersection', pixel_size_rule='minimal', interpolator='nn',
- reference_window_input=None, reference_pixel_size_input=None):
+def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_rule: str = 'minimal',
+ interpolator: str = 'nn', reference_window_input: dict = None,
+ reference_pixel_size_input: str = None) -> list[OTBObject]:
"""Given several inputs, this function handles the potential resampling and cropping to same extent.
WARNING: Not fully implemented / tested
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 742fee1..d38dfb3 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -26,7 +26,7 @@ logger_handler.setLevel(getattr(logging, LOG_LEVEL))
logger.addHandler(logger_handler)
-def set_logger_level(level):
+def set_logger_level(level: str):
"""Allow user to change the current logging level.
Args:
@@ -36,7 +36,7 @@ def set_logger_level(level):
logger_handler.setLevel(getattr(logging, level))
-def find_otb(prefix=OTB_ROOT, scan=True, scan_userdir=True):
+def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = True):
"""Try to load OTB bindings or scan system, help user in case of failure, set env variables.
Path precedence : OTB_ROOT > python bindings directory
@@ -98,7 +98,7 @@ def find_otb(prefix=OTB_ROOT, scan=True, scan_userdir=True):
raise SystemExit("Failed to import OTB. Exiting.") from e
-def set_environment(prefix):
+def set_environment(prefix: str):
"""Set environment variables (before OTB import), raise error if anything is wrong.
Args:
@@ -156,7 +156,7 @@ def set_environment(prefix):
os.environ["PROJ_LIB"] = proj_lib
-def __find_lib(prefix=None, otb_module=None):
+def __find_lib(prefix: str = None, otb_module=None):
"""Try to find OTB external libraries directory.
Args:
@@ -187,7 +187,7 @@ def __find_lib(prefix=None, otb_module=None):
return None
-def __find_python_api(lib_dir):
+def __find_python_api(lib_dir: Path):
"""Try to find the python path.
Args:
@@ -206,7 +206,7 @@ def __find_python_api(lib_dir):
return None
-def __find_apps_path(lib_dir):
+def __find_apps_path(lib_dir: Path):
"""Try to find the OTB applications path.
Args:
@@ -225,7 +225,7 @@ def __find_apps_path(lib_dir):
return ""
-def __find_otb_root(scan_userdir=False):
+def __find_otb_root(scan_userdir: bool = False):
"""Search for OTB root directory in well known locations.
Args:
@@ -271,7 +271,7 @@ def __find_otb_root(scan_userdir=False):
return prefix
-def __suggest_fix_import(error_message, prefix):
+def __suggest_fix_import(error_message: str, prefix: str):
"""Help user to fix the OTB installation with appropriate log messages."""
logger.critical("An error occurred while importing OTB Python API")
logger.critical("OTB error message was '%s'", error_message)
--
GitLab
From 071ed7308ebc75e86651117ffe3de49d143491bf Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 24 Jan 2023 09:38:20 +0000
Subject: [PATCH 034/399] Resolve "Update pyproject.toml to PEP621"
---
.gitlab-ci.yml | 6 ++---
pyproject.toml | 68 ++++++++++++++++++++++++++++++++++++++++++++++----
setup.cfg | 3 ---
setup.py | 32 ++----------------------
4 files changed, 68 insertions(+), 41 deletions(-)
delete mode 100644 setup.cfg
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 203b789..a38149e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -36,16 +36,16 @@ flake8:
pydocstyle:
extends: .static_analysis
before_script:
- - pip install pydocstyle
+ - pip install pydocstyle tomli
script:
- - pydocstyle $PWD/pyotb --convention=google
+ - pydocstyle $PWD/pyotb
pylint:
extends: .static_analysis
before_script:
- pip install pylint
script:
- - pylint --max-line-length=120 $PWD/pyotb --disable=too-many-nested-blocks,too-many-locals,too-many-statements,too-few-public-methods,too-many-instance-attributes,too-many-arguments,invalid-name,fixme,too-many-return-statements,too-many-lines,too-many-branches,import-outside-toplevel,wrong-import-position,wrong-import-order,import-error,missing-class-docstring
+ - pylint $PWD/pyotb
# ---------------------------------- Documentation ----------------------------------
diff --git a/pyproject.toml b/pyproject.toml
index ddb1465..631be88 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,7 +1,65 @@
[build-system]
-requires = [
- "setuptools>=42",
- "wheel",
- "numpy>=1.13,<1.23"
-]
+requires = ["setuptools >= 61.0", "wheel"]
build-backend = "setuptools.build_meta"
+
+[project]
+name = "pyotb"
+description = "Library to enable easy use of the Orfeo ToolBox (OTB) in Python"
+authors = [{name = "Rémi Cresson", email = "remi.cresson@inrae.fr"}, {name = "Nicolas Narçon"}, {name = "Vincent Delbar"}]
+requires-python = ">=3.7"
+keywords = ["gis", "remote sensing", "otb", "orfeotoolbox", "orfeo toolbox"]
+dependencies = ["numpy>=1.16"]
+readme = "README.md"
+license = { file = "LICENSE" }
+dynamic = ["version"]
+classifiers=[
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.7",
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Topic :: Scientific/Engineering :: GIS",
+ "Topic :: Scientific/Engineering :: Image Processing",
+ "License :: OSI Approved :: Apache Software License",
+ "Operating System :: OS Independent",
+]
+
+[project.optional-dependencies]
+dev = ["pytest", "pylint", "codespell", "pydocstyle", "tomli"]
+
+[project.urls]
+documentation = "https://pyotb.readthedocs.io"
+homepage = "https://github.com/orfeotoolbox/pyotb"
+repository = "https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb"
+
+[tool.setuptools]
+packages = ["pyotb"]
+
+[tool.setuptools.dynamic]
+version = {attr = "pyotb.__version__"}
+
+[tool.pylint]
+max-line-length = 120
+disable = [
+ "fixme",
+ "import-error",
+ "import-outside-toplevel",
+ "wrong-import-position",
+ "wrong-import-order",
+ "invalid-name",
+ "too-many-nested-blocks",
+ "too-many-locals",
+ "too-many-statements",
+ "too-many-instance-attributes",
+ "too-many-arguments",
+ "too-many-return-statements",
+ "too-many-lines",
+ "too-many-branches",
+]
+
+[tool.pydocstyle]
+convention = "google"
+
+[tool.black]
+line-length = 120
diff --git a/setup.cfg b/setup.cfg
deleted file mode 100644
index 778d274..0000000
--- a/setup.cfg
+++ /dev/null
@@ -1,3 +0,0 @@
-[metadata]
-description-file=README.md
-
diff --git a/setup.py b/setup.py
index 519ba5d..6068493 100644
--- a/setup.py
+++ b/setup.py
@@ -1,31 +1,3 @@
-# -*- coding: utf-8 -*-
-import setuptools
+from setuptools import setup
-with open("README.md", "r", encoding="utf-8") as fh:
- long_description = fh.read()
-
-setuptools.setup(
- name="pyotb",
- version="1.5.4",
- author="Nicolas Narçon",
- author_email="nicolas.narcon@gmail.com",
- description="Library to enable easy use of the Orfeo Tool Box (OTB) in Python",
- long_description=long_description,
- long_description_content_type="text/markdown",
- url="https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/",
- classifiers=[
- "Programming Language :: Python :: 3",
- "Programming Language :: Python :: 3.6",
- "Programming Language :: Python :: 3.7",
- "Programming Language :: Python :: 3.8",
- "Programming Language :: Python :: 3.9",
- "Topic :: Scientific/Engineering :: GIS",
- "Topic :: Scientific/Engineering :: Image Processing",
- "License :: OSI Approved :: Apache Software License",
- "Operating System :: OS Independent",
- ],
- packages=setuptools.find_packages(),
- python_requires=">=3.6",
- keywords="remote sensing, otb, orfeotoolbox, orfeo toolbox",
-)
-#package_dir={"": "src"},
+setup()
--
GitLab
From 9733ddbae069f30c6be5eb3c93999d0e18d9ba16 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 13 Jan 2023 14:27:45 +0100
Subject: [PATCH 035/399] ENH: add property elapsed time
---
pyotb/apps.py | 8 ++++++--
pyotb/core.py | 6 +++++-
2 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/pyotb/apps.py b/pyotb/apps.py
index a2af7ed..b63da8a 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -29,7 +29,7 @@ def get_available_applications(as_subprocess: bool = False) -> list[str]:
env = os.environ.copy()
if "PYTHONPATH" not in env:
env["PYTHONPATH"] = ""
- env["PYTHONPATH"] = ":" + str(Path(otb.__file__).parent)
+ env["PYTHONPATH"] += ":" + str(Path(otb.__file__).parent)
env["OTB_LOGGER_LEVEL"] = "CRITICAL" # in order to suppress warnings while listing applications
pycmd = "import otbApplication; print(otbApplication.Registry.GetAvailableApplications())"
cmd_args = [sys.executable, "-c", pycmd]
@@ -49,7 +49,7 @@ def get_available_applications(as_subprocess: bool = False) -> list[str]:
except (ValueError, SyntaxError, AssertionError):
logger.debug("Failed to decode output or convert to tuple:\nstdout=%s\nstderr=%s", stdout, stderr)
if not app_list:
- logger.info("Failed to list applications in an independent process. Falling back to local python import")
+ logger.debug("Failed to list applications in an independent process. Falling back to local python import")
# Find applications using the normal way
if not app_list:
app_list = otb.Registry.GetAvailableApplications()
@@ -68,6 +68,10 @@ class App(OTBObject):
super().__init__(*args, **kwargs)
self.description = self.app.GetDocLongDescription()
+ def elapsed_time(self):
+ """Get elapsed time between app init and end of exec or file writing."""
+ return self.time_end - self.time_start
+
@property
def used_outputs(self) -> list[str]:
"""List of used application outputs."""
diff --git a/pyotb/core.py b/pyotb/core.py
index 88496b6..c3649b5 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -4,6 +4,7 @@ from __future__ import annotations
from typing import Any
from pathlib import Path
from ast import literal_eval
+from time import perf_counter
import numpy as np
import otbApplication as otb
@@ -44,7 +45,7 @@ class OTBObject:
create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
self.app = create(name)
self.parameters_keys = tuple(self.app.GetParametersKeys())
- # Output parameters types
+ self.time_start, self.time_end = 0, 0
self.all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys}
self.out_param_types = {k: v for k, v in self.all_param_types.items()
if v in (otb.ParameterType_OutputImage,
@@ -191,11 +192,13 @@ class OTBObject:
def execute(self):
"""Execute and write to disk if any output parameter has been set during init."""
logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters)
+ self.time_start = perf_counter()
try:
self.app.Execute()
except (RuntimeError, FileNotFoundError) as e:
raise Exception(f"{self.name}: error during during app execution") from e
self.frozen = False
+ self.time_end = perf_counter()
logger.debug("%s: execution ended", self.name)
self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics
@@ -207,6 +210,7 @@ class OTBObject:
except RuntimeError:
logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name)
self.app.ExecuteAndWriteOutput()
+ self.time_end = perf_counter()
def write(self, *args, filename_extension: str = "", pixel_type: dict[str, str] | str = None,
preserve_dtype: bool = False, **kwargs):
--
GitLab
From a56158153fac1865a3e6d0c40a0fd9e040952331 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 13 Jan 2023 14:27:55 +0100
Subject: [PATCH 036/399] FIX: import error help message
---
pyotb/helpers.py | 4 ----
1 file changed, 4 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index d38dfb3..ebccc48 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -294,16 +294,12 @@ def __suggest_fix_import(error_message: str, prefix: str):
logger.critical("You may need to install cmake in order to recompile python bindings")
else:
logger.critical("Unable to automatically locate python dynamic library of %s", sys.executable)
- return
elif sys.platform == "win32":
if error_message.startswith("DLL load failed"):
if sys.version_info.minor != 7:
logger.critical("You need Python 3.5 (OTB releases 6.4 to 7.4) or Python 3.7 (since OTB 8)")
- issue_link = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2010"
- logger.critical("Another workaround is to recompile Python bindings with cmake, see %s", issue_link)
else:
logger.critical("It seems that your env variables aren't properly set,"
" first use 'call otbenv.bat' then try to import pyotb once again")
- return
docs_link = "https://www.orfeo-toolbox.org/CookBook/Installation.html"
logger.critical("You can verify installation requirements for your OS at %s", docs_link)
--
GitLab
From 06c983d4be574cb6d0c6f0ffe65a94cb059d90d7 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 13 Jan 2023 14:28:37 +0100
Subject: [PATCH 037/399] ENH: add properties data and metadata
---
pyotb/core.py | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/pyotb/core.py b/pyotb/core.py
index c3649b5..f2fe4a8 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -85,6 +85,22 @@ class OTBObject:
"""Get the name of first output image parameter."""
return self.get_first_key(param_types=[otb.ParameterType_OutputImage])
+ @property
+ def data(self):
+ """Expose app's output data values in a dictionary."""
+ skip_keys = tuple(self.out_param_types) + tuple(self.parameters) + ("ram", "elev.default")
+ keys = (k for k in self.parameters_keys if k not in skip_keys)
+ def _check(v):
+ return not isinstance(v, otb.ApplicationProxy) and v not in ("", None, [], ())
+ return {str(k): self[k] for k in keys if _check(self[k])}
+
+ @property
+ def metadata(self):
+ if not self.key_output_image:
+ raise TypeError(f"{self.name}: this application has no raster output")
+ return dict(self.app.GetMetadataDictionary(self.key_output_image))
+
+
@property
def dtype(self) -> np.dtype:
"""Expose the pixel type of output image using numpy convention.
@@ -186,6 +202,11 @@ class OTBObject:
# Convert output param path to Output object
if key in self.out_param_types:
value = Output(self, key, value)
+ elif isinstance(key, str):
+ try:
+ value = literal_eval(value)
+ except (ValueError, SyntaxError):
+ pass
# Save attribute
setattr(self, key, value)
--
GitLab
From f066af5d22464f98f68930538aa5074de9f7eae6 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 13 Jan 2023 14:28:54 +0100
Subject: [PATCH 038/399] ENH: add functions get_infos and get_statistics
---
pyotb/core.py | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/pyotb/core.py b/pyotb/core.py
index f2fe4a8..dc02141 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -322,6 +322,16 @@ class OTBObject:
for key in keys:
self.app.SetParameterOutputImagePixelType(key, dtype)
+ def get_infos(self):
+ if not self.key_output_image:
+ raise TypeError(f"{self.name}: this application has no raster output")
+ return OTBObject("ReadImageInfo", self, quiet=True).data
+
+ def get_statistics(self):
+ if not self.key_output_image:
+ raise TypeError(f"{self.name}: this application has no raster output")
+ return OTBObject("ComputeImagesStatistics", self, quiet=True).data
+
def read_values_at_coords(self, row: int, col: int, bands: int = None) -> list[int | float] | int | float:
"""Get pixel value(s) at a given YX coordinates.
--
GitLab
From 408d20d7acf0c54b84566a62db24426cba410c98 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 13 Jan 2023 14:44:31 +0100
Subject: [PATCH 039/399] ENH: add tests for new features + reorganize
---
tests/test_core.py | 58 ++++++++++++++++++++++++++++++---------------
tests/test_numpy.py | 4 ++++
2 files changed, 43 insertions(+), 19 deletions(-)
diff --git a/tests/test_core.py b/tests/test_core.py
index 2eadef7..436f99e 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -8,6 +8,12 @@ import pytest
FILEPATH = os.environ["TEST_INPUT_IMAGE"]
INPUT = pyotb.Input(FILEPATH)
+TEST_IMAGE_STATS = {
+ 'out.mean': [79.5505, 109.225, 115.456, 249.349],
+ 'out.min': [33, 64, 91, 47],
+ 'out.max': [255, 255, 230, 255],
+ 'out.std': [51.0754, 35.3152, 23.4514, 20.3827]
+}
# Input settings
@@ -20,7 +26,7 @@ def test_wrong_key():
pyotb.BandMath(INPUT, expression="im1b1")
-# OTBObject's properties
+# OTBObject properties
def test_key_input():
assert INPUT.key_input == INPUT.key_input_image == "in"
@@ -41,10 +47,39 @@ def test_transform():
assert INPUT.transform == (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0)
+def test_data():
+ assert pyotb.ComputeImagesStatistics(INPUT).data == TEST_IMAGE_STATS
+
+
+def test_metadata():
+ assert INPUT.metadata["Metadata_1"] == "TIFFTAG_SOFTWARE=CSinG - 13 SEPTEMBRE 2012"
+
+
def test_nonraster_property():
with pytest.raises(TypeError):
pyotb.ReadImageInfo(INPUT).dtype
+def test_elapsed_time():
+ assert pyotb.ReadImageInfo(INPUT).elapsed_time < 1
+
+# Other functions
+def test_get_infos():
+ infos = INPUT.get_infos()
+ assert (infos["sizex"], infos["sizey"]) == (251, 304)
+
+
+def test_get_statistics():
+ assert INPUT.get_statistics() == TEST_IMAGE_STATS
+
+
+def test_xy_to_rowcol():
+ assert INPUT.xy_to_rowcol(760101, 6945977) == (19, 7)
+
+
+def test_write():
+ INPUT.write("/tmp/missing_dir/test_write.tif")
+ assert INPUT.out.exists()
+
# Slicer
def test_slicer_shape():
@@ -85,7 +120,7 @@ def test_binary_mask_where():
assert res.exp == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)"
-# Apps
+# Essential apps
def test_app_readimageinfo():
info = pyotb.ReadImageInfo(INPUT, quiet=True)
assert (info.sizex, info.sizey) == (251, 304)
@@ -94,12 +129,12 @@ def test_app_readimageinfo():
def test_app_computeimagestats():
stats = pyotb.ComputeImagesStatistics([INPUT], quiet=True)
- assert stats["out.min"] == "[33, 64, 91, 47]"
+ assert stats["out.min"] == TEST_IMAGE_STATS["out.min"]
def test_app_computeimagestats_sliced():
slicer_stats = pyotb.ComputeImagesStatistics(il=[INPUT[:10, :10, 0]], quiet=True)
- assert slicer_stats["out.min"] == "[180]"
+ assert slicer_stats["out.min"] == [180]
def test_read_values_at_coords():
@@ -107,21 +142,6 @@ def test_read_values_at_coords():
assert INPUT[10, 20, :] == [207, 192, 172, 255]
-# XY => RowCol
-def test_xy_to_rowcol():
- assert INPUT.xy_to_rowcol(760101, 6945977) == (19, 7)
-
-
-def test_pixel_coords_numpy_equals_otb():
- assert INPUT[19,7] == list(INPUT.to_numpy()[19,7])
-
-
-# Create dir before write
-def test_write():
- INPUT.write("/tmp/missing_dir/test_write.tif")
- assert INPUT.out.filepath.exists()
-
-
# BandMath NDVI == RadiometricIndices NDVI ?
def test_ndvi_comparison():
ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (INPUT[:, :, -1] + INPUT[:, :, 0])
diff --git a/tests/test_numpy.py b/tests/test_numpy.py
index 4240cb9..dd33425 100644
--- a/tests/test_numpy.py
+++ b/tests/test_numpy.py
@@ -36,6 +36,10 @@ def test_convert_to_array():
assert INPUT.shape == array.shape
+def test_pixel_coords_otb_equals_numpy():
+ assert INPUT[19,7] == list(INPUT.to_numpy()[19,7])
+
+
def test_add_noise_array():
white_noise = np.random.normal(0, 50, size=INPUT.shape)
noisy_image = INPUT + white_noise
--
GitLab
From 8390bd43aa8387ef28cdd8bda9fc0726d39b56c3 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 13 Jan 2023 14:49:23 +0100
Subject: [PATCH 040/399] FIX: missing property decorator
---
pyotb/apps.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/pyotb/apps.py b/pyotb/apps.py
index b63da8a..5092fee 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -68,6 +68,7 @@ class App(OTBObject):
super().__init__(*args, **kwargs)
self.description = self.app.GetDocLongDescription()
+ @property
def elapsed_time(self):
"""Get elapsed time between app init and end of exec or file writing."""
return self.time_end - self.time_start
--
GitLab
From 0b6d767a4e3ccb02de3294acc990df1cd9f25d8f Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 13 Jan 2023 15:02:40 +0100
Subject: [PATCH 041/399] STYLE: linting and docstrings
---
pyotb/core.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index dc02141..d4f32c0 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -90,17 +90,18 @@ class OTBObject:
"""Expose app's output data values in a dictionary."""
skip_keys = tuple(self.out_param_types) + tuple(self.parameters) + ("ram", "elev.default")
keys = (k for k in self.parameters_keys if k not in skip_keys)
+
def _check(v):
return not isinstance(v, otb.ApplicationProxy) and v not in ("", None, [], ())
return {str(k): self[k] for k in keys if _check(self[k])}
@property
def metadata(self):
+ """Return first output image metadata dictionary"""
if not self.key_output_image:
raise TypeError(f"{self.name}: this application has no raster output")
return dict(self.app.GetMetadataDictionary(self.key_output_image))
-
@property
def dtype(self) -> np.dtype:
"""Expose the pixel type of output image using numpy convention.
@@ -323,11 +324,13 @@ class OTBObject:
self.app.SetParameterOutputImagePixelType(key, dtype)
def get_infos(self):
+ """Return a dict output of ReadImageInfo for the first image output"""
if not self.key_output_image:
raise TypeError(f"{self.name}: this application has no raster output")
return OTBObject("ReadImageInfo", self, quiet=True).data
def get_statistics(self):
+ """Return a dict output of ComputeImagesStatistics for the first image output"""
if not self.key_output_image:
raise TypeError(f"{self.name}: this application has no raster output")
return OTBObject("ComputeImagesStatistics", self, quiet=True).data
--
GitLab
From 5cfa66f08e6949b412fdd1981d95d1790017f4bd Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 13 Jan 2023 15:11:23 +0100
Subject: [PATCH 042/399] STYLE: docstrings
---
pyotb/core.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index d4f32c0..6186ef2 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -97,7 +97,7 @@ class OTBObject:
@property
def metadata(self):
- """Return first output image metadata dictionary"""
+ """Return first output image metadata dictionary."""
if not self.key_output_image:
raise TypeError(f"{self.name}: this application has no raster output")
return dict(self.app.GetMetadataDictionary(self.key_output_image))
@@ -324,13 +324,13 @@ class OTBObject:
self.app.SetParameterOutputImagePixelType(key, dtype)
def get_infos(self):
- """Return a dict output of ReadImageInfo for the first image output"""
+ """Return a dict output of ReadImageInfo for the first image output."""
if not self.key_output_image:
raise TypeError(f"{self.name}: this application has no raster output")
return OTBObject("ReadImageInfo", self, quiet=True).data
def get_statistics(self):
- """Return a dict output of ComputeImagesStatistics for the first image output"""
+ """Return a dict output of ComputeImagesStatistics for the first image output."""
if not self.key_output_image:
raise TypeError(f"{self.name}: this application has no raster output")
return OTBObject("ComputeImagesStatistics", self, quiet=True).data
--
GitLab
From 359a813260b87ae8b2998f4bb43384f74c89634f Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 16 Jan 2023 13:36:48 +0100
Subject: [PATCH 043/399] FIX: check value is str before literal_eval
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 6186ef2..0a8a91c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -203,7 +203,7 @@ class OTBObject:
# Convert output param path to Output object
if key in self.out_param_types:
value = Output(self, key, value)
- elif isinstance(key, str):
+ elif isinstance(value, str):
try:
value = literal_eval(value)
except (ValueError, SyntaxError):
--
GitLab
From 844465de41d79fcba15e87ce4884038f9155628e Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 16 Jan 2023 13:56:31 +0100
Subject: [PATCH 044/399] ENH: avoid some keys in data property
---
pyotb/core.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 0a8a91c..ddaa62b 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -88,7 +88,8 @@ class OTBObject:
@property
def data(self):
"""Expose app's output data values in a dictionary."""
- skip_keys = tuple(self.out_param_types) + tuple(self.parameters) + ("ram", "elev.default")
+ skip_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem")
+ skip_keys = skip_keys + tuple(self.out_param_types) + tuple(self.parameters)
keys = (k for k in self.parameters_keys if k not in skip_keys)
def _check(v):
--
GitLab
From 41d372d4ebfc8f50369725a2f2b64024c931b0db Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 24 Jan 2023 11:15:04 +0100
Subject: [PATCH 045/399] CI: add missing pylint exception
---
pyproject.toml | 1 +
1 file changed, 1 insertion(+)
diff --git a/pyproject.toml b/pyproject.toml
index 631be88..2ef1c71 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -54,6 +54,7 @@ disable = [
"too-many-instance-attributes",
"too-many-arguments",
"too-many-return-statements",
+ "too-many-public-methods",
"too-many-lines",
"too-many-branches",
]
--
GitLab
From bdde3adc1d26f6eea00a8c37649df52acb7c42e8 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 24 Jan 2023 14:37:14 +0100
Subject: [PATCH 046/399] FIX: Output object must inherit from OTBObject
---
pyotb/core.py | 24 ++++++++++++++----------
1 file changed, 14 insertions(+), 10 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index ddaa62b..df585a8 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -556,11 +556,9 @@ class OTBObject:
AttributeError: when `name` is not an attribute of self.app
"""
- try:
- res = getattr(self.app, name)
- return res
- except AttributeError as e:
- raise AttributeError(f"{self.name}: could not find attribute `{name}`") from e
+ if name in dir(self.app):
+ return getattr(self.app, name)
+ raise AttributeError(f"{self.name}: could not find attribute `{name}`")
def __getitem__(self, key):
"""Override the default __getitem__ behaviour.
@@ -1260,20 +1258,21 @@ class Input(OTBObject):
return f"<pyotb.Input object from {self.path}>"
-class Output:
+class Output(OTBObject):
"""Object that behave like a pointer to a specific application output file."""
- def __init__(self, source_app: OTBObject, param_key: str, filepath: str = None, mkdir: bool = True):
+ def __init__(self, pyotb_app: OTBObject, param_key: str, filepath: str = None, mkdir: bool = True): # pylint: disable=super-init-not-called
"""Constructor for an Output object.
Args:
- source_app: The pyotb App to store reference from
+ pyotb_app: The pyotb App to store reference from
param_key: Output parameter key of the target app
filepath: path of the output file (if not in memory)
mkdir: create missing parent directories
"""
- self.source_app = source_app
+ self.pyotb_app, self.app = pyotb_app, pyotb_app.app
+ self.parameters = pyotb_app.parameters
self.param_key = param_key
self.filepath = None
if filepath:
@@ -1282,7 +1281,12 @@ class Output:
self.filepath = Path(filepath)
if mkdir:
self.make_parent_dirs()
- self.name = f"Output {param_key} from {self.source_app.name}"
+ self.name = f"Output {param_key} from {self.pyotb_app.name}"
+
+ @property
+ def key_output_image(self):
+ """Overwrite OTBObject prop, in order to use Operation special methods with the right Output param_key."""
+ return self.param_key
def exists(self) -> bool:
"""Check file exist."""
--
GitLab
From d0922315221b8449d7b764f5572aeaa884ae2679 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 24 Jan 2023 14:45:55 +0100
Subject: [PATCH 047/399] STYLE: max line length
---
pyotb/core.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index df585a8..acb09eb 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1261,7 +1261,8 @@ class Input(OTBObject):
class Output(OTBObject):
"""Object that behave like a pointer to a specific application output file."""
- def __init__(self, pyotb_app: OTBObject, param_key: str, filepath: str = None, mkdir: bool = True): # pylint: disable=super-init-not-called
+ def __init__(self, pyotb_app: OTBObject, param_key: str,
+ filepath: str = None, mkdir: bool = True): # pylint: disable=super-init-not-called
"""Constructor for an Output object.
Args:
--
GitLab
From 6c67f2fa091c97c11ae3dd577ac4f59f2ea3c154 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 24 Jan 2023 14:47:00 +0100
Subject: [PATCH 048/399] STYLE: pylint ignore on wrong line
---
pyotb/core.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index acb09eb..c7ceb04 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1261,13 +1261,13 @@ class Input(OTBObject):
class Output(OTBObject):
"""Object that behave like a pointer to a specific application output file."""
- def __init__(self, pyotb_app: OTBObject, param_key: str,
- filepath: str = None, mkdir: bool = True): # pylint: disable=super-init-not-called
+ def __init__(self, pyotb_app: OTBObject, # pylint: disable=super-init-not-called
+ param_key: str, filepath: str = None, mkdir: bool = True):
"""Constructor for an Output object.
Args:
pyotb_app: The pyotb App to store reference from
- param_key: Output parameter key of the target app
+ param_key: Output parameter key of the target app
filepath: path of the output file (if not in memory)
mkdir: create missing parent directories
--
GitLab
From 8ab686b2d2045d013d828f588b107f39d4da6550 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 24 Jan 2023 14:50:53 +0100
Subject: [PATCH 049/399] STYLE: whitespace
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index c7ceb04..69a0fd2 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1267,7 +1267,7 @@ class Output(OTBObject):
Args:
pyotb_app: The pyotb App to store reference from
- param_key: Output parameter key of the target app
+ param_key: Output parameter key of the target app
filepath: path of the output file (if not in memory)
mkdir: create missing parent directories
--
GitLab
From c8d19a1a91ff7ca677952ceb6fd5d9912c43df45 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 24 Jan 2023 15:49:38 +0100
Subject: [PATCH 050/399] STYLE: remove useless pylint exceptions
---
pyotb/apps.py | 4 ++--
pyotb/core.py | 2 +-
pyotb/functions.py | 6 +++---
pyproject.toml | 16 +++++-----------
4 files changed, 11 insertions(+), 17 deletions(-)
diff --git a/pyotb/apps.py b/pyotb/apps.py
index 5092fee..ad71f54 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -3,9 +3,10 @@
from __future__ import annotations
import os
import sys
+import subprocess
from pathlib import Path
-import otbApplication as otb
+import otbApplication as otb # pylint: disable=import-error
from .core import OTBObject
from .helpers import logger
@@ -34,7 +35,6 @@ def get_available_applications(as_subprocess: bool = False) -> list[str]:
pycmd = "import otbApplication; print(otbApplication.Registry.GetAvailableApplications())"
cmd_args = [sys.executable, "-c", pycmd]
try:
- import subprocess # pylint: disable=import-outside-toplevel
params = {"env": env, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE}
with subprocess.Popen(cmd_args, **params) as p:
logger.debug('Exec "%s \'%s\'"', ' '.join(cmd_args[:-1]), pycmd)
diff --git a/pyotb/core.py b/pyotb/core.py
index 69a0fd2..6f0a4dc 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -7,7 +7,7 @@ from ast import literal_eval
from time import perf_counter
import numpy as np
-import otbApplication as otb
+import otbApplication as otb # pylint: disable=import-error
from .helpers import logger
diff --git a/pyotb/functions.py b/pyotb/functions.py
index aa161e6..71c298b 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -4,8 +4,9 @@ from __future__ import annotations
import inspect
import os
import sys
-import textwrap
import uuid
+import textwrap
+import subprocess
from collections import Counter
from .core import OTBObject, Input, Operation, LogicalOperation, get_nbchannels, Output
@@ -217,7 +218,7 @@ def run_tf_function(func):
"""
try:
- from .apps import TensorflowModelServe
+ from .apps import TensorflowModelServe # pylint: disable=import-outside-toplevel
except ImportError:
logger.error('Could not run Tensorflow function: failed to import TensorflowModelServe.'
'Check that you have OTBTF configured (https://github.com/remicres/otbtf#how-to-install)')
@@ -303,7 +304,6 @@ def run_tf_function(func):
pycmd = get_tf_pycmd(out_savedmodel, channels, scalar_inputs)
cmd_args = [sys.executable, "-c", pycmd]
try:
- import subprocess
subprocess.run(cmd_args, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
except subprocess.SubprocessError:
logger.debug("Failed to call subprocess")
diff --git a/pyproject.toml b/pyproject.toml
index 2ef1c71..9538fc1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -26,7 +26,7 @@ classifiers=[
]
[project.optional-dependencies]
-dev = ["pytest", "pylint", "codespell", "pydocstyle", "tomli"]
+dev = ["pytest", "pylint", "codespell", "pydocstyle", "tomli"]
[project.urls]
documentation = "https://pyotb.readthedocs.io"
@@ -43,20 +43,14 @@ version = {attr = "pyotb.__version__"}
max-line-length = 120
disable = [
"fixme",
- "import-error",
- "import-outside-toplevel",
- "wrong-import-position",
- "wrong-import-order",
"invalid-name",
- "too-many-nested-blocks",
+ "too-many-lines",
"too-many-locals",
+ "too-many-branches",
"too-many-statements",
- "too-many-instance-attributes",
- "too-many-arguments",
- "too-many-return-statements",
"too-many-public-methods",
- "too-many-lines",
- "too-many-branches",
+ "too-many-instance-attributes",
+ "wrong-import-position",
]
[tool.pydocstyle]
--
GitLab
From 8ad470c51fe96ef1a51ae9c3cb47b1f392690e0e Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 11:00:07 +0000
Subject: [PATCH 051/399] WIP: refac classes
---
pyotb/core.py | 676 ++++++++++++++++++++++++++------------------------
1 file changed, 349 insertions(+), 327 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 6f0a4dc..87bb08c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1,10 +1,11 @@
# -*- coding: utf-8 -*-
"""This module is the core of pyotb."""
from __future__ import annotations
-from typing import Any
-from pathlib import Path
+
from ast import literal_eval
+from pathlib import Path
from time import perf_counter
+from typing import Any
import numpy as np
import otbApplication as otb # pylint: disable=import-error
@@ -13,50 +14,30 @@ from .helpers import logger
class OTBObject:
- """Base class that gathers common operations for any OTB application."""
-
- def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, **kwargs):
- """Common constructor for OTB applications. Handles in-memory connection between apps.
-
- Args:
- name: name of the app, e.g. 'BandMath'
- *args: used for passing application parameters. Can be :
- - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
- (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
- - string, App or Output, useful when the user wants to specify the input "in"
- - list, useful when the user wants to specify the input list 'il'
- frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___
- quiet: whether to print logs of the OTB app
+ def __init__(self, name: str, app: otb.Application, image_dic: dict = None):
+ """
+ name:
+ app:
image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as
the result of app.ExportImage(). Use it when the app takes a numpy array as input.
See this related issue for why it is necessary to keep reference of object:
https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824
- **kwargs: used for passing application parameters.
- e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif'
"""
- self.parameters = {}
self.name = name
- self.frozen = frozen
- self.quiet = quiet
+ self.app = app
self.image_dic = image_dic
- self.exports_dic = {}
- create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
- self.app = create(name)
+
self.parameters_keys = tuple(self.app.GetParametersKeys())
- self.time_start, self.time_end = 0, 0
self.all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys}
self.out_param_types = {k: v for k, v in self.all_param_types.items()
if v in (otb.ParameterType_OutputImage,
otb.ParameterType_OutputVectorData,
otb.ParameterType_OutputFilename)}
- if args or kwargs:
- self.set_parameters(*args, **kwargs)
- if not self.frozen:
- self.execute()
- if any(key in self.parameters for key in self.out_param_types):
- self.flush() # auto flush if any output param was provided during app init
+
+ self.transform = None # TODO: value?
+ self.exports_dic = {}
def get_first_key(self, param_types: list[str]) -> str:
"""Get the first output param key for specific file types."""
@@ -85,17 +66,6 @@ class OTBObject:
"""Get the name of first output image parameter."""
return self.get_first_key(param_types=[otb.ParameterType_OutputImage])
- @property
- def data(self):
- """Expose app's output data values in a dictionary."""
- skip_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem")
- skip_keys = skip_keys + tuple(self.out_param_types) + tuple(self.parameters)
- keys = (k for k in self.parameters_keys if k not in skip_keys)
-
- def _check(v):
- return not isinstance(v, otb.ApplicationProxy) and v not in ("", None, [], ())
- return {str(k): self[k] for k in keys if _check(self[k])}
-
@property
def metadata(self):
"""Return first output image metadata dictionary."""
@@ -145,185 +115,6 @@ class OTBObject:
origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2
return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y
- def set_parameters(self, *args, **kwargs):
- """Set some parameters of the app.
-
- When useful, e.g. for images list, this function appends the parameters
- instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths
-
- Args:
- *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
- (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
- - string, App or Output, useful when the user implicitly wants to set the param "in"
- - list, useful when the user implicitly wants to set the param "il"
- **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif'
-
- Raises:
- Exception: when the setting of a parameter failed
-
- """
- parameters = kwargs
- parameters.update(self.__parse_args(args))
- # Going through all arguments
- for key, obj in parameters.items():
- if key not in self.parameters_keys:
- raise KeyError(f'{self.name}: unknown parameter name "{key}"')
- # When the parameter expects a list, if needed, change the value to list
- if is_key_list(self, key) and not isinstance(obj, (list, tuple)):
- obj = [obj]
- logger.info('%s: argument for parameter "%s" was converted to list', self.name, key)
- try:
- # This is when we actually call self.app.SetParameter*
- self.__set_param(key, obj)
- except (RuntimeError, TypeError, ValueError, KeyError) as e:
- raise Exception(
- f"{self.name}: something went wrong before execution "
- f"(while setting parameter '{key}' to '{obj}')"
- ) from e
- # Update _parameters using values from OtbApplication object
- otb_params = self.app.GetParameters().items()
- otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params}
- # Update param dict and save values as object attributes
- self.parameters.update({**parameters, **otb_params})
- self.save_objects()
-
- def save_objects(self):
- """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`.
-
- This is useful when the key contains reserved characters such as a point eg "io.out"
- """
- for key in self.parameters_keys:
- if key in dir(self.__class__):
- continue # skip forbidden attribute since it is already used by the class
- value = self.parameters.get(key) # basic parameters
- if value is None:
- try:
- value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results)
- except RuntimeError:
- continue # this is when there is no value for key
- # Convert output param path to Output object
- if key in self.out_param_types:
- value = Output(self, key, value)
- elif isinstance(value, str):
- try:
- value = literal_eval(value)
- except (ValueError, SyntaxError):
- pass
- # Save attribute
- setattr(self, key, value)
-
- def execute(self):
- """Execute and write to disk if any output parameter has been set during init."""
- logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters)
- self.time_start = perf_counter()
- try:
- self.app.Execute()
- except (RuntimeError, FileNotFoundError) as e:
- raise Exception(f"{self.name}: error during during app execution") from e
- self.frozen = False
- self.time_end = perf_counter()
- logger.debug("%s: execution ended", self.name)
- self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics
-
- def flush(self):
- """Flush data to disk, this is when WriteOutput is actually called."""
- try:
- logger.debug("%s: flushing data to disk", self.name)
- self.app.WriteOutput()
- except RuntimeError:
- logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name)
- self.app.ExecuteAndWriteOutput()
- self.time_end = perf_counter()
-
- def write(self, *args, filename_extension: str = "", pixel_type: dict[str, str] | str = None,
- preserve_dtype: bool = False, **kwargs):
- """Set output pixel type and write the output raster files.
-
- Args:
- *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key contains
- non-standard characters such as a point, e.g. {'io.out':'output.tif'}
- - string, useful when there is only one output, e.g. 'output.tif'
- - None if output file was passed during App init
- filename_extension: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES")
- Will be used for all outputs (Default value = "")
- pixel_type: Can be : - dictionary {output_parameter_key: pixeltype} when specifying for several outputs
- - str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several
- outputs, all outputs are written with this unique type.
- Valid pixel types are uint8, uint16, uint32, int16, int32, float, double,
- cint16, cint32, cfloat, cdouble. (Default value = None)
- preserve_dtype: propagate main input pixel type to outputs, in case pixel_type is None
- **kwargs: keyword arguments e.g. out='output.tif'
-
- """
- # Gather all input arguments in kwargs dict
- for arg in args:
- if isinstance(arg, dict):
- kwargs.update(arg)
- elif isinstance(arg, str) and kwargs:
- logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg)
- elif isinstance(arg, str) and self.key_output_image:
- kwargs.update({self.key_output_image: arg})
-
- # Append filename extension to filenames
- if filename_extension:
- logger.debug("%s: using extended filename for outputs: %s", self.name, filename_extension)
- if not filename_extension.startswith("?"):
- filename_extension = "?" + filename_extension
- for key, value in kwargs.items():
- if self.out_param_types[key] == otb.ParameterType_OutputImage and '?' not in value:
- kwargs[key] = value + filename_extension
-
- # Manage output pixel types
- dtypes = {}
- if pixel_type:
- if isinstance(pixel_type, str):
- type_name = self.app.ConvertPixelTypeToNumpy(parse_pixel_type(pixel_type))
- logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name)
- for key in kwargs:
- if self.out_param_types[key] == otb.ParameterType_OutputImage:
- dtypes[key] = parse_pixel_type(pixel_type)
- elif isinstance(pixel_type, dict):
- dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()}
- elif preserve_dtype:
- self.propagate_dtype() # all outputs will have the same type as the main input raster
-
- # Set parameters and flush to disk
- for key, output_filename in kwargs.items():
- if key in dtypes:
- self.propagate_dtype(key, dtypes[key])
- self.set_parameters({key: output_filename})
- self.flush()
-
- def propagate_dtype(self, target_key: str = None, dtype: int = None):
- """Propagate a pixel type from main input to every outputs, or to a target output key only.
-
- With multiple inputs (if dtype is not provided), the type of the first input is considered.
- With multiple outputs (if target_key is not provided), all outputs will be converted to the same pixel type.
-
- Args:
- target_key: output param key to change pixel type
- dtype: data type to use
-
- """
- if not dtype:
- param = self.parameters.get(self.key_input_image)
- if not param:
- logger.warning("%s: could not propagate pixel type from inputs to output", self.name)
- return
- if isinstance(param, (list, tuple)):
- param = param[0] # first image in "il"
- try:
- dtype = get_pixel_type(param)
- except (TypeError, RuntimeError):
- logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param)
- return
- if target_key:
- keys = [target_key]
- else:
- keys = [k for k, v in self.out_param_types.items() if v == otb.ParameterType_OutputImage]
- for key in keys:
- self.app.SetParameterOutputImagePixelType(key, dtype)
-
def get_infos(self):
"""Return a dict output of ReadImageInfo for the first image output."""
if not self.key_output_image:
@@ -473,87 +264,31 @@ class OTBObject:
row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x
return abs(int(row)), int(col)
- # Private functions
- def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]:
- """Gather all input arguments in kwargs dict.
+ # Special functions
+ def __hash__(self):
+ """Override the default behaviour of the hash function.
+
+ Returns:
+ self hash
+
+ """
+ return id(self)
+
+ def __getattr__(self, name):
+ """This method is called when the default attribute access fails.
+
+ We choose to access the attribute `name` of self.app.
+ Thus, any method of otbApplication can be used transparently on OTBObject objects,
+ e.g. SetParameterOutputImagePixelType() or ExportImage() work
Args:
- args: the list of arguments passed to set_parameters()
+ name: attribute name
Returns:
- a dictionary with the right keyword depending on the object
+ attribute
- """
- kwargs = {}
- for arg in args:
- if isinstance(arg, dict):
- kwargs.update(arg)
- elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.key_input):
- kwargs.update({self.key_input: arg})
- return kwargs
-
- def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]):
- """Set one parameter, decide which otb.Application method to use depending on target object."""
- if obj is None or (isinstance(obj, (list, tuple)) and not obj):
- self.app.ClearValue(key)
- return
- if key not in self.parameters_keys:
- raise KeyError(
- f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}"
- )
- # Single-parameter cases
- if isinstance(obj, OTBObject):
- self.app.ConnectImage(key, obj.app, obj.key_output_image)
- elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB
- self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0])
- elif key == "ram": # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200
- self.app.SetParameterInt("ram", int(obj))
- elif not isinstance(obj, list): # any other parameters (str, int...)
- self.app.SetParameterValue(key, obj)
- # Images list
- elif is_key_images_list(self, key):
- # To enable possible in-memory connections, we go through the list and set the parameters one by one
- for inp in obj:
- if isinstance(inp, OTBObject):
- self.app.ConnectImage(key, inp.app, inp.key_output_image)
- elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB
- self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0])
- else: # here `input` should be an image filepath
- # Append `input` to the list, do not overwrite any previously set element of the image list
- self.app.AddParameterStringList(key, inp)
- # List of any other types (str, int...)
- else:
- self.app.SetParameterValue(key, obj)
-
- # Special functions
- def __hash__(self):
- """Override the default behaviour of the hash function.
-
- Returns:
- self hash
-
- """
- return id(self)
-
- def __str__(self):
- """Return a nice string representation with object id."""
- return f"<pyotb.App {self.name} object id {id(self)}>"
-
- def __getattr__(self, name):
- """This method is called when the default attribute access fails.
-
- We choose to access the attribute `name` of self.app.
- Thus, any method of otbApplication can be used transparently on OTBObject objects,
- e.g. SetParameterOutputImagePixelType() or ExportImage() work
-
- Args:
- name: attribute name
-
- Returns:
- attribute
-
- Raises:
- AttributeError: when `name` is not an attribute of self.app
+ Raises:
+ AttributeError: when `name` is not an attribute of self.app
"""
if name in dir(self.app):
@@ -598,7 +333,7 @@ class OTBObject:
key = key + (slice(None, None, None),)
return Slicer(self, *key)
- def __add__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __add__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default addition and flavours it with BandMathX.
Args:
@@ -612,7 +347,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("+", self, other)
- def __sub__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __sub__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default subtraction and flavours it with BandMathX.
Args:
@@ -626,7 +361,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("-", self, other)
- def __mul__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __mul__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default subtraction and flavours it with BandMathX.
Args:
@@ -640,7 +375,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("*", self, other)
- def __truediv__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __truediv__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default subtraction and flavours it with BandMathX.
Args:
@@ -654,7 +389,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("/", self, other)
- def __radd__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __radd__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default reverse addition and flavours it with BandMathX.
Args:
@@ -668,7 +403,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("+", other, self)
- def __rsub__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __rsub__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default subtraction and flavours it with BandMathX.
Args:
@@ -682,7 +417,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("-", other, self)
- def __rmul__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __rmul__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default multiplication and flavours it with BandMathX.
Args:
@@ -696,7 +431,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("*", other, self)
- def __rtruediv__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __rtruediv__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default division and flavours it with BandMathX.
Args:
@@ -719,7 +454,7 @@ class OTBObject:
"""
return Operation("abs", self)
- def __ge__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __ge__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default greater or equal and flavours it with BandMathX.
Args:
@@ -733,7 +468,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation(">=", self, other)
- def __le__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __le__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default less or equal and flavours it with BandMathX.
Args:
@@ -747,7 +482,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation("<=", self, other)
- def __gt__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __gt__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default greater operator and flavours it with BandMathX.
Args:
@@ -761,7 +496,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation(">", self, other)
- def __lt__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __lt__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default less operator and flavours it with BandMathX.
Args:
@@ -775,7 +510,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation("<", self, other)
- def __eq__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __eq__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default eq operator and flavours it with BandMathX.
Args:
@@ -789,7 +524,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation("==", self, other)
- def __ne__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __ne__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default different operator and flavours it with BandMathX.
Args:
@@ -803,7 +538,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation("!=", self, other)
- def __or__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __or__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default or operator and flavours it with BandMathX.
Args:
@@ -817,7 +552,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation("||", self, other)
- def __and__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __and__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default and operator and flavours it with BandMathX.
Args:
@@ -890,10 +625,300 @@ class OTBObject:
return NotImplemented
-class Slicer(OTBObject):
+class App(OTBObject):
+ """Base class that gathers common operations for any OTB application."""
+
+ def __init__(self, otb_app_name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None,
+ **kwargs):
+ """Common constructor for OTB applications. Handles in-memory connection between apps.
+
+ Args:
+ otb_app_name: name of the OTB application, e.g. 'BandMath'
+ *args: used for passing application parameters. Can be :
+ - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
+ (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
+ - string or OTBObject, useful when the user wants to specify the input "in"
+ - list, useful when the user wants to specify the input list 'il'
+ frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___
+ quiet: whether to print logs of the OTB app
+ image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as
+ the result of app.ExportImage(). Use it when the app takes a numpy array as input.
+ See this related issue for why it is necessary to keep reference of object:
+ https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824
+ **kwargs: used for passing application parameters.
+ e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif'
+
+ """
+
+ self.frozen = frozen
+ self.quiet = quiet
+ create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
+ super().__init__(name=f"OTB Application {otb_app_name}", app=create(otb_app_name), image_dic=image_dic)
+
+ # Set parameters
+ self.parameters = {}
+ if args or kwargs:
+ self.set_parameters(*args, **kwargs)
+ if not self.frozen:
+ self.execute()
+ if any(key in self.parameters for key in self.out_param_types):
+ self.flush() # auto flush if any output param was provided during app init
+
+ # Elapsed time
+ self.time_start, self.time_end = 0, 0
+
+ @property
+ def data(self):
+ """Expose app's output data values in a dictionary."""
+ skip_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem")
+ skip_keys = skip_keys + tuple(self.out_param_types) + tuple(self.parameters)
+ keys = (k for k in self.parameters_keys if k not in skip_keys)
+
+ def _check(v):
+ return not isinstance(v, otb.ApplicationProxy) and v not in ("", None, [], ())
+
+ return {str(k): self[k] for k in keys if _check(self[k])}
+
+ def set_parameters(self, *args, **kwargs):
+ """Set some parameters of the app.
+
+ When useful, e.g. for images list, this function appends the parameters
+ instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths
+
+ Args:
+ *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
+ (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
+ - string or OTBObject, useful when the user implicitly wants to set the param "in"
+ - list, useful when the user implicitly wants to set the param "il"
+ **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif'
+
+ Raises:
+ Exception: when the setting of a parameter failed
+
+ """
+ parameters = kwargs
+ parameters.update(self.__parse_args(args))
+ # Going through all arguments
+ for key, obj in parameters.items():
+ if key not in self.parameters_keys:
+ raise KeyError(f'{self.name}: unknown parameter name "{key}"')
+ # When the parameter expects a list, if needed, change the value to list
+ if is_key_list(self, key) and not isinstance(obj, (list, tuple)):
+ obj = [obj]
+ logger.info('%s: argument for parameter "%s" was converted to list', self.name, key)
+ try:
+ # This is when we actually call self.app.SetParameter*
+ self.__set_param(key, obj)
+ except (RuntimeError, TypeError, ValueError, KeyError) as e:
+ raise Exception(
+ f"{self.name}: something went wrong before execution "
+ f"(while setting parameter '{key}' to '{obj}')"
+ ) from e
+ # Update _parameters using values from OtbApplication object
+ otb_params = self.app.GetParameters().items()
+ otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params}
+ # Update param dict and save values as object attributes
+ self.parameters.update({**parameters, **otb_params})
+ self.save_objects()
+
+ def save_objects(self):
+ """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`.
+
+ This is useful when the key contains reserved characters such as a point eg "io.out"
+ """
+ for key in self.parameters_keys:
+ if key in dir(self.__class__):
+ continue # skip forbidden attribute since it is already used by the class
+ value = self.parameters.get(key) # basic parameters
+ if value is None:
+ try:
+ value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results)
+ except RuntimeError:
+ continue # this is when there is no value for key
+ # Convert output param path to Output object
+ if key in self.out_param_types:
+ value = Output(self, key, value)
+ elif isinstance(value, str):
+ try:
+ value = literal_eval(value)
+ except (ValueError, SyntaxError):
+ pass
+ # Save attribute
+ setattr(self, key, value)
+
+ def execute(self):
+ """Execute and write to disk if any output parameter has been set during init."""
+ logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters)
+ self.time_start = perf_counter()
+ try:
+ self.app.Execute()
+ except (RuntimeError, FileNotFoundError) as e:
+ raise Exception(f"{self.name}: error during during app execution") from e
+ self.frozen = False
+ self.time_end = perf_counter()
+ logger.debug("%s: execution ended", self.name)
+ self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics
+
+ def flush(self):
+ """Flush data to disk, this is when WriteOutput is actually called."""
+ try:
+ logger.debug("%s: flushing data to disk", self.name)
+ self.app.WriteOutput()
+ except RuntimeError:
+ logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name)
+ self.app.ExecuteAndWriteOutput()
+ self.time_end = perf_counter()
+
+ def write(self, *args, filename_extension: str = "", pixel_type: dict[str, str] | str = None,
+ preserve_dtype: bool = False, **kwargs):
+ """Set output pixel type and write the output raster files.
+
+ Args:
+ *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key contains
+ non-standard characters such as a point, e.g. {'io.out':'output.tif'}
+ - string, useful when there is only one output, e.g. 'output.tif'
+ - None if output file was passed during App init
+ filename_extension: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES")
+ Will be used for all outputs (Default value = "")
+ pixel_type: Can be : - dictionary {output_parameter_key: pixeltype} when specifying for several outputs
+ - str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several
+ outputs, all outputs are written with this unique type.
+ Valid pixel types are uint8, uint16, uint32, int16, int32, float, double,
+ cint16, cint32, cfloat, cdouble. (Default value = None)
+ preserve_dtype: propagate main input pixel type to outputs, in case pixel_type is None
+ **kwargs: keyword arguments e.g. out='output.tif'
+
+ """
+ # Gather all input arguments in kwargs dict
+ for arg in args:
+ if isinstance(arg, dict):
+ kwargs.update(arg)
+ elif isinstance(arg, str) and kwargs:
+ logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg)
+ elif isinstance(arg, str) and self.key_output_image:
+ kwargs.update({self.key_output_image: arg})
+
+ # Append filename extension to filenames
+ if filename_extension:
+ logger.debug("%s: using extended filename for outputs: %s", self.name, filename_extension)
+ if not filename_extension.startswith("?"):
+ filename_extension = "?" + filename_extension
+ for key, value in kwargs.items():
+ if self.out_param_types[key] == otb.ParameterType_OutputImage and '?' not in value:
+ kwargs[key] = value + filename_extension
+
+ # Manage output pixel types
+ dtypes = {}
+ if pixel_type:
+ if isinstance(pixel_type, str):
+ type_name = self.app.ConvertPixelTypeToNumpy(parse_pixel_type(pixel_type))
+ logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name)
+ for key in kwargs:
+ if self.out_param_types[key] == otb.ParameterType_OutputImage:
+ dtypes[key] = parse_pixel_type(pixel_type)
+ elif isinstance(pixel_type, dict):
+ dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()}
+ elif preserve_dtype:
+ self.propagate_dtype() # all outputs will have the same type as the main input raster
+
+ # Set parameters and flush to disk
+ for key, output_filename in kwargs.items():
+ if key in dtypes:
+ self.propagate_dtype(key, dtypes[key])
+ self.set_parameters({key: output_filename})
+ self.flush()
+
+ def propagate_dtype(self, target_key: str = None, dtype: int = None):
+ """Propagate a pixel type from main input to every outputs, or to a target output key only.
+
+ With multiple inputs (if dtype is not provided), the type of the first input is considered.
+ With multiple outputs (if target_key is not provided), all outputs will be converted to the same pixel type.
+
+ Args:
+ target_key: output param key to change pixel type
+ dtype: data type to use
+
+ """
+ if not dtype:
+ param = self.parameters.get(self.key_input_image)
+ if not param:
+ logger.warning("%s: could not propagate pixel type from inputs to output", self.name)
+ return
+ if isinstance(param, (list, tuple)):
+ param = param[0] # first image in "il"
+ try:
+ dtype = get_pixel_type(param)
+ except (TypeError, RuntimeError):
+ logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param)
+ return
+ if target_key:
+ keys = [target_key]
+ else:
+ keys = [k for k, v in self.out_param_types.items() if v == otb.ParameterType_OutputImage]
+ for key in keys:
+ self.app.SetParameterOutputImagePixelType(key, dtype)
+
+ # Private functions
+ def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]:
+ """Gather all input arguments in kwargs dict.
+
+ Args:
+ args: the list of arguments passed to set_parameters()
+
+ Returns:
+ a dictionary with the right keyword depending on the object
+
+ """
+ kwargs = {}
+ for arg in args:
+ if isinstance(arg, dict):
+ kwargs.update(arg)
+ elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.key_input):
+ kwargs.update({self.key_input: arg})
+ return kwargs
+
+ def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]):
+ """Set one parameter, decide which otb.Application method to use depending on target object."""
+ if obj is None or (isinstance(obj, (list, tuple)) and not obj):
+ self.app.ClearValue(key)
+ return
+ if key not in self.parameters_keys:
+ raise KeyError(
+ f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}"
+ )
+ # Single-parameter cases
+ if isinstance(obj, OTBObject):
+ self.app.ConnectImage(key, obj.app, obj.key_output_image)
+ elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB
+ self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0])
+ elif key == "ram": # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200
+ self.app.SetParameterInt("ram", int(obj))
+ elif not isinstance(obj, list): # any other parameters (str, int...)
+ self.app.SetParameterValue(key, obj)
+ # Images list
+ elif is_key_images_list(self, key):
+ # To enable possible in-memory connections, we go through the list and set the parameters one by one
+ for inp in obj:
+ if isinstance(inp, OTBObject):
+ self.app.ConnectImage(key, inp.app, inp.key_output_image)
+ elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB
+ self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0])
+ else: # here `input` should be an image filepath
+ # Append `input` to the list, do not overwrite any previously set element of the image list
+ self.app.AddParameterStringList(key, inp)
+ # List of any other types (str, int...)
+ else:
+ self.app.SetParameterValue(key, obj)
+
+ def __str__(self):
+ """Return a nice string representation with object id."""
+ return f"<pyotb.App {self.name} object id {id(self)}>"
+
+
+class Slicer(App):
"""Slicer objects i.e. when we call something like raster[:, :, 2] from Python."""
- def __init__(self, obj: OTBObject | Output | str, rows: int, cols: int, channels: int):
+ def __init__(self, obj: OTBObject | str, rows: int, cols: int, channels: int):
"""Create a slicer object, that can be used directly for writing or inside a BandMath.
It contains :
@@ -957,7 +982,7 @@ class Slicer(OTBObject):
self.execute()
-class Operation(OTBObject):
+class Operation(App):
"""Class for arithmetic/math operations done in Python.
Example:
@@ -986,7 +1011,7 @@ class Operation(OTBObject):
Args:
operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
- *inputs: inputs. Can be App, Output, Input, Operation, Slicer, filepath, int or float
+ *inputs: inputs. Can be OTBObject, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
@@ -1018,7 +1043,7 @@ class Operation(OTBObject):
super().__init__(name, il=self.unique_inputs, exp=self.exp, quiet=True)
self.name = f'Operation exp="{self.exp}"'
- def create_fake_exp(self, operator: str, inputs: list[OTBObject | Output | str | int | float],
+ def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float],
nb_bands: int = None):
"""Create a 'fake' expression.
@@ -1026,7 +1051,7 @@ class Operation(OTBObject):
Args:
operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
- inputs: inputs. Can be App, Output, Input, Operation, Slicer, filepath, int or float
+ inputs: inputs. Can be OTBObject, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
@@ -1110,7 +1135,7 @@ class Operation(OTBObject):
return exp_bands, exp
@staticmethod
- def create_one_input_fake_exp(x: OTBObject | Output | str,
+ def create_one_input_fake_exp(x: OTBObject | str,
band: int, keep_logical: bool = False) -> tuple(str, list[OTBObject], int):
"""This an internal function, only to be used by `create_fake_exp`.
@@ -1193,7 +1218,7 @@ class LogicalOperation(Operation):
super().__init__(operator, *inputs, nb_bands=nb_bands)
self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands)
- def create_fake_exp(self, operator: str, inputs: list[OTBObject | Output | str | int | float],
+ def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float],
nb_bands: int = None):
"""Create a 'fake' expression.
@@ -1202,7 +1227,7 @@ class LogicalOperation(Operation):
Args:
operator: str (one of >, <, >=, <=, ==, !=, &, |)
- inputs: Can be App, Output, Input, Operation, Slicer, filepath, int or float
+ inputs: Can be OTBObject, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
@@ -1237,7 +1262,7 @@ class LogicalOperation(Operation):
self.fake_exp_bands.append(fake_exp)
-class Input(OTBObject):
+class Input(App):
"""Class for transforming a filepath to pyOTB object."""
def __init__(self, path: str):
@@ -1272,9 +1297,7 @@ class Output(OTBObject):
mkdir: create missing parent directories
"""
- self.pyotb_app, self.app = pyotb_app, pyotb_app.app
- self.parameters = pyotb_app.parameters
- self.param_key = param_key
+ super().__init__(name=f"Output {param_key} from {self.pyotb_app.name}", app=pyotb_app)
self.filepath = None
if filepath:
if '?' in filepath:
@@ -1282,7 +1305,6 @@ class Output(OTBObject):
self.filepath = Path(filepath)
if mkdir:
self.make_parent_dirs()
- self.name = f"Output {param_key} from {self.pyotb_app.name}"
@property
def key_output_image(self):
--
GitLab
From 2af2c562ab9c15e10cf86120aa3bd875c86f51ce Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 11:13:36 +0000
Subject: [PATCH 052/399] WIP: code and tests
---
pyotb/apps.py | 37 +------------------------------------
pyotb/core.py | 26 ++++++++++++++++++++++++++
tests/bug_todel.py | 7 +++++++
tests/bug_todel2.py | 10 ++++++++++
tests/test_core.py | 11 ++++-------
tests/test_numpy.py | 10 +++++-----
tests/test_pipeline.py | 2 --
tests/test_serialization.py | 14 ++++++--------
tests/test_todel.py | 6 ++++++
tests/tests_data.py | 3 +++
10 files changed, 68 insertions(+), 58 deletions(-)
create mode 100644 tests/bug_todel.py
create mode 100644 tests/bug_todel2.py
create mode 100644 tests/test_todel.py
create mode 100644 tests/tests_data.py
diff --git a/pyotb/apps.py b/pyotb/apps.py
index ad71f54..9421b28 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -7,7 +7,7 @@ import subprocess
from pathlib import Path
import otbApplication as otb # pylint: disable=import-error
-from .core import OTBObject
+from .core import App
from .helpers import logger
@@ -59,41 +59,6 @@ def get_available_applications(as_subprocess: bool = False) -> list[str]:
return app_list
-class App(OTBObject):
- """Base class for UI related functions, will be subclassed using app name as class name, see CODE_TEMPLATE."""
- _name = ""
-
- def __init__(self, *args, **kwargs):
- """Default App constructor, adds UI specific attributes and functions."""
- super().__init__(*args, **kwargs)
- self.description = self.app.GetDocLongDescription()
-
- @property
- def elapsed_time(self):
- """Get elapsed time between app init and end of exec or file writing."""
- return self.time_end - self.time_start
-
- @property
- def used_outputs(self) -> list[str]:
- """List of used application outputs."""
- return [getattr(self, key) for key in self.out_param_types if key in self.parameters]
-
- def find_outputs(self) -> tuple[str]:
- """Find output files on disk using path found in parameters.
-
- Returns:
- list of files found on disk
-
- """
- files, missing = [], []
- for out in self.used_outputs:
- dest = files if out.exists() else missing
- dest.append(str(out.filepath.absolute()))
- for filename in missing:
- logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
- return tuple(files)
-
-
class OTBTFApp(App):
"""Helper for OTBTF."""
@staticmethod
diff --git a/pyotb/core.py b/pyotb/core.py
index 87bb08c..b97c195 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -654,6 +654,7 @@ class App(OTBObject):
self.quiet = quiet
create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
super().__init__(name=f"OTB Application {otb_app_name}", app=create(otb_app_name), image_dic=image_dic)
+ self.description = self.app.GetDocLongDescription()
# Set parameters
self.parameters = {}
@@ -667,6 +668,31 @@ class App(OTBObject):
# Elapsed time
self.time_start, self.time_end = 0, 0
+ @property
+ def elapsed_time(self):
+ """Get elapsed time between app init and end of exec or file writing."""
+ return self.time_end - self.time_start
+
+ @property
+ def used_outputs(self) -> list[str]:
+ """List of used application outputs."""
+ return [getattr(self, key) for key in self.out_param_types if key in self.parameters]
+
+ def find_outputs(self) -> tuple[str]:
+ """Find output files on disk using path found in parameters.
+
+ Returns:
+ list of files found on disk
+
+ """
+ files, missing = [], []
+ for out in self.used_outputs:
+ dest = files if out.exists() else missing
+ dest.append(str(out.filepath.absolute()))
+ for filename in missing:
+ logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
+ return tuple(files)
+
@property
def data(self):
"""Expose app's output data values in a dictionary."""
diff --git a/tests/bug_todel.py b/tests/bug_todel.py
new file mode 100644
index 0000000..ae3e64d
--- /dev/null
+++ b/tests/bug_todel.py
@@ -0,0 +1,7 @@
+import pyotb
+
+img_pth = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/" \
+ "otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false"
+bm = pyotb.BandMath({"il": [img_pth], "exp": "im1b1"})
+bm.write("/tmp/toto.tif") # Comment this line --> Works
+print(bm.shape)
\ No newline at end of file
diff --git a/tests/bug_todel2.py b/tests/bug_todel2.py
new file mode 100644
index 0000000..cafc42d
--- /dev/null
+++ b/tests/bug_todel2.py
@@ -0,0 +1,10 @@
+import otbApplication
+img_pth = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false"
+app = otbApplication.Registry.CreateApplication("BandMath")
+app.SetParameterStringList("il", [img_pth])
+app.SetParameterString("exp", "im1b1")
+app.Execute()
+print(app.GetImageSize("out")) # WORKS
+app.SetParameterString("out", "/tmp/toto.tif")
+app.WriteOutput()
+print(app.GetImageSize("out")) # ERROR
diff --git a/tests/test_core.py b/tests/test_core.py
index 436f99e..086c26e 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -1,13 +1,8 @@
-import os
-import pyotb
-from ast import literal_eval
-from pathlib import Path
-
import pytest
+import pyotb
+from tests_data import INPUT
-FILEPATH = os.environ["TEST_INPUT_IMAGE"]
-INPUT = pyotb.Input(FILEPATH)
TEST_IMAGE_STATS = {
'out.mean': [79.5505, 109.225, 115.456, 249.349],
'out.min': [33, 64, 91, 47],
@@ -59,9 +54,11 @@ def test_nonraster_property():
with pytest.raises(TypeError):
pyotb.ReadImageInfo(INPUT).dtype
+
def test_elapsed_time():
assert pyotb.ReadImageInfo(INPUT).elapsed_time < 1
+
# Other functions
def test_get_infos():
infos = INPUT.get_infos()
diff --git a/tests/test_numpy.py b/tests/test_numpy.py
index dd33425..0f42435 100644
--- a/tests/test_numpy.py
+++ b/tests/test_numpy.py
@@ -1,10 +1,10 @@
-import os
import numpy as np
import pyotb
+from tests_data import INPUT
+import numpy as np
-
-FILEPATH = os.environ["TEST_INPUT_IMAGE"]
-INPUT = pyotb.Input(FILEPATH)
+import pyotb
+from tests_data import INPUT
def test_export():
@@ -37,7 +37,7 @@ def test_convert_to_array():
def test_pixel_coords_otb_equals_numpy():
- assert INPUT[19,7] == list(INPUT.to_numpy()[19,7])
+ assert INPUT[19, 7] == list(INPUT.to_numpy()[19, 7])
def test_add_noise_array():
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index 3c88153..f3fe4cf 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -1,9 +1,7 @@
-import sys
import os
import itertools
import pytest
import pyotb
-from pyotb.helpers import logger
# List of buildings blocks, we can add other pyotb objects here
diff --git a/tests/test_serialization.py b/tests/test_serialization.py
index e64f2f1..b56e447 100644
--- a/tests/test_serialization.py
+++ b/tests/test_serialization.py
@@ -1,18 +1,16 @@
-import os
import pyotb
-
-filepath = os.environ["TEST_INPUT_IMAGE"]
+from tests_data import FILEPATH
def test_pipeline_simple():
# BandMath -> OrthoRectification -> ManageNoData
- app1 = pyotb.BandMath({'il': [filepath], 'exp': 'im1b1'})
+ app1 = pyotb.BandMath({'il': [FILEPATH], 'exp': 'im1b1'})
app2 = pyotb.OrthoRectification({'io.in': app1})
app3 = pyotb.ManageNoData({'in': app2})
summary = app3.summarize()
reference = {'name': 'ManageNoData', 'parameters': {'in': {
'name': 'OrthoRectification', 'parameters': {'io.in': {
- 'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}},
+ 'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}},
'map': 'utm',
'outputs.isotropic': True}},
'mode': 'buildmask'}}
@@ -21,19 +19,19 @@ def test_pipeline_simple():
def test_pipeline_diamond():
# Diamond graph
- app1 = pyotb.BandMath({'il': [filepath], 'exp': 'im1b1'})
+ app1 = pyotb.BandMath({'il': [FILEPATH], 'exp': 'im1b1'})
app2 = pyotb.OrthoRectification({'io.in': app1})
app3 = pyotb.ManageNoData({'in': app2})
app4 = pyotb.BandMathX({'il': [app2, app3], 'exp': 'im1+im2'})
summary = app4.summarize()
reference = {'name': 'BandMathX', 'parameters': {'il': [
{'name': 'OrthoRectification', 'parameters': {'io.in': {
- 'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}},
+ 'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}},
'map': 'utm',
'outputs.isotropic': True}},
{'name': 'ManageNoData', 'parameters': {'in': {
'name': 'OrthoRectification', 'parameters': {
- 'io.in': {'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}},
+ 'io.in': {'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}},
'map': 'utm',
'outputs.isotropic': True}},
'mode': 'buildmask'}}
diff --git a/tests/test_todel.py b/tests/test_todel.py
new file mode 100644
index 0000000..fa70ec7
--- /dev/null
+++ b/tests/test_todel.py
@@ -0,0 +1,6 @@
+import pyotb
+
+img_pth = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/" \
+ "otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false"
+bm = pyotb.MeanShiftSmoothing({"in": img_pth, "fout": "/tmp/toto.tif", "foutpos": "/tmp/titi.tif"})
+print(bm.find_outputs())
\ No newline at end of file
diff --git a/tests/tests_data.py b/tests/tests_data.py
new file mode 100644
index 0000000..b190f40
--- /dev/null
+++ b/tests/tests_data.py
@@ -0,0 +1,3 @@
+import pyotb
+FILEPATH = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false"
+INPUT = pyotb.Input(FILEPATH)
--
GitLab
From b773a4bf06ea2794438ab27f161da899654bc481 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 12:26:08 +0100
Subject: [PATCH 053/399] WIP: code and tests
---
pyotb/core.py | 30 +++++++++++++++---------------
1 file changed, 15 insertions(+), 15 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index b97c195..8cf8d71 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -629,7 +629,7 @@ class App(OTBObject):
"""Base class that gathers common operations for any OTB application."""
def __init__(self, otb_app_name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None,
- **kwargs):
+ name: str = None, **kwargs):
"""Common constructor for OTB applications. Handles in-memory connection between apps.
Args:
@@ -645,6 +645,7 @@ class App(OTBObject):
the result of app.ExportImage(). Use it when the app takes a numpy array as input.
See this related issue for why it is necessary to keep reference of object:
https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824
+ name: override the application name
**kwargs: used for passing application parameters.
e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif'
@@ -653,7 +654,7 @@ class App(OTBObject):
self.frozen = frozen
self.quiet = quiet
create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
- super().__init__(name=f"OTB Application {otb_app_name}", app=create(otb_app_name), image_dic=image_dic)
+ super().__init__(name=name or f"OTB Application {otb_app_name}", app=create(otb_app_name), image_dic=image_dic)
self.description = self.app.GetDocLongDescription()
# Set parameters
@@ -958,8 +959,8 @@ class Slicer(App):
channels: channels, can be slicing, list or int
"""
- super().__init__("ExtractROI", {"in": obj, "mode": "extent"}, quiet=True, frozen=True)
- self.name = "Slicer"
+ super().__init__(otb_app_name="ExtractROI", args={"in": obj, "mode": "extent"}, quiet=True, frozen=True,
+ name="Slicer")
self.rows, self.cols = rows, cols
parameters = {}
@@ -1029,7 +1030,7 @@ class Operation(App):
"""
- def __init__(self, operator: str, *inputs, nb_bands: int = None):
+ def __init__(self, operator: str, *inputs, nb_bands: int = None, name: str = None):
"""Given some inputs and an operator, this function enables to transform this into an OTB application.
Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator.
@@ -1039,6 +1040,7 @@ class Operation(App):
operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
*inputs: inputs. Can be OTBObject, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
+ name: override the Operation name
"""
self.operator = operator
@@ -1065,9 +1067,8 @@ class Operation(App):
self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)]
self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands)
# Execute app
- name = "BandMath" if len(self.exp_bands) == 1 else "BandMathX"
- super().__init__(name, il=self.unique_inputs, exp=self.exp, quiet=True)
- self.name = f'Operation exp="{self.exp}"'
+ super().__init__(otb_app_name="BandMath" if len(self.exp_bands) == 1 else "BandMathX", il=self.unique_inputs,
+ exp=self.exp, quiet=True, name=name or f'Operation exp="{self.exp}"')
def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float],
nb_bands: int = None):
@@ -1241,14 +1242,13 @@ class LogicalOperation(Operation):
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
- super().__init__(operator, *inputs, nb_bands=nb_bands)
+ super().__init__(operator=operator, inputs=inputs, nb_bands=nb_bands, name="LogicalOperation")
self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands)
- def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float],
- nb_bands: int = None):
+ def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None):
"""Create a 'fake' expression.
- E.g for the operation input1 > input2, we create a fake expression that is like
+ e.g for the operation input1 > input2, we create a fake expression that is like
"str(input1) > str(input2) ? 1 : 0" and a logical fake expression that is like "str(input1) > str(input2)"
Args:
@@ -1299,8 +1299,7 @@ class Input(App):
"""
self.path = path
- super().__init__("ExtractROI", {"in": path}, frozen=True)
- self.name = f"Input from {path}"
+ super().__init__(otb_app_name="ExtractROI", args={"in": path}, frozen=True, name=f"Input from {path}")
self.propagate_dtype()
self.execute()
@@ -1323,7 +1322,8 @@ class Output(OTBObject):
mkdir: create missing parent directories
"""
- super().__init__(name=f"Output {param_key} from {self.pyotb_app.name}", app=pyotb_app)
+ super().__init__(name=f"Output {param_key} from {self.pyotb_app.name}", app=pyotb_app.app)
+ self.parent_pyotb_app = pyotb_app # keep trace of parent app
self.filepath = None
if filepath:
if '?' in filepath:
--
GitLab
From 6a6edc07b1277535c29c7046404f51a201b154a1 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 11:28:14 +0000
Subject: [PATCH 054/399] WIP: code and tests
---
pyotb/core.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 8cf8d71..96a1f97 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -36,7 +36,6 @@ class OTBObject:
otb.ParameterType_OutputVectorData,
otb.ParameterType_OutputFilename)}
- self.transform = None # TODO: value?
self.exports_dic = {}
def get_first_key(self, param_types: list[str]) -> str:
--
GitLab
From e6738110f00f4f85f1fbc2297f6f2ce0a4befcbc Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 11:33:39 +0000
Subject: [PATCH 055/399] WIP: code and tests
---
pyotb/core.py | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 96a1f97..6c16a4f 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -958,8 +958,7 @@ class Slicer(App):
channels: channels, can be slicing, list or int
"""
- super().__init__(otb_app_name="ExtractROI", args={"in": obj, "mode": "extent"}, quiet=True, frozen=True,
- name="Slicer")
+ super().__init__("ExtractROI", {"in": obj, "mode": "extent"}, quiet=True, frozen=True, name="Slicer")
self.rows, self.cols = rows, cols
parameters = {}
@@ -1066,7 +1065,7 @@ class Operation(App):
self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)]
self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands)
# Execute app
- super().__init__(otb_app_name="BandMath" if len(self.exp_bands) == 1 else "BandMathX", il=self.unique_inputs,
+ super().__init__("BandMath" if len(self.exp_bands) == 1 else "BandMathX", il=self.unique_inputs,
exp=self.exp, quiet=True, name=name or f'Operation exp="{self.exp}"')
def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float],
@@ -1298,7 +1297,7 @@ class Input(App):
"""
self.path = path
- super().__init__(otb_app_name="ExtractROI", args={"in": path}, frozen=True, name=f"Input from {path}")
+ super().__init__("ExtractROI", {"in": path}, frozen=True, name=f"Input from {path}")
self.propagate_dtype()
self.execute()
--
GitLab
From 3745d6f55861815e5824326b374ac87ea2ce19aa Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 12:49:50 +0100
Subject: [PATCH 056/399] WIP: code and tests
---
pyotb/core.py | 49 +++++++++++++++++++++++-------------------
tests/test_pipeline.py | 3 +--
2 files changed, 28 insertions(+), 24 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 6c16a4f..ce30564 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -273,26 +273,31 @@ class OTBObject:
"""
return id(self)
- def __getattr__(self, name):
- """This method is called when the default attribute access fails.
-
- We choose to access the attribute `name` of self.app.
- Thus, any method of otbApplication can be used transparently on OTBObject objects,
- e.g. SetParameterOutputImagePixelType() or ExportImage() work
-
- Args:
- name: attribute name
-
- Returns:
- attribute
-
- Raises:
- AttributeError: when `name` is not an attribute of self.app
-
- """
- if name in dir(self.app):
- return getattr(self.app, name)
- raise AttributeError(f"{self.name}: could not find attribute `{name}`")
+ # TODO: Remove completely this code? I think this is (1) dangerous, because it
+ # can break things in otb.Application, just by setting some mutable attribute,
+ # (2) harder to debug. And (3) I feel that this is the wrong design. The good
+ # design would be to inherit from otb.Application.
+ #
+ # def __getattr__(self, name):
+ # """This method is called when the default attribute access fails.
+ #
+ # We choose to access the attribute `name` of self.app.
+ # Thus, any method of otbApplication can be used transparently on OTBObject objects,
+ # e.g. SetParameterOutputImagePixelType() or ExportImage() work
+ #
+ # Args:
+ # name: attribute name
+ #
+ # Returns:
+ # attribute
+ #
+ # Raises:
+ # AttributeError: when `name` is not an attribute of self.app
+ #
+ # """
+ # if name in dir(self.app):
+ # return getattr(self.app, name)
+ # raise AttributeError(f"{self.name}: could not find attribute `{name}`")
def __getitem__(self, key):
"""Override the default __getitem__ behaviour.
@@ -1320,7 +1325,7 @@ class Output(OTBObject):
mkdir: create missing parent directories
"""
- super().__init__(name=f"Output {param_key} from {self.pyotb_app.name}", app=pyotb_app.app)
+ super().__init__(name=f"Output {param_key} from {pyotb_app.name}", app=pyotb_app.app)
self.parent_pyotb_app = pyotb_app # keep trace of parent app
self.filepath = None
if filepath:
@@ -1405,7 +1410,7 @@ def get_pixel_type(inp: str | OTBObject) -> str:
raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}")
pixel_type = getattr(otb, f'ImagePixelType_{datatype_to_pixeltype[datatype]}')
elif isinstance(inp, OTBObject):
- pixel_type = inp.GetParameterOutputImagePixelType(inp.key_output_image)
+ pixel_type = inp.app.GetParameterOutputImagePixelType(inp.key_output_image)
else:
raise TypeError(f'Could not get the pixel type of {type(inp)} object {inp}')
return pixel_type
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index f3fe4cf..3acc354 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -2,6 +2,7 @@ import os
import itertools
import pytest
import pyotb
+from tests_data import INPUT, FILEPATH
# List of buildings blocks, we can add other pyotb objects here
@@ -21,8 +22,6 @@ PYOTB_BLOCKS = [
PIPELINES_LENGTH = [1, 2, 3]
ALL_BLOCKS = PYOTB_BLOCKS + OTBAPPS_BLOCKS
-FILEPATH = os.environ["TEST_INPUT_IMAGE"]
-INPUT = pyotb.Input(FILEPATH)
def generate_pipeline(inp, building_blocks):
--
GitLab
From ef5f6be2c54a8855b799d032a22e9ec8d498748b Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 12:59:13 +0100
Subject: [PATCH 057/399] WIP: code and tests
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index ce30564..9dd40c1 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1389,7 +1389,7 @@ def get_pixel_type(inp: str | OTBObject) -> str:
"""
if isinstance(inp, str):
try:
- info = OTBObject("ReadImageInfo", inp, quiet=True)
+ info = App("ReadImageInfo", inp, quiet=True)
except Exception as info_err: # this happens when we pass a str that is not a filepath
raise TypeError(f"Could not get the pixel type of `{inp}`. Not a filepath or wrong filepath") from info_err
datatype = info.GetParameterString("datatype") # which is such as short, float...
--
GitLab
From 17f0b6e519cf8ae24f766774263061a9751e3026 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 13:08:10 +0100
Subject: [PATCH 058/399] WIP: code and tests
---
pyotb/core.py | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 9dd40c1..3801dfe 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -118,13 +118,13 @@ class OTBObject:
"""Return a dict output of ReadImageInfo for the first image output."""
if not self.key_output_image:
raise TypeError(f"{self.name}: this application has no raster output")
- return OTBObject("ReadImageInfo", self, quiet=True).data
+ return App("ReadImageInfo", self, quiet=True).data
def get_statistics(self):
"""Return a dict output of ComputeImagesStatistics for the first image output."""
if not self.key_output_image:
raise TypeError(f"{self.name}: this application has no raster output")
- return OTBObject("ComputeImagesStatistics", self, quiet=True).data
+ return App("ComputeImagesStatistics", self, quiet=True).data
def read_values_at_coords(self, row: int, col: int, bands: int = None) -> list[int | float] | int | float:
"""Get pixel value(s) at a given YX coordinates.
@@ -139,7 +139,7 @@ class OTBObject:
"""
channels = []
- app = OTBObject("PixelValue", self, coordx=col, coordy=row, frozen=True, quiet=True)
+ app = App("PixelValue", self, coordx=col, coordy=row, frozen=True, quiet=True)
if bands is not None:
if isinstance(bands, int):
if bands < 0:
@@ -619,7 +619,7 @@ class OTBObject:
result_dic = image_dic
result_dic["array"] = result_array
# Importing back to OTB, pass the result_dic just to keep reference
- app = OTBObject("ExtractROI", image_dic=result_dic, frozen=True, quiet=True)
+ app = App("ExtractROI", image_dic=result_dic, frozen=True, quiet=True)
if result_array.shape[2] == 1:
app.ImportImage("in", result_dic)
else:
@@ -1245,7 +1245,7 @@ class LogicalOperation(Operation):
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
- super().__init__(operator=operator, inputs=inputs, nb_bands=nb_bands, name="LogicalOperation")
+ super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation")
self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands)
def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None):
@@ -1315,7 +1315,7 @@ class Output(OTBObject):
"""Object that behave like a pointer to a specific application output file."""
def __init__(self, pyotb_app: OTBObject, # pylint: disable=super-init-not-called
- param_key: str, filepath: str = None, mkdir: bool = True):
+ param_key: str = None, filepath: str = None, mkdir: bool = True):
"""Constructor for an Output object.
Args:
@@ -1327,6 +1327,7 @@ class Output(OTBObject):
"""
super().__init__(name=f"Output {param_key} from {pyotb_app.name}", app=pyotb_app.app)
self.parent_pyotb_app = pyotb_app # keep trace of parent app
+ self.param_key = param_key or super().key_output_image
self.filepath = None
if filepath:
if '?' in filepath:
@@ -1342,10 +1343,12 @@ class Output(OTBObject):
def exists(self) -> bool:
"""Check file exist."""
+ assert self.filepath, "Filepath not set"
return self.filepath.exists()
def make_parent_dirs(self):
"""Create missing parent directories."""
+ assert self.filepath, "Filepath not set"
if not self.filepath.parent.exists():
self.filepath.parent.mkdir(parents=True)
@@ -1369,7 +1372,7 @@ def get_nbchannels(inp: str | OTBObject) -> int:
else:
# Executing the app, without printing its log
try:
- info = OTBObject("ReadImageInfo", inp, quiet=True)
+ info = App("ReadImageInfo", inp, quiet=True)
nb_channels = info.GetParameterInt("numberbands")
except Exception as e: # this happens when we pass a str that is not a filepath
raise TypeError(f'Could not get the number of channels of `{inp}`. Not a filepath or wrong filepath') from e
@@ -1392,7 +1395,7 @@ def get_pixel_type(inp: str | OTBObject) -> str:
info = App("ReadImageInfo", inp, quiet=True)
except Exception as info_err: # this happens when we pass a str that is not a filepath
raise TypeError(f"Could not get the pixel type of `{inp}`. Not a filepath or wrong filepath") from info_err
- datatype = info.GetParameterString("datatype") # which is such as short, float...
+ datatype = info.app.GetParameterString("datatype") # which is such as short, float...
if not datatype:
raise TypeError(f"Unable to read pixel type of image {inp}")
datatype_to_pixeltype = {
--
GitLab
From 6d7e798f4e0e99786f90bf544990d0ac94536a53 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 12:47:05 +0000
Subject: [PATCH 059/399] WIP: code and tests
---
pyotb/core.py | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 3801dfe..077b373 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -94,7 +94,7 @@ class OTBObject:
"""
if not self.key_output_image:
- raise TypeError(f"{self.name}: this application has no raster output")
+ raise TypeError(f"\"{self.name}\" has no raster output")
width, height = self.app.GetImageSize(self.key_output_image)
bands = self.app.GetImageNbBands(self.key_output_image)
return height, width, bands
@@ -189,7 +189,7 @@ class OTBObject:
params[k] = p.summarize()
elif isinstance(p, list): # parameter list
params[k] = [pi.summarize() if isinstance(pi, OTBObject) else pi for pi in p]
- return {"name": self.name, "parameters": params}
+ return {"name": self.app.GetName(), "parameters": params}
def export(self, key: str = None, preserve_dtype: bool = True) -> dict[str, dict[str, np.ndarray]]:
"""Export a specific output image as numpy array and store it in object exports_dic.
@@ -619,13 +619,13 @@ class OTBObject:
result_dic = image_dic
result_dic["array"] = result_array
# Importing back to OTB, pass the result_dic just to keep reference
- app = App("ExtractROI", image_dic=result_dic, frozen=True, quiet=True)
+ pyotb_app = App("ExtractROI", image_dic=result_dic, frozen=True, quiet=True)
if result_array.shape[2] == 1:
- app.ImportImage("in", result_dic)
+ pyotb_app.app.ImportImage("in", result_dic)
else:
- app.ImportVectorImage("in", result_dic)
- app.execute()
- return app
+ pyotb_app.app.ImportVectorImage("in", result_dic)
+ pyotb_app.execute()
+ return pyotb_app
return NotImplemented
--
GitLab
From 9b3afb6e061bd4ad9504a504f62c69e5b4ef14a5 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 12:58:56 +0000
Subject: [PATCH 060/399] DOC: code and tests
---
pyotb/core.py | 40 ++++++++----------------------------
tests/bug_todel.py | 7 -------
tests/bug_todel2.py | 10 ---------
tests/test-core.xml | 1 +
tests/test-numpy.xml | 1 +
tests/test-pipeline.xml | 1 +
tests/test-serialization.xml | 1 +
7 files changed, 13 insertions(+), 48 deletions(-)
delete mode 100644 tests/bug_todel.py
delete mode 100644 tests/bug_todel2.py
create mode 100644 tests/test-core.xml
create mode 100644 tests/test-numpy.xml
create mode 100644 tests/test-pipeline.xml
create mode 100644 tests/test-serialization.xml
diff --git a/pyotb/core.py b/pyotb/core.py
index 077b373..0f04689 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -14,12 +14,17 @@ from .helpers import logger
class OTBObject:
+ """
+ Base class for all pyotb objects
+ """
+
def __init__(self, name: str, app: otb.Application, image_dic: dict = None):
- """
- name:
- app:
+ """Constructor for an OTBObject.
+
+ name: name of the object (e.g. "Slicer")
+ app: OTB application instance
image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as
- the result of app.ExportImage(). Use it when the app takes a numpy array as input.
+ the result of self.app.ExportImage(). Use it when the app takes a numpy array as input.
See this related issue for why it is necessary to keep reference of object:
https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824
@@ -273,32 +278,6 @@ class OTBObject:
"""
return id(self)
- # TODO: Remove completely this code? I think this is (1) dangerous, because it
- # can break things in otb.Application, just by setting some mutable attribute,
- # (2) harder to debug. And (3) I feel that this is the wrong design. The good
- # design would be to inherit from otb.Application.
- #
- # def __getattr__(self, name):
- # """This method is called when the default attribute access fails.
- #
- # We choose to access the attribute `name` of self.app.
- # Thus, any method of otbApplication can be used transparently on OTBObject objects,
- # e.g. SetParameterOutputImagePixelType() or ExportImage() work
- #
- # Args:
- # name: attribute name
- #
- # Returns:
- # attribute
- #
- # Raises:
- # AttributeError: when `name` is not an attribute of self.app
- #
- # """
- # if name in dir(self.app):
- # return getattr(self.app, name)
- # raise AttributeError(f"{self.name}: could not find attribute `{name}`")
-
def __getitem__(self, key):
"""Override the default __getitem__ behaviour.
@@ -654,7 +633,6 @@ class App(OTBObject):
e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif'
"""
-
self.frozen = frozen
self.quiet = quiet
create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
diff --git a/tests/bug_todel.py b/tests/bug_todel.py
deleted file mode 100644
index ae3e64d..0000000
--- a/tests/bug_todel.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import pyotb
-
-img_pth = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/" \
- "otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false"
-bm = pyotb.BandMath({"il": [img_pth], "exp": "im1b1"})
-bm.write("/tmp/toto.tif") # Comment this line --> Works
-print(bm.shape)
\ No newline at end of file
diff --git a/tests/bug_todel2.py b/tests/bug_todel2.py
deleted file mode 100644
index cafc42d..0000000
--- a/tests/bug_todel2.py
+++ /dev/null
@@ -1,10 +0,0 @@
-import otbApplication
-img_pth = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false"
-app = otbApplication.Registry.CreateApplication("BandMath")
-app.SetParameterStringList("il", [img_pth])
-app.SetParameterString("exp", "im1b1")
-app.Execute()
-print(app.GetImageSize("out")) # WORKS
-app.SetParameterString("out", "/tmp/toto.tif")
-app.WriteOutput()
-print(app.GetImageSize("out")) # ERROR
diff --git a/tests/test-core.xml b/tests/test-core.xml
new file mode 100644
index 0000000..718a6f5
--- /dev/null
+++ b/tests/test-core.xml
@@ -0,0 +1 @@
+<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="0" time="0.007" timestamp="2022-09-20T19:17:53.478514" hostname="787f55c467b8" /></testsuites>
\ No newline at end of file
diff --git a/tests/test-numpy.xml b/tests/test-numpy.xml
new file mode 100644
index 0000000..f3cd59b
--- /dev/null
+++ b/tests/test-numpy.xml
@@ -0,0 +1 @@
+<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="0" time="0.007" timestamp="2022-09-20T19:17:53.684237" hostname="787f55c467b8" /></testsuites>
\ No newline at end of file
diff --git a/tests/test-pipeline.xml b/tests/test-pipeline.xml
new file mode 100644
index 0000000..d1af1d1
--- /dev/null
+++ b/tests/test-pipeline.xml
@@ -0,0 +1 @@
+<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="0" time="0.007" timestamp="2022-09-20T19:17:53.890479" hostname="787f55c467b8" /></testsuites>
\ No newline at end of file
diff --git a/tests/test-serialization.xml b/tests/test-serialization.xml
new file mode 100644
index 0000000..0e957e3
--- /dev/null
+++ b/tests/test-serialization.xml
@@ -0,0 +1 @@
+<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="0" time="0.007" timestamp="2022-09-20T19:17:54.099124" hostname="787f55c467b8" /></testsuites>
\ No newline at end of file
--
GitLab
From 91c614be509390d2e6c7eb5f0344cc6aea921007 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 12:59:19 +0000
Subject: [PATCH 061/399] DOC: code and tests
---
tests/test-core.xml | 1 -
tests/test-numpy.xml | 1 -
tests/test-pipeline.xml | 1 -
tests/test-serialization.xml | 1 -
4 files changed, 4 deletions(-)
delete mode 100644 tests/test-core.xml
delete mode 100644 tests/test-numpy.xml
delete mode 100644 tests/test-pipeline.xml
delete mode 100644 tests/test-serialization.xml
diff --git a/tests/test-core.xml b/tests/test-core.xml
deleted file mode 100644
index 718a6f5..0000000
--- a/tests/test-core.xml
+++ /dev/null
@@ -1 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="0" time="0.007" timestamp="2022-09-20T19:17:53.478514" hostname="787f55c467b8" /></testsuites>
\ No newline at end of file
diff --git a/tests/test-numpy.xml b/tests/test-numpy.xml
deleted file mode 100644
index f3cd59b..0000000
--- a/tests/test-numpy.xml
+++ /dev/null
@@ -1 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="0" time="0.007" timestamp="2022-09-20T19:17:53.684237" hostname="787f55c467b8" /></testsuites>
\ No newline at end of file
diff --git a/tests/test-pipeline.xml b/tests/test-pipeline.xml
deleted file mode 100644
index d1af1d1..0000000
--- a/tests/test-pipeline.xml
+++ /dev/null
@@ -1 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="0" time="0.007" timestamp="2022-09-20T19:17:53.890479" hostname="787f55c467b8" /></testsuites>
\ No newline at end of file
diff --git a/tests/test-serialization.xml b/tests/test-serialization.xml
deleted file mode 100644
index 0e957e3..0000000
--- a/tests/test-serialization.xml
+++ /dev/null
@@ -1 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="0" time="0.007" timestamp="2022-09-20T19:17:54.099124" hostname="787f55c467b8" /></testsuites>
\ No newline at end of file
--
GitLab
From f38b86ad4df854d25483f4bec53b3bf0612fc55b Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 14:00:48 +0000
Subject: [PATCH 062/399] DOC: code and tests
---
pyotb/core.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 0f04689..35885e9 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -14,13 +14,12 @@ from .helpers import logger
class OTBObject:
- """
- Base class for all pyotb objects
- """
+ """Base class for all pyotb objects."""
def __init__(self, name: str, app: otb.Application, image_dic: dict = None):
"""Constructor for an OTBObject.
+ Args:
name: name of the object (e.g. "Slicer")
app: OTB application instance
image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as
@@ -28,7 +27,6 @@ class OTBObject:
See this related issue for why it is necessary to keep reference of object:
https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824
-
"""
self.name = name
self.app = app
--
GitLab
From ec8d1b38e09c6d8d44b239451847edbff2113eac Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 15:59:25 +0100
Subject: [PATCH 063/399] DOC: code and tests
---
pyotb/core.py | 18 +-----------------
pyotb/functions.py | 8 ++++----
2 files changed, 5 insertions(+), 21 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 35885e9..f97c1a8 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -178,22 +178,6 @@ class OTBObject:
return list(range(0, nb_channels, step))
raise ValueError(f"{self.name}: '{bands}' cannot be interpreted as valid slicing.")
- def summarize(self) -> dict:
- """Serialize an object and its pipeline into a dictionary.
-
- Returns:
- nested dictionary summarizing the pipeline
-
- """
- params = self.parameters
- for k, p in params.items():
- # In the following, we replace each parameter which is an OTBObject, with its summary.
- if isinstance(p, OTBObject): # single parameter
- params[k] = p.summarize()
- elif isinstance(p, list): # parameter list
- params[k] = [pi.summarize() if isinstance(pi, OTBObject) else pi for pi in p]
- return {"name": self.app.GetName(), "parameters": params}
-
def export(self, key: str = None, preserve_dtype: bool = True) -> dict[str, dict[str, np.ndarray]]:
"""Export a specific output image as numpy array and store it in object exports_dic.
@@ -1330,7 +1314,7 @@ class Output(OTBObject):
def __str__(self) -> str:
"""Return a nice string representation with source app name and object id."""
- return f"<pyotb.Output {self.source_app.name} object, id {id(self)}>"
+ return f"<pyotb.Output {self.name} object, id {id(self)}>"
def get_nbchannels(inp: str | OTBObject) -> int:
diff --git a/pyotb/functions.py b/pyotb/functions.py
index 71c298b..763dcb7 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -355,9 +355,9 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
metadatas = {}
for inp in inputs:
if isinstance(inp, str): # this is for filepaths
- metadata = Input(inp).GetImageMetaData('out')
+ metadata = Input(inp).app.GetImageMetaData('out')
elif isinstance(inp, OTBObject):
- metadata = inp.GetImageMetaData(inp.output_param)
+ metadata = inp.app.GetImageMetaData(inp.output_param)
else:
raise TypeError(f"Wrong input : {inp}")
metadatas[inp] = metadata
@@ -429,7 +429,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
inputs = new_inputs
# Update metadatas
- metadatas = {input: input.GetImageMetaData('out') for input in inputs}
+ metadatas = {input: input.app.GetImageMetaData('out') for input in inputs}
# Get a metadata of an arbitrary image. This is just to compare later with other images
any_metadata = next(iter(metadatas.values()))
@@ -464,7 +464,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
inputs = new_inputs
# Update metadatas
- metadatas = {inp: inp.GetImageMetaData('out') for inp in inputs}
+ metadatas = {inp: inp.app.GetImageMetaData('out') for inp in inputs}
# Final superimposition to be sure to have the exact same image sizes
# Getting the sizes of images
--
GitLab
From 919d0761151ac7a26148e5df98198c617e962e89 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 16:04:37 +0100
Subject: [PATCH 064/399] DOC: code and tests
---
pyotb/core.py | 2 +-
pyotb/functions.py | 14 ++++++--------
2 files changed, 7 insertions(+), 9 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index f97c1a8..7abac39 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1333,7 +1333,7 @@ def get_nbchannels(inp: str | OTBObject) -> int:
# Executing the app, without printing its log
try:
info = App("ReadImageInfo", inp, quiet=True)
- nb_channels = info.GetParameterInt("numberbands")
+ nb_channels = info.app.GetParameterInt("numberbands")
except Exception as e: # this happens when we pass a str that is not a filepath
raise TypeError(f'Could not get the number of channels of `{inp}`. Not a filepath or wrong filepath') from e
return nb_channels
diff --git a/pyotb/functions.py b/pyotb/functions.py
index 763dcb7..d25610a 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -9,12 +9,11 @@ import textwrap
import subprocess
from collections import Counter
-from .core import OTBObject, Input, Operation, LogicalOperation, get_nbchannels, Output
+from .core import OTBObject, App, Operation, LogicalOperation, Input, get_nbchannels
from .helpers import logger
-def where(cond: OTBObject | Output | str, x: OTBObject | Output | str | int | float,
- y: OTBObject | Output | str | int | float) -> Operation:
+def where(cond: OTBObject | str, x: OTBObject | str | int | float, y: OTBObject | str | int | float) -> Operation:
"""Functionally similar to numpy.where. Where cond is True (!=0), returns x. Else returns y.
Args:
@@ -64,8 +63,7 @@ def where(cond: OTBObject | Output | str, x: OTBObject | Output | str | int | fl
return operation
-def clip(a: OTBObject | Output | str, a_min: OTBObject | Output | str | int | float,
- a_max: OTBObject | Output | str | int | float):
+def clip(a: OTBObject | str, a_min: OTBObject | str | int | float, a_max: OTBObject | str | int | float):
"""Clip values of image in a range of values.
Args:
@@ -416,7 +414,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
'mode.extent.ulx': ulx, 'mode.extent.uly': lry, # bug in OTB <= 7.3 :
'mode.extent.lrx': lrx, 'mode.extent.lry': uly, # ULY/LRY are inverted
}
- new_input = OTBObject('ExtractROI', params)
+ new_input = App('ExtractROI', params)
# TODO: OTB 7.4 fixes this bug, how to handle different versions of OTB?
new_inputs.append(new_input)
# Potentially update the reference inputs for later resampling
@@ -457,7 +455,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
new_inputs = []
for inp in inputs:
if metadatas[inp]['GeoTransform'][1] != pixel_size:
- superimposed = OTBObject('Superimpose', inr=reference_input, inm=inp, interpolator=interpolator)
+ superimposed = App('Superimpose', inr=reference_input, inm=inp, interpolator=interpolator)
new_inputs.append(superimposed)
else:
new_inputs.append(inp)
@@ -482,7 +480,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
new_inputs = []
for inp in inputs:
if image_sizes[inp] != most_common_image_size:
- superimposed = OTBObject('Superimpose', inr=same_size_images[0], inm=inp, interpolator=interpolator)
+ superimposed = App('Superimpose', inr=same_size_images[0], inm=inp, interpolator=interpolator)
new_inputs.append(superimposed)
else:
new_inputs.append(inp)
--
GitLab
From 39193a72c77c445220f450603c7f581e34bc8005 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 16:07:59 +0100
Subject: [PATCH 065/399] DOC: code and tests
---
pyotb/core.py | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/pyotb/core.py b/pyotb/core.py
index 7abac39..dc0296b 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1183,6 +1183,23 @@ class Operation(App):
fake_exp = f"{x}b{band}"
return fake_exp, inputs, nb_channels
+ def summarize(self) -> dict:
+ """Serialize an object and its pipeline into a dictionary.
+
+ Returns:
+ nested dictionary summarizing the pipeline
+
+ """
+ params = self.parameters
+ for k, p in params.items():
+ # In the following, we replace each parameter which is an OTBObject, with its summary.
+ if isinstance(p, App): # single parameter
+ params[k] = p.summarize()
+ elif isinstance(p, list): # parameter list
+ params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p]
+ return {"name": self.name, "parameters": params}
+
+
def __str__(self) -> str:
"""Return a nice string representation with operator and object id."""
return f"<pyotb.Operation `{self.operator}` object, id {id(self)}>"
--
GitLab
From 3078e0e0eaff1c40808595e5af9ab59fc551a802 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 16:10:25 +0100
Subject: [PATCH 066/399] DOC: code and tests
---
pyotb/core.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index dc0296b..3e96a41 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1199,7 +1199,6 @@ class Operation(App):
params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p]
return {"name": self.name, "parameters": params}
-
def __str__(self) -> str:
"""Return a nice string representation with operator and object id."""
return f"<pyotb.Operation `{self.operator}` object, id {id(self)}>"
--
GitLab
From 6eb1c98276357bbd25f3cc60bb8d2e649298605e Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 16:12:13 +0100
Subject: [PATCH 067/399] DOC: code and tests
---
tests/test_todel.py | 6 ------
1 file changed, 6 deletions(-)
delete mode 100644 tests/test_todel.py
diff --git a/tests/test_todel.py b/tests/test_todel.py
deleted file mode 100644
index fa70ec7..0000000
--- a/tests/test_todel.py
+++ /dev/null
@@ -1,6 +0,0 @@
-import pyotb
-
-img_pth = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/" \
- "otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false"
-bm = pyotb.MeanShiftSmoothing({"in": img_pth, "fout": "/tmp/toto.tif", "foutpos": "/tmp/titi.tif"})
-print(bm.find_outputs())
\ No newline at end of file
--
GitLab
From 495104255dcd521f03297abf59b60f8fbb8b833e Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 16:25:40 +0100
Subject: [PATCH 068/399] REFAC: move summarize() to App
---
pyotb/core.py | 32 ++++++++++++++++----------------
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 3e96a41..9850264 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -901,6 +901,22 @@ class App(OTBObject):
else:
self.app.SetParameterValue(key, obj)
+ def summarize(self) -> dict:
+ """Serialize an object and its pipeline into a dictionary.
+
+ Returns:
+ nested dictionary summarizing the pipeline
+
+ """
+ params = self.parameters
+ for k, p in params.items():
+ # In the following, we replace each parameter which is an OTBObject, with its summary.
+ if isinstance(p, App): # single parameter
+ params[k] = p.summarize()
+ elif isinstance(p, list): # parameter list
+ params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p]
+ return {"name": self.name, "parameters": params}
+
def __str__(self):
"""Return a nice string representation with object id."""
return f"<pyotb.App {self.name} object id {id(self)}>"
@@ -1183,22 +1199,6 @@ class Operation(App):
fake_exp = f"{x}b{band}"
return fake_exp, inputs, nb_channels
- def summarize(self) -> dict:
- """Serialize an object and its pipeline into a dictionary.
-
- Returns:
- nested dictionary summarizing the pipeline
-
- """
- params = self.parameters
- for k, p in params.items():
- # In the following, we replace each parameter which is an OTBObject, with its summary.
- if isinstance(p, App): # single parameter
- params[k] = p.summarize()
- elif isinstance(p, list): # parameter list
- params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p]
- return {"name": self.name, "parameters": params}
-
def __str__(self) -> str:
"""Return a nice string representation with operator and object id."""
return f"<pyotb.Operation `{self.operator}` object, id {id(self)}>"
--
GitLab
From c01d2d942331ce3733161584fbcb1d83960fe290 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 25 Jan 2023 16:33:50 +0100
Subject: [PATCH 069/399] FIX: summarize() with self.app.GetName()
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 9850264..fb07a7b 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -915,7 +915,7 @@ class App(OTBObject):
params[k] = p.summarize()
elif isinstance(p, list): # parameter list
params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p]
- return {"name": self.name, "parameters": params}
+ return {"name": self.app.GetName(), "parameters": params}
def __str__(self):
"""Return a nice string representation with object id."""
--
GitLab
From e778895bcc8c1333bbf81eccf318906b996b944b Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 25 Jan 2023 23:34:41 +0100
Subject: [PATCH 070/399] ENH: remove OTBObject
---
pyotb/__init__.py | 1 +
pyotb/core.py | 432 ++++++++++++++++++++---------------------
pyotb/functions.py | 10 +-
tests/test_core.py | 10 +-
tests/test_numpy.py | 6 +-
tests/test_pipeline.py | 5 +-
6 files changed, 225 insertions(+), 239 deletions(-)
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index 6164870..e3bc23a 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -9,6 +9,7 @@ otb = find_otb()
from .apps import *
from .core import (
+ App,
Input,
Output,
get_nbchannels,
diff --git a/pyotb/core.py b/pyotb/core.py
index fb07a7b..7928356 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -6,6 +6,7 @@ from ast import literal_eval
from pathlib import Path
from time import perf_counter
from typing import Any
+from abc import ABC, abstractmethod
import numpy as np
import otbApplication as otb # pylint: disable=import-error
@@ -13,66 +14,23 @@ import otbApplication as otb # pylint: disable=import-error
from .helpers import logger
-class OTBObject:
- """Base class for all pyotb objects."""
-
- def __init__(self, name: str, app: otb.Application, image_dic: dict = None):
- """Constructor for an OTBObject.
-
- Args:
- name: name of the object (e.g. "Slicer")
- app: OTB application instance
- image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as
- the result of self.app.ExportImage(). Use it when the app takes a numpy array as input.
- See this related issue for why it is necessary to keep reference of object:
- https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824
-
- """
- self.name = name
- self.app = app
- self.image_dic = image_dic
-
- self.parameters_keys = tuple(self.app.GetParametersKeys())
- self.all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys}
- self.out_param_types = {k: v for k, v in self.all_param_types.items()
- if v in (otb.ParameterType_OutputImage,
- otb.ParameterType_OutputVectorData,
- otb.ParameterType_OutputFilename)}
-
- self.exports_dic = {}
-
- def get_first_key(self, param_types: list[str]) -> str:
- """Get the first output param key for specific file types."""
- for key, param_type in sorted(self.all_param_types.items()):
- if param_type in param_types:
- return key
- return None
-
- @property
- def key_input(self) -> str:
- """Get the name of first input parameter, raster > vector > file."""
- return self.get_first_key(param_types=[otb.ParameterType_InputImage,
- otb.ParameterType_InputImageList]) \
- or self.get_first_key(param_types=[otb.ParameterType_InputVectorData,
- otb.ParameterType_InputVectorDataList]) \
- or self.get_first_key(param_types=[otb.ParameterType_InputFilename,
- otb.ParameterType_InputFilenameList])
+class RasterInterface(ABC):
+ """Abstraction of an image object."""
+ app: otb.Application
+ exports_dic: dict
@property
- def key_input_image(self) -> str:
- """Get the name of first input image parameter."""
- return self.get_first_key(param_types=[otb.ParameterType_InputImage, otb.ParameterType_InputImageList])
+ @abstractmethod
+ def key_output_image(self):
+ """Returns the name of a parameter associated to an image. Property defined in App and Output."""
- @property
- def key_output_image(self) -> str:
- """Get the name of first output image parameter."""
- return self.get_first_key(param_types=[otb.ParameterType_OutputImage])
+ @abstractmethod
+ def write(self):
+ """Write image, this is defined in App. Output will use App.write for a specific key."""
@property
def metadata(self):
"""Return first output image metadata dictionary."""
- if not self.key_output_image:
- raise TypeError(f"{self.name}: this application has no raster output")
return dict(self.app.GetMetadataDictionary(self.key_output_image))
@property
@@ -83,8 +41,6 @@ class OTBObject:
dtype: pixel type of the output image
"""
- if not self.key_output_image:
- raise TypeError(f"{self.name}: this application has no raster output")
enum = self.app.GetParameterOutputImagePixelType(self.key_output_image)
return self.app.ConvertPixelTypeToNumpy(enum)
@@ -96,8 +52,6 @@ class OTBObject:
shape: (height, width, bands)
"""
- if not self.key_output_image:
- raise TypeError(f"\"{self.name}\" has no raster output")
width, height = self.app.GetImageSize(self.key_output_image)
bands = self.app.GetImageNbBands(self.key_output_image)
return height, width, bands
@@ -109,8 +63,6 @@ class OTBObject:
Returns:
transform: (X spacing, X offset, X origin, Y offset, Y spacing, Y origin)
"""
- if not self.key_output_image:
- raise TypeError(f"{self.name}: this application has no raster output")
spacing_x, spacing_y = self.app.GetImageSpacing(self.key_output_image)
origin_x, origin_y = self.app.GetImageOrigin(self.key_output_image)
# Shift image origin since OTB is giving coordinates of pixel center instead of corners
@@ -119,14 +71,10 @@ class OTBObject:
def get_infos(self):
"""Return a dict output of ReadImageInfo for the first image output."""
- if not self.key_output_image:
- raise TypeError(f"{self.name}: this application has no raster output")
return App("ReadImageInfo", self, quiet=True).data
def get_statistics(self):
"""Return a dict output of ComputeImagesStatistics for the first image output."""
- if not self.key_output_image:
- raise TypeError(f"{self.name}: this application has no raster output")
return App("ComputeImagesStatistics", self, quiet=True).data
def read_values_at_coords(self, row: int, col: int, bands: int = None) -> list[int | float] | int | float:
@@ -151,7 +99,7 @@ class OTBObject:
elif isinstance(bands, slice):
channels = self.channels_list_from_slice(bands)
elif not isinstance(bands, list):
- raise TypeError(f"{self.name}: type '{type(bands)}' cannot be interpreted as a valid slicing")
+ raise TypeError(f"{self.app.GetName()}: type '{type(bands)}' cannot be interpreted as a valid slicing")
if channels:
app.app.Execute()
app.set_parameters({"cl": [f"Channel{n + 1}" for n in channels]})
@@ -176,7 +124,7 @@ class OTBObject:
return list(range(0, stop, step))
if start is None and stop is None:
return list(range(0, nb_channels, step))
- raise ValueError(f"{self.name}: '{bands}' cannot be interpreted as valid slicing.")
+ raise ValueError(f"{self.app.GetName()}: '{bands}' cannot be interpreted as valid slicing.")
def export(self, key: str = None, preserve_dtype: bool = True) -> dict[str, dict[str, np.ndarray]]:
"""Export a specific output image as numpy array and store it in object exports_dic.
@@ -184,7 +132,7 @@ class OTBObject:
Args:
key: parameter key to export, if None then the default one will be used
preserve_dtype: when set to True, the numpy array is converted to the same pixel type as
- the OTBObject first output. Default is True
+ the App first output. Default is True
Returns:
the exported numpy array
@@ -204,7 +152,7 @@ class OTBObject:
Args:
key: the output parameter name to export as numpy array
preserve_dtype: when set to True, the numpy array is converted to the same pixel type as
- the OTBObject first output. Default is True
+ the App first output. Default is True
copy: whether to copy the output array, default is False
required to True if preserve_dtype is False and the source app reference is lost
@@ -250,55 +198,7 @@ class OTBObject:
row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x
return abs(int(row)), int(col)
- # Special functions
- def __hash__(self):
- """Override the default behaviour of the hash function.
-
- Returns:
- self hash
-
- """
- return id(self)
-
- def __getitem__(self, key):
- """Override the default __getitem__ behaviour.
-
- This function enables 2 things :
- - access attributes like that : object['any_attribute']
- - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3]
- selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]]
- selecting 1000x1000 subset : object[:1000, :1000]
- - access pixel value(s) at a specified row, col index
-
- Args:
- key: attribute key
-
- Returns:
- attribute, pixel values or Slicer
-
- """
- # Accessing string attributes
- if isinstance(key, str):
- return self.__dict__.get(key)
- # Accessing pixel value(s) using Y/X coordinates
- if isinstance(key, tuple) and len(key) >= 2:
- row, col = key[0], key[1]
- if isinstance(row, int) and isinstance(col, int):
- if row < 0 or col < 0:
- raise ValueError(f"{self.name}: can't read pixel value at negative coordinates ({row}, {col})")
- channels = None
- if len(key) == 3:
- channels = key[2]
- return self.read_values_at_coords(row, col, channels)
- # Slicing
- if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)):
- raise ValueError(f'"{key}"cannot be interpreted as valid slicing. Slicing should be 2D or 3D.')
- if isinstance(key, tuple) and len(key) == 2:
- # Adding a 3rd dimension
- key = key + (slice(None, None, None),)
- return Slicer(self, *key)
-
- def __add__(self, other: OTBObject | str | int | float) -> Operation:
+ def __add__(self, other: App | str | int | float) -> Operation:
"""Overrides the default addition and flavours it with BandMathX.
Args:
@@ -312,7 +212,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("+", self, other)
- def __sub__(self, other: OTBObject | str | int | float) -> Operation:
+ def __sub__(self, other: App | str | int | float) -> Operation:
"""Overrides the default subtraction and flavours it with BandMathX.
Args:
@@ -326,7 +226,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("-", self, other)
- def __mul__(self, other: OTBObject | str | int | float) -> Operation:
+ def __mul__(self, other: App | str | int | float) -> Operation:
"""Overrides the default subtraction and flavours it with BandMathX.
Args:
@@ -340,7 +240,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("*", self, other)
- def __truediv__(self, other: OTBObject | str | int | float) -> Operation:
+ def __truediv__(self, other: App | str | int | float) -> Operation:
"""Overrides the default subtraction and flavours it with BandMathX.
Args:
@@ -354,7 +254,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("/", self, other)
- def __radd__(self, other: OTBObject | str | int | float) -> Operation:
+ def __radd__(self, other: App | str | int | float) -> Operation:
"""Overrides the default reverse addition and flavours it with BandMathX.
Args:
@@ -368,7 +268,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("+", other, self)
- def __rsub__(self, other: OTBObject | str | int | float) -> Operation:
+ def __rsub__(self, other: App | str | int | float) -> Operation:
"""Overrides the default subtraction and flavours it with BandMathX.
Args:
@@ -382,7 +282,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("-", other, self)
- def __rmul__(self, other: OTBObject | str | int | float) -> Operation:
+ def __rmul__(self, other: App | str | int | float) -> Operation:
"""Overrides the default multiplication and flavours it with BandMathX.
Args:
@@ -396,7 +296,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("*", other, self)
- def __rtruediv__(self, other: OTBObject | str | int | float) -> Operation:
+ def __rtruediv__(self, other: App | str | int | float) -> Operation:
"""Overrides the default division and flavours it with BandMathX.
Args:
@@ -419,7 +319,7 @@ class OTBObject:
"""
return Operation("abs", self)
- def __ge__(self, other: OTBObject | str | int | float) -> Operation:
+ def __ge__(self, other: App | str | int | float) -> Operation:
"""Overrides the default greater or equal and flavours it with BandMathX.
Args:
@@ -433,7 +333,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation(">=", self, other)
- def __le__(self, other: OTBObject | str | int | float) -> Operation:
+ def __le__(self, other: App | str | int | float) -> Operation:
"""Overrides the default less or equal and flavours it with BandMathX.
Args:
@@ -447,7 +347,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation("<=", self, other)
- def __gt__(self, other: OTBObject | str | int | float) -> Operation:
+ def __gt__(self, other: App | str | int | float) -> Operation:
"""Overrides the default greater operator and flavours it with BandMathX.
Args:
@@ -461,7 +361,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation(">", self, other)
- def __lt__(self, other: OTBObject | str | int | float) -> Operation:
+ def __lt__(self, other: App | str | int | float) -> Operation:
"""Overrides the default less operator and flavours it with BandMathX.
Args:
@@ -475,7 +375,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation("<", self, other)
- def __eq__(self, other: OTBObject | str | int | float) -> Operation:
+ def __eq__(self, other: App | str | int | float) -> Operation:
"""Overrides the default eq operator and flavours it with BandMathX.
Args:
@@ -489,7 +389,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation("==", self, other)
- def __ne__(self, other: OTBObject | str | int | float) -> Operation:
+ def __ne__(self, other: App | str | int | float) -> Operation:
"""Overrides the default different operator and flavours it with BandMathX.
Args:
@@ -503,7 +403,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation("!=", self, other)
- def __or__(self, other: OTBObject | str | int | float) -> Operation:
+ def __or__(self, other: App | str | int | float) -> Operation:
"""Overrides the default or operator and flavours it with BandMathX.
Args:
@@ -517,7 +417,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation("||", self, other)
- def __and__(self, other: OTBObject | str | int | float) -> Operation:
+ def __and__(self, other: App | str | int | float) -> Operation:
"""Overrides the default and operator and flavours it with BandMathX.
Args:
@@ -543,7 +443,7 @@ class OTBObject:
"""
return self.to_numpy()
- def __array_ufunc__(self, ufunc, method, *inputs, **kwargs) -> OTBObject:
+ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs) -> App:
"""This is called whenever a numpy function is called on a pyotb object.
Operation is performed in numpy, then imported back to pyotb with the same georeference as input.
@@ -566,7 +466,7 @@ class OTBObject:
for inp in inputs:
if isinstance(inp, (float, int, np.ndarray, np.generic)):
arrays.append(inp)
- elif isinstance(inp, OTBObject):
+ elif isinstance(inp, App):
if not inp.exports_dic:
inp.export()
image_dic = inp.exports_dic[inp.key_output_image]
@@ -590,19 +490,18 @@ class OTBObject:
return NotImplemented
-class App(OTBObject):
+class App(RasterInterface):
"""Base class that gathers common operations for any OTB application."""
- def __init__(self, otb_app_name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None,
- name: str = None, **kwargs):
+ def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, **kwargs):
"""Common constructor for OTB applications. Handles in-memory connection between apps.
Args:
- otb_app_name: name of the OTB application, e.g. 'BandMath'
+ name: name of the app, e.g. 'BandMath'
*args: used for passing application parameters. Can be :
- dictionary containing key-arguments enumeration. Useful when a key is python-reserved
(e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
- - string or OTBObject, useful when the user wants to specify the input "in"
+ - string, App or Output, useful when the user wants to specify the input "in"
- list, useful when the user wants to specify the input list 'il'
frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___
quiet: whether to print logs of the OTB app
@@ -610,19 +509,26 @@ class App(OTBObject):
the result of app.ExportImage(). Use it when the app takes a numpy array as input.
See this related issue for why it is necessary to keep reference of object:
https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824
- name: override the application name
+
**kwargs: used for passing application parameters.
e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif'
"""
+ self.parameters = {}
+ self.name = name
self.frozen = frozen
self.quiet = quiet
+ self.image_dic = image_dic
+ self.exports_dic = {}
create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
- super().__init__(name=name or f"OTB Application {otb_app_name}", app=create(otb_app_name), image_dic=image_dic)
- self.description = self.app.GetDocLongDescription()
-
- # Set parameters
- self.parameters = {}
+ self.app = create(name)
+ self.parameters_keys = tuple(self.app.GetParametersKeys())
+ self.all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys}
+ self.out_param_types = {k: v for k, v in self.all_param_types.items()
+ if v in (otb.ParameterType_OutputImage,
+ otb.ParameterType_OutputVectorData,
+ otb.ParameterType_OutputFilename)}
+ self.time_start, self.time_end = 0, 0
if args or kwargs:
self.set_parameters(*args, **kwargs)
if not self.frozen:
@@ -630,8 +536,32 @@ class App(OTBObject):
if any(key in self.parameters for key in self.out_param_types):
self.flush() # auto flush if any output param was provided during app init
- # Elapsed time
- self.time_start, self.time_end = 0, 0
+ def get_first_key(self, param_types: list[str]) -> str:
+ """Get the first output param key for specific file types."""
+ for key, param_type in sorted(self.all_param_types.items()):
+ if param_type in param_types:
+ return key
+ return None
+
+ @property
+ def key_input(self) -> str:
+ """Get the name of first input parameter, raster > vector > file."""
+ return self.get_first_key(param_types=[otb.ParameterType_InputImage,
+ otb.ParameterType_InputImageList]) \
+ or self.get_first_key(param_types=[otb.ParameterType_InputVectorData,
+ otb.ParameterType_InputVectorDataList]) \
+ or self.get_first_key(param_types=[otb.ParameterType_InputFilename,
+ otb.ParameterType_InputFilenameList])
+
+ @property
+ def key_input_image(self) -> str:
+ """Get the name of first input image parameter."""
+ return self.get_first_key(param_types=[otb.ParameterType_InputImage, otb.ParameterType_InputImageList])
+
+ @property
+ def key_output_image(self) -> str:
+ """Get the name of first output image parameter."""
+ return self.get_first_key(param_types=[otb.ParameterType_OutputImage])
@property
def elapsed_time(self):
@@ -679,7 +609,7 @@ class App(OTBObject):
Args:
*args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
(e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
- - string or OTBObject, useful when the user implicitly wants to set the param "in"
+ - string or App, useful when the user implicitly wants to set the param "in"
- list, useful when the user implicitly wants to set the param "il"
**kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif'
@@ -712,6 +642,36 @@ class App(OTBObject):
self.parameters.update({**parameters, **otb_params})
self.save_objects()
+ def propagate_dtype(self, target_key: str = None, dtype: int = None):
+ """Propagate a pixel type from main input to every outputs, or to a target output key only.
+
+ With multiple inputs (if dtype is not provided), the type of the first input is considered.
+ With multiple outputs (if target_key is not provided), all outputs will be converted to the same pixel type.
+
+ Args:
+ target_key: output param key to change pixel type
+ dtype: data type to use
+
+ """
+ if not dtype:
+ param = self.parameters.get(self.key_input_image)
+ if not param:
+ logger.warning("%s: could not propagate pixel type from inputs to output", self.name)
+ return
+ if isinstance(param, (list, tuple)):
+ param = param[0] # first image in "il"
+ try:
+ dtype = get_pixel_type(param)
+ except (TypeError, RuntimeError):
+ logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param)
+ return
+ if target_key:
+ keys = [target_key]
+ else:
+ keys = [k for k, v in self.out_param_types.items() if v == otb.ParameterType_OutputImage]
+ for key in keys:
+ self.app.SetParameterOutputImagePixelType(key, dtype)
+
def save_objects(self):
"""Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`.
@@ -819,38 +779,24 @@ class App(OTBObject):
self.set_parameters({key: output_filename})
self.flush()
- def propagate_dtype(self, target_key: str = None, dtype: int = None):
- """Propagate a pixel type from main input to every outputs, or to a target output key only.
-
- With multiple inputs (if dtype is not provided), the type of the first input is considered.
- With multiple outputs (if target_key is not provided), all outputs will be converted to the same pixel type.
+ def summarize(self) -> dict:
+ """Serialize an object and its pipeline into a dictionary.
- Args:
- target_key: output param key to change pixel type
- dtype: data type to use
+ Returns:
+ nested dictionary summarizing the pipeline
"""
- if not dtype:
- param = self.parameters.get(self.key_input_image)
- if not param:
- logger.warning("%s: could not propagate pixel type from inputs to output", self.name)
- return
- if isinstance(param, (list, tuple)):
- param = param[0] # first image in "il"
- try:
- dtype = get_pixel_type(param)
- except (TypeError, RuntimeError):
- logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param)
- return
- if target_key:
- keys = [target_key]
- else:
- keys = [k for k, v in self.out_param_types.items() if v == otb.ParameterType_OutputImage]
- for key in keys:
- self.app.SetParameterOutputImagePixelType(key, dtype)
+ params = self.parameters
+ for k, p in params.items():
+ # In the following, we replace each parameter which is an App, with its summary.
+ if isinstance(p, App): # single parameter
+ params[k] = p.summarize()
+ elif isinstance(p, list): # parameter list
+ params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p]
+ return {"name": self.app.GetName(), "parameters": params}
# Private functions
- def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]:
+ def __parse_args(self, args: list[str | App | dict | list]) -> dict[str, Any]:
"""Gather all input arguments in kwargs dict.
Args:
@@ -864,11 +810,11 @@ class App(OTBObject):
for arg in args:
if isinstance(arg, dict):
kwargs.update(arg)
- elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.key_input):
+ elif isinstance(arg, (str, App)) or isinstance(arg, list) and is_key_list(self, self.key_input):
kwargs.update({self.key_input: arg})
return kwargs
- def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]):
+ def __set_param(self, key: str, obj: list | tuple | App | otb.Application | list[Any]):
"""Set one parameter, decide which otb.Application method to use depending on target object."""
if obj is None or (isinstance(obj, (list, tuple)) and not obj):
self.app.ClearValue(key)
@@ -878,7 +824,7 @@ class App(OTBObject):
f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}"
)
# Single-parameter cases
- if isinstance(obj, OTBObject):
+ if isinstance(obj, App):
self.app.ConnectImage(key, obj.app, obj.key_output_image)
elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB
self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0])
@@ -890,7 +836,7 @@ class App(OTBObject):
elif is_key_images_list(self, key):
# To enable possible in-memory connections, we go through the list and set the parameters one by one
for inp in obj:
- if isinstance(inp, OTBObject):
+ if isinstance(inp, App):
self.app.ConnectImage(key, inp.app, inp.key_output_image)
elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB
self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0])
@@ -901,21 +847,53 @@ class App(OTBObject):
else:
self.app.SetParameterValue(key, obj)
- def summarize(self) -> dict:
- """Serialize an object and its pipeline into a dictionary.
+ # Special functions
+ def __hash__(self):
+ """Override the default behaviour of the hash function.
Returns:
- nested dictionary summarizing the pipeline
+ self hash
"""
- params = self.parameters
- for k, p in params.items():
- # In the following, we replace each parameter which is an OTBObject, with its summary.
- if isinstance(p, App): # single parameter
- params[k] = p.summarize()
- elif isinstance(p, list): # parameter list
- params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p]
- return {"name": self.app.GetName(), "parameters": params}
+ return id(self)
+
+ def __getitem__(self, key):
+ """Override the default __getitem__ behaviour.
+
+ This function enables 2 things :
+ - access attributes like that : object['any_attribute']
+ - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3]
+ selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]]
+ selecting 1000x1000 subset : object[:1000, :1000]
+ - access pixel value(s) at a specified row, col index
+
+ Args:
+ key: attribute key
+
+ Returns:
+ attribute, pixel values or Slicer
+
+ """
+ # Accessing string attributes
+ if isinstance(key, str):
+ return self.__dict__.get(key)
+ # Accessing pixel value(s) using Y/X coordinates
+ if isinstance(key, tuple) and len(key) >= 2:
+ row, col = key[0], key[1]
+ if isinstance(row, int) and isinstance(col, int):
+ if row < 0 or col < 0:
+ raise ValueError(f"{self.name}: can't read pixel value at negative coordinates ({row}, {col})")
+ channels = None
+ if len(key) == 3:
+ channels = key[2]
+ return self.read_values_at_coords(row, col, channels)
+ # Slicing
+ if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)):
+ raise ValueError(f'"{key}"cannot be interpreted as valid slicing. Slicing should be 2D or 3D.')
+ if isinstance(key, tuple) and len(key) == 2:
+ # Adding a 3rd dimension
+ key = key + (slice(None, None, None),)
+ return Slicer(self, *key)
def __str__(self):
"""Return a nice string representation with object id."""
@@ -925,7 +903,7 @@ class App(OTBObject):
class Slicer(App):
"""Slicer objects i.e. when we call something like raster[:, :, 2] from Python."""
- def __init__(self, obj: OTBObject | str, rows: int, cols: int, channels: int):
+ def __init__(self, obj: App | str, rows: int, cols: int, channels: int):
"""Create a slicer object, that can be used directly for writing or inside a BandMath.
It contains :
@@ -939,7 +917,8 @@ class Slicer(App):
channels: channels, can be slicing, list or int
"""
- super().__init__("ExtractROI", {"in": obj, "mode": "extent"}, quiet=True, frozen=True, name="Slicer")
+ self.name = "Slicer"
+ super().__init__("ExtractROI", obj, mode="extent", quiet=True, frozen=True)
self.rows, self.cols = rows, cols
parameters = {}
@@ -1009,7 +988,7 @@ class Operation(App):
"""
- def __init__(self, operator: str, *inputs, nb_bands: int = None, name: str = None):
+ def __init__(self, operator: str, *inputs, nb_bands: int = None):
"""Given some inputs and an operator, this function enables to transform this into an OTB application.
Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator.
@@ -1017,7 +996,7 @@ class Operation(App):
Args:
operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
- *inputs: inputs. Can be OTBObject, filepath, int or float
+ *inputs: inputs. Can be App, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
name: override the Operation name
@@ -1045,11 +1024,12 @@ class Operation(App):
# Getting unique image inputs, in the order im1, im2, im3 ...
self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)]
self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands)
+ self.name = f'Operation exp="{self.exp}"'
+ appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX"
# Execute app
- super().__init__("BandMath" if len(self.exp_bands) == 1 else "BandMathX", il=self.unique_inputs,
- exp=self.exp, quiet=True, name=name or f'Operation exp="{self.exp}"')
+ super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True)
- def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float],
+ def create_fake_exp(self, operator: str, inputs: list[App | str | int | float],
nb_bands: int = None):
"""Create a 'fake' expression.
@@ -1057,7 +1037,7 @@ class Operation(App):
Args:
operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
- inputs: inputs. Can be OTBObject, filepath, int or float
+ inputs: inputs. Can be App, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
@@ -1092,14 +1072,10 @@ class Operation(App):
cond_band = 1
else:
cond_band = band
- fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp(
- inp, cond_band, keep_logical=True
- )
+ fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp(inp, cond_band, keep_logical=True)
else:
# Any other input
- fake_exp, corresponding_inputs, nb_channels = self.create_one_input_fake_exp(
- inp, band, keep_logical=False
- )
+ fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp(inp, band, keep_logical=False)
fake_exps.append(fake_exp)
# Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates)
if i == 0 and corresponding_inputs and nb_channels:
@@ -1141,8 +1117,7 @@ class Operation(App):
return exp_bands, exp
@staticmethod
- def create_one_input_fake_exp(x: OTBObject | str,
- band: int, keep_logical: bool = False) -> tuple(str, list[OTBObject], int):
+ def make_fake_exp(x: App | str, band: int, keep_logical: bool = False) -> tuple(str, list[App], int):
"""This an internal function, only to be used by `create_fake_exp`.
Enable to create a fake expression just for one input and one band.
@@ -1211,6 +1186,7 @@ class LogicalOperation(Operation):
logical expression (e.g. "im1b1 > 0")
"""
+ name = "LogicalOperation"
def __init__(self, operator: str, *inputs, nb_bands: int = None):
"""Constructor for a LogicalOperation object.
@@ -1221,10 +1197,10 @@ class LogicalOperation(Operation):
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
- super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation")
+ super().__init__(operator, *inputs, nb_bands=nb_bands)
self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands)
- def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None):
+ def create_fake_exp(self, operator: str, inputs: list[App | str | int | float], nb_bands: int = None):
"""Create a 'fake' expression.
e.g for the operation input1 > input2, we create a fake expression that is like
@@ -1232,7 +1208,7 @@ class LogicalOperation(Operation):
Args:
operator: str (one of >, <, >=, <=, ==, !=, &, |)
- inputs: Can be OTBObject, filepath, int or float
+ inputs: Can be App, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
@@ -1251,7 +1227,7 @@ class LogicalOperation(Operation):
for i, band in enumerate(range(1, nb_bands + 1)):
fake_exps = []
for inp in inputs:
- fake_exp, corresp_inputs, nb_channels = super().create_one_input_fake_exp(inp, band, keep_logical=True)
+ fake_exp, corresp_inputs, nb_channels = super().make_fake_exp(inp, band, keep_logical=True)
fake_exps.append(fake_exp)
# Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates)
if i == 0 and corresp_inputs and nb_channels:
@@ -1278,7 +1254,8 @@ class Input(App):
"""
self.path = path
- super().__init__("ExtractROI", {"in": path}, frozen=True, name=f"Input from {path}")
+ self.name = f"Input from {path}"
+ super().__init__("ExtractROI", {"in": path}, frozen=True)
self.propagate_dtype()
self.execute()
@@ -1287,11 +1264,10 @@ class Input(App):
return f"<pyotb.Input object from {self.path}>"
-class Output(OTBObject):
+class Output(RasterInterface):
"""Object that behave like a pointer to a specific application output file."""
- def __init__(self, pyotb_app: OTBObject, # pylint: disable=super-init-not-called
- param_key: str = None, filepath: str = None, mkdir: bool = True):
+ def __init__(self, pyotb_app: App, param_key: str = None, filepath: str = None, mkdir: bool = True):
"""Constructor for an Output object.
Args:
@@ -1301,9 +1277,10 @@ class Output(OTBObject):
mkdir: create missing parent directories
"""
- super().__init__(name=f"Output {param_key} from {pyotb_app.name}", app=pyotb_app.app)
- self.parent_pyotb_app = pyotb_app # keep trace of parent app
- self.param_key = param_key or super().key_output_image
+ self.name = f"Output {param_key} from {pyotb_app.name}"
+ self.parent_app = pyotb_app # keep trace of parent app
+ self.app = pyotb_app.app
+ self.param_key = param_key
self.filepath = None
if filepath:
if '?' in filepath:
@@ -1314,7 +1291,6 @@ class Output(OTBObject):
@property
def key_output_image(self):
- """Overwrite OTBObject prop, in order to use Operation special methods with the right Output param_key."""
return self.param_key
def exists(self) -> bool:
@@ -1328,12 +1304,18 @@ class Output(OTBObject):
if not self.filepath.parent.exists():
self.filepath.parent.mkdir(parents=True)
+ def write(self, filepath: str = "", **kwargs):
+ """Write output to disk, filepath is not required if it was provided to parent App during init"""
+ if not filepath and self.filepath:
+ return self.parent_app.write({self.key_output_image: self.filepath}, **kwargs)
+ return self.parent_app.write({self.key_output_image: filepath}, **kwargs)
+
def __str__(self) -> str:
"""Return a nice string representation with source app name and object id."""
return f"<pyotb.Output {self.name} object, id {id(self)}>"
-def get_nbchannels(inp: str | OTBObject) -> int:
+def get_nbchannels(inp: str | App) -> int:
"""Get the nb of bands of input image.
Args:
@@ -1343,7 +1325,7 @@ def get_nbchannels(inp: str | OTBObject) -> int:
number of bands in image
"""
- if isinstance(inp, OTBObject):
+ if isinstance(inp, App):
nb_channels = inp.shape[-1]
else:
# Executing the app, without printing its log
@@ -1355,7 +1337,7 @@ def get_nbchannels(inp: str | OTBObject) -> int:
return nb_channels
-def get_pixel_type(inp: str | OTBObject) -> str:
+def get_pixel_type(inp: str | App) -> str:
"""Get the encoding of input image pixels.
Args:
@@ -1363,7 +1345,7 @@ def get_pixel_type(inp: str | OTBObject) -> str:
Returns:
pixel_type: OTB enum e.g. `otbApplication.ImagePixelType_uint8', which actually is an int.
- For an App with several outputs, only the pixel type of the first output is returned
+ make_fake_exp For an App with several outputs, only the pixel type of the first output is returned
"""
if isinstance(inp, str):
@@ -1388,7 +1370,7 @@ def get_pixel_type(inp: str | OTBObject) -> str:
if datatype not in datatype_to_pixeltype:
raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}")
pixel_type = getattr(otb, f'ImagePixelType_{datatype_to_pixeltype[datatype]}')
- elif isinstance(inp, OTBObject):
+ elif isinstance(inp, App):
pixel_type = inp.app.GetParameterOutputImagePixelType(inp.key_output_image)
else:
raise TypeError(f'Could not get the pixel type of {type(inp)} object {inp}')
@@ -1412,7 +1394,7 @@ def parse_pixel_type(pixel_type: str | int) -> int:
raise ValueError(f'Bad pixel type specification ({pixel_type})')
-def is_key_list(pyotb_app: OTBObject, key: str) -> bool:
+def is_key_list(pyotb_app: App, key: str) -> bool:
"""Check if a key of the App is an input parameter list."""
return pyotb_app.app.GetParameterType(key) in (
otb.ParameterType_InputImageList,
@@ -1423,7 +1405,7 @@ def is_key_list(pyotb_app: OTBObject, key: str) -> bool:
)
-def is_key_images_list(pyotb_app: OTBObject, key: str) -> bool:
+def is_key_images_list(pyotb_app: App, key: str) -> bool:
"""Check if a key of the App is an input parameter image list."""
return pyotb_app.app.GetParameterType(key) in (
otb.ParameterType_InputImageList,
@@ -1431,6 +1413,6 @@ def is_key_images_list(pyotb_app: OTBObject, key: str) -> bool:
)
-def get_out_images_param_keys(app: OTBObject) -> list[str]:
+def get_out_images_param_keys(app: App) -> list[str]:
"""Return every output parameter keys of an OTB app."""
return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage]
diff --git a/pyotb/functions.py b/pyotb/functions.py
index d25610a..250f79e 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -9,11 +9,11 @@ import textwrap
import subprocess
from collections import Counter
-from .core import OTBObject, App, Operation, LogicalOperation, Input, get_nbchannels
+from .core import App, Operation, LogicalOperation, Input, get_nbchannels
from .helpers import logger
-def where(cond: OTBObject | str, x: OTBObject | str | int | float, y: OTBObject | str | int | float) -> Operation:
+def where(cond: App | str, x: App | str | int | float, y: App | str | int | float) -> Operation:
"""Functionally similar to numpy.where. Where cond is True (!=0), returns x. Else returns y.
Args:
@@ -63,7 +63,7 @@ def where(cond: OTBObject | str, x: OTBObject | str | int | float, y: OTBObject
return operation
-def clip(a: OTBObject | str, a_min: OTBObject | str | int | float, a_max: OTBObject | str | int | float):
+def clip(a: App | str, a_min: App | str | int | float, a_max: App | str | int | float):
"""Clip values of image in a range of values.
Args:
@@ -324,7 +324,7 @@ def run_tf_function(func):
def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_rule: str = 'minimal',
interpolator: str = 'nn', reference_window_input: dict = None,
- reference_pixel_size_input: str = None) -> list[OTBObject]:
+ reference_pixel_size_input: str = None) -> list[App]:
"""Given several inputs, this function handles the potential resampling and cropping to same extent.
WARNING: Not fully implemented / tested
@@ -354,7 +354,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
for inp in inputs:
if isinstance(inp, str): # this is for filepaths
metadata = Input(inp).app.GetImageMetaData('out')
- elif isinstance(inp, OTBObject):
+ elif isinstance(inp, App):
metadata = inp.app.GetImageMetaData(inp.output_param)
else:
raise TypeError(f"Wrong input : {inp}")
diff --git a/tests/test_core.py b/tests/test_core.py
index 086c26e..0d1fc6f 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -1,3 +1,4 @@
+import os
import pytest
import pyotb
@@ -56,7 +57,7 @@ def test_nonraster_property():
def test_elapsed_time():
- assert pyotb.ReadImageInfo(INPUT).elapsed_time < 1
+ assert 0 < pyotb.ReadImageInfo(INPUT).elapsed_time < 1
# Other functions
@@ -76,6 +77,13 @@ def test_xy_to_rowcol():
def test_write():
INPUT.write("/tmp/missing_dir/test_write.tif")
assert INPUT.out.exists()
+ os.remove("/tmp/missing_dir/test_write.tif")
+
+
+def test_output_write():
+ INPUT.out.write("/tmp/missing_dir/test_write.tif")
+ assert INPUT.out.exists()
+ os.remove("/tmp/missing_dir/test_write.tif")
# Slicer
diff --git a/tests/test_numpy.py b/tests/test_numpy.py
index 0f42435..682fe2a 100644
--- a/tests/test_numpy.py
+++ b/tests/test_numpy.py
@@ -1,10 +1,6 @@
import numpy as np
import pyotb
from tests_data import INPUT
-import numpy as np
-
-import pyotb
-from tests_data import INPUT
def test_export():
@@ -43,7 +39,7 @@ def test_pixel_coords_otb_equals_numpy():
def test_add_noise_array():
white_noise = np.random.normal(0, 50, size=INPUT.shape)
noisy_image = INPUT + white_noise
- assert isinstance(noisy_image, pyotb.core.OTBObject)
+ assert isinstance(noisy_image, pyotb.core.App)
assert noisy_image.shape == INPUT.shape
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index 3acc354..76f9872 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -103,8 +103,7 @@ PIPELINES, NAMES = make_pipelines_list()
@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES)
def test_pipeline_shape(pipe):
- for i, app in enumerate(pipe):
- print(app.shape)
+ for app in pipe:
assert bool(app.shape)
@@ -116,7 +115,7 @@ def test_pipeline_shape_nointermediate(pipe):
@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES)
def test_pipeline_shape_backward(pipe):
- for i, app in enumerate(reversed(pipe)):
+ for app in reversed(pipe):
assert bool(app.shape)
--
GitLab
From ab38817c10fd43d067f7aa09557abf5947268902 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 00:02:50 +0100
Subject: [PATCH 071/399] CI: linting + test use filepath
---
pyotb/core.py | 6 +++---
tests/test_core.py | 5 +++--
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 7928356..4d3d848 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -902,6 +902,7 @@ class App(RasterInterface):
class Slicer(App):
"""Slicer objects i.e. when we call something like raster[:, :, 2] from Python."""
+ name = "Slicer"
def __init__(self, obj: App | str, rows: int, cols: int, channels: int):
"""Create a slicer object, that can be used directly for writing or inside a BandMath.
@@ -917,7 +918,6 @@ class Slicer(App):
channels: channels, can be slicing, list or int
"""
- self.name = "Slicer"
super().__init__("ExtractROI", obj, mode="extent", quiet=True, frozen=True)
self.rows, self.cols = rows, cols
parameters = {}
@@ -1186,7 +1186,6 @@ class LogicalOperation(Operation):
logical expression (e.g. "im1b1 > 0")
"""
- name = "LogicalOperation"
def __init__(self, operator: str, *inputs, nb_bands: int = None):
"""Constructor for a LogicalOperation object.
@@ -1291,6 +1290,7 @@ class Output(RasterInterface):
@property
def key_output_image(self):
+ """Force the right key to be used when accessing the RasterInterface."""
return self.param_key
def exists(self) -> bool:
@@ -1305,7 +1305,7 @@ class Output(RasterInterface):
self.filepath.parent.mkdir(parents=True)
def write(self, filepath: str = "", **kwargs):
- """Write output to disk, filepath is not required if it was provided to parent App during init"""
+ """Write output to disk, filepath is not required if it was provided to parent App during init."""
if not filepath and self.filepath:
return self.parent_app.write({self.key_output_image: self.filepath}, **kwargs)
return self.parent_app.write({self.key_output_image: filepath}, **kwargs)
diff --git a/tests/test_core.py b/tests/test_core.py
index 0d1fc6f..13aa1b0 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -75,15 +75,16 @@ def test_xy_to_rowcol():
def test_write():
- INPUT.write("/tmp/missing_dir/test_write.tif")
+ INPUT.write("/tmp/test_write.tif")
assert INPUT.out.exists()
- os.remove("/tmp/missing_dir/test_write.tif")
+ INPUT.out.filepath.unlink()
def test_output_write():
INPUT.out.write("/tmp/missing_dir/test_write.tif")
assert INPUT.out.exists()
os.remove("/tmp/missing_dir/test_write.tif")
+ INPUT.out.filepath.unlink()
# Slicer
--
GitLab
From 14f236940ee641bdc6f2e846761258b3c3d1078c Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 00:08:44 +0100
Subject: [PATCH 072/399] CI: remove bad test
---
tests/test_core.py | 7 -------
1 file changed, 7 deletions(-)
diff --git a/tests/test_core.py b/tests/test_core.py
index 13aa1b0..ccd386f 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -80,13 +80,6 @@ def test_write():
INPUT.out.filepath.unlink()
-def test_output_write():
- INPUT.out.write("/tmp/missing_dir/test_write.tif")
- assert INPUT.out.exists()
- os.remove("/tmp/missing_dir/test_write.tif")
- INPUT.out.filepath.unlink()
-
-
# Slicer
def test_slicer_shape():
extract = INPUT[:50, :60, :3]
--
GitLab
From 5fd852ef19b776b6d5fa85491f6aa47473de8a4b Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 00:21:54 +0100
Subject: [PATCH 073/399] FIX: do not overwrite Slicer and Operation name with
App name
---
pyotb/core.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 4d3d848..e90286f 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -902,7 +902,6 @@ class App(RasterInterface):
class Slicer(App):
"""Slicer objects i.e. when we call something like raster[:, :, 2] from Python."""
- name = "Slicer"
def __init__(self, obj: App | str, rows: int, cols: int, channels: int):
"""Create a slicer object, that can be used directly for writing or inside a BandMath.
@@ -919,6 +918,7 @@ class Slicer(App):
"""
super().__init__("ExtractROI", obj, mode="extent", quiet=True, frozen=True)
+ self.name = "Slicer"
self.rows, self.cols = rows, cols
parameters = {}
@@ -1024,10 +1024,10 @@ class Operation(App):
# Getting unique image inputs, in the order im1, im2, im3 ...
self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)]
self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands)
- self.name = f'Operation exp="{self.exp}"'
appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX"
# Execute app
super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True)
+ self.name = f'Operation exp="{self.exp}"'
def create_fake_exp(self, operator: str, inputs: list[App | str | int | float],
nb_bands: int = None):
--
GitLab
From 4ca9f91f385d64cd8043b6b0855b6915f8cbd016 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 10:37:17 +0100
Subject: [PATCH 074/399] ENH: trying to make Slicer fake_exp functions clearer
---
pyotb/core.py | 31 +++++++++++++++----------------
1 file changed, 15 insertions(+), 16 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index e90286f..4df617f 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1008,7 +1008,7 @@ class Operation(App):
self.nb_channels = {}
self.fake_exp_bands = []
self.logical_fake_exp_bands = []
- self.create_fake_exp(operator, inputs, nb_bands=nb_bands)
+ self.build_fake_expressions(operator, inputs, nb_bands=nb_bands)
# Transforming images to the adequate im#, e.g. `input1` to "im1"
# creating a dictionary that is like {str(input1): 'im1', 'image2.tif': 'im2', ...}.
# NB: the keys of the dictionary are strings-only, instead of 'complex' objects, to enable easy serialization
@@ -1029,9 +1029,8 @@ class Operation(App):
super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True)
self.name = f'Operation exp="{self.exp}"'
- def create_fake_exp(self, operator: str, inputs: list[App | str | int | float],
- nb_bands: int = None):
- """Create a 'fake' expression.
+ def build_fake_expressions(self, operator: str, inputs: list[App | str | int | float], nb_bands: int = None):
+ """Create a list of 'fake' expressions, one for each band.
E.g for the operation input1 + input2, we create a fake expression that is like "str(input1) + str(input2)"
@@ -1062,7 +1061,7 @@ class Operation(App):
# Create a list of fake expressions, each item of the list corresponding to one band
self.fake_exp_bands.clear()
for i, band in enumerate(range(1, nb_bands + 1)):
- fake_exps = []
+ expressions = []
for k, inp in enumerate(inputs):
# Generating the fake expression of the current input,
# this is a special case for the condition of the ternary operator `cond ? x : y`
@@ -1076,7 +1075,7 @@ class Operation(App):
else:
# Any other input
fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp(inp, band, keep_logical=False)
- fake_exps.append(fake_exp)
+ expressions.append(fake_exp)
# Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates)
if i == 0 and corresponding_inputs and nb_channels:
self.inputs.extend(corresponding_inputs)
@@ -1084,13 +1083,13 @@ class Operation(App):
# Generating the fake expression of the whole operation
if len(inputs) == 1: # this is only for 'abs'
- fake_exp = f"({operator}({fake_exps[0]}))"
+ fake_exp = f"({operator}({expressions[0]}))"
elif len(inputs) == 2:
# We create here the "fake" expression. For example, for a BandMathX expression such as '2 * im1 + im2',
# the false expression stores the expression 2 * str(input1) + str(input2)
- fake_exp = f"({fake_exps[0]} {operator} {fake_exps[1]})"
+ fake_exp = f"({expressions[0]} {operator} {expressions[1]})"
elif len(inputs) == 3 and operator == "?": # this is only for ternary expression
- fake_exp = f"({fake_exps[0]} ? {fake_exps[1]} : {fake_exps[2]})"
+ fake_exp = f"({expressions[0]} ? {expressions[1]} : {expressions[2]})"
self.fake_exp_bands.append(fake_exp)
def get_real_exp(self, fake_exp_bands: str) -> tuple(list[str], str):
@@ -1118,7 +1117,7 @@ class Operation(App):
@staticmethod
def make_fake_exp(x: App | str, band: int, keep_logical: bool = False) -> tuple(str, list[App], int):
- """This an internal function, only to be used by `create_fake_exp`.
+ """This an internal function, only to be used by `build_fake_expressions`.
Enable to create a fake expression just for one input and one band.
@@ -1199,8 +1198,8 @@ class LogicalOperation(Operation):
super().__init__(operator, *inputs, nb_bands=nb_bands)
self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands)
- def create_fake_exp(self, operator: str, inputs: list[App | str | int | float], nb_bands: int = None):
- """Create a 'fake' expression.
+ def build_fake_expressions(self, operator: str, inputs: list[App | str | int | float], nb_bands: int = None):
+ """Create a list of 'fake' expressions, one for each band.
e.g for the operation input1 > input2, we create a fake expression that is like
"str(input1) > str(input2) ? 1 : 0" and a logical fake expression that is like "str(input1) > str(input2)"
@@ -1224,17 +1223,17 @@ class LogicalOperation(Operation):
# Create a list of fake exp, each item of the list corresponding to one band
for i, band in enumerate(range(1, nb_bands + 1)):
- fake_exps = []
+ expressions = []
for inp in inputs:
fake_exp, corresp_inputs, nb_channels = super().make_fake_exp(inp, band, keep_logical=True)
- fake_exps.append(fake_exp)
+ expressions.append(fake_exp)
# Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates)
if i == 0 and corresp_inputs and nb_channels:
self.inputs.extend(corresp_inputs)
self.nb_channels.update(nb_channels)
# We create here the "fake" expression. For example, for a BandMathX expression such as 'im1 > im2',
# the logical fake expression stores the expression "str(input1) > str(input2)"
- logical_fake_exp = f"({fake_exps[0]} {operator} {fake_exps[1]})"
+ logical_fake_exp = f"({expressions[0]} {operator} {expressions[1]})"
# We keep the logical expression, useful if later combined with other logical operations
self.logical_fake_exp_bands.append(logical_fake_exp)
# We create a valid BandMath expression, e.g. "str(input1) > str(input2) ? 1 : 0"
@@ -1345,7 +1344,7 @@ def get_pixel_type(inp: str | App) -> str:
Returns:
pixel_type: OTB enum e.g. `otbApplication.ImagePixelType_uint8', which actually is an int.
- make_fake_exp For an App with several outputs, only the pixel type of the first output is returned
+ For an App with several outputs, only the pixel type of the first output is returned
"""
if isinstance(inp, str):
--
GitLab
From 923ce740dc61af2c443e86d510b551b65d93a403 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 11:03:03 +0100
Subject: [PATCH 075/399] STYLE: type hints + write allows Path type
---
pyotb/core.py | 81 ++++++++++++++++++++++++++-------------------------
1 file changed, 42 insertions(+), 39 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 4df617f..9278f8f 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -29,7 +29,7 @@ class RasterInterface(ABC):
"""Write image, this is defined in App. Output will use App.write for a specific key."""
@property
- def metadata(self):
+ def metadata(self) -> dict[str, (str, float, list[float])]:
"""Return first output image metadata dictionary."""
return dict(self.app.GetMetadataDictionary(self.key_output_image))
@@ -45,7 +45,7 @@ class RasterInterface(ABC):
return self.app.ConvertPixelTypeToNumpy(enum)
@property
- def shape(self) -> tuple(int):
+ def shape(self) -> tuple[int]:
"""Enables to retrieve the shape of a pyotb object using numpy convention.
Returns:
@@ -57,7 +57,7 @@ class RasterInterface(ABC):
return height, width, bands
@property
- def transform(self) -> tuple(int):
+ def transform(self) -> tuple[int]:
"""Get image affine transform, rasterio style (see https://www.perrygeo.com/python-affine-transforms.html).
Returns:
@@ -69,11 +69,11 @@ class RasterInterface(ABC):
origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2
return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y
- def get_infos(self):
+ def get_infos(self) -> dict[str, (str, float, list[float])]:
"""Return a dict output of ReadImageInfo for the first image output."""
return App("ReadImageInfo", self, quiet=True).data
- def get_statistics(self):
+ def get_statistics(self) -> dict[str, (str, float, list[float])]:
"""Return a dict output of ComputeImagesStatistics for the first image output."""
return App("ComputeImagesStatistics", self, quiet=True).data
@@ -536,7 +536,7 @@ class App(RasterInterface):
if any(key in self.parameters for key in self.out_param_types):
self.flush() # auto flush if any output param was provided during app init
- def get_first_key(self, param_types: list[str]) -> str:
+ def get_first_key(self, param_types: list[int]) -> str:
"""Get the first output param key for specific file types."""
for key, param_type in sorted(self.all_param_types.items()):
if param_type in param_types:
@@ -564,7 +564,7 @@ class App(RasterInterface):
return self.get_first_key(param_types=[otb.ParameterType_OutputImage])
@property
- def elapsed_time(self):
+ def elapsed_time(self) -> float:
"""Get elapsed time between app init and end of exec or file writing."""
return self.time_end - self.time_start
@@ -573,23 +573,8 @@ class App(RasterInterface):
"""List of used application outputs."""
return [getattr(self, key) for key in self.out_param_types if key in self.parameters]
- def find_outputs(self) -> tuple[str]:
- """Find output files on disk using path found in parameters.
-
- Returns:
- list of files found on disk
-
- """
- files, missing = [], []
- for out in self.used_outputs:
- dest = files if out.exists() else missing
- dest.append(str(out.filepath.absolute()))
- for filename in missing:
- logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
- return tuple(files)
-
@property
- def data(self):
+ def data(self) -> dict[str, float, list[float]]:
"""Expose app's output data values in a dictionary."""
skip_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem")
skip_keys = skip_keys + tuple(self.out_param_types) + tuple(self.parameters)
@@ -746,8 +731,8 @@ class App(RasterInterface):
kwargs.update(arg)
elif isinstance(arg, str) and kwargs:
logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg)
- elif isinstance(arg, str) and self.key_output_image:
- kwargs.update({self.key_output_image: arg})
+ elif isinstance(arg, (str, Path)) and self.key_output_image:
+ kwargs.update({self.key_output_image: str(arg)})
# Append filename extension to filenames
if filename_extension:
@@ -779,7 +764,22 @@ class App(RasterInterface):
self.set_parameters({key: output_filename})
self.flush()
- def summarize(self) -> dict:
+ def find_outputs(self) -> tuple[str]:
+ """Find output files on disk using path found in parameters.
+
+ Returns:
+ list of files found on disk
+
+ """
+ files, missing = [], []
+ for out in self.used_outputs:
+ dest = files if out.exists() else missing
+ dest.append(str(out.filepath.absolute()))
+ for filename in missing:
+ logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
+ return tuple(files)
+
+ def summarize(self) -> dict[str, str | dict[str, Any]]:
"""Serialize an object and its pipeline into a dictionary.
Returns:
@@ -848,7 +848,7 @@ class App(RasterInterface):
self.app.SetParameterValue(key, obj)
# Special functions
- def __hash__(self):
+ def __hash__(self) -> int:
"""Override the default behaviour of the hash function.
Returns:
@@ -857,7 +857,7 @@ class App(RasterInterface):
"""
return id(self)
- def __getitem__(self, key):
+ def __getitem__(self, key) -> Any | list[int | float] | int | float | Slicer :
"""Override the default __getitem__ behaviour.
This function enables 2 things :
@@ -895,7 +895,7 @@ class App(RasterInterface):
key = key + (slice(None, None, None),)
return Slicer(self, *key)
- def __str__(self):
+ def __str__(self) -> str:
"""Return a nice string representation with object id."""
return f"<pyotb.App {self.name} object id {id(self)}>"
@@ -1092,7 +1092,7 @@ class Operation(App):
fake_exp = f"({expressions[0]} ? {expressions[1]} : {expressions[2]})"
self.fake_exp_bands.append(fake_exp)
- def get_real_exp(self, fake_exp_bands: str) -> tuple(list[str], str):
+ def get_real_exp(self, fake_exp_bands: str) -> tuple[list[str], str]:
"""Generates the BandMathX expression.
Args:
@@ -1116,7 +1116,7 @@ class Operation(App):
return exp_bands, exp
@staticmethod
- def make_fake_exp(x: App | str, band: int, keep_logical: bool = False) -> tuple(str, list[App], int):
+ def make_fake_exp(x: App | str, band: int, keep_logical: bool = False) -> tuple[str, list[App], int]:
"""This an internal function, only to be used by `build_fake_expressions`.
Enable to create a fake expression just for one input and one band.
@@ -1126,7 +1126,7 @@ class Operation(App):
band: which band to consider (bands start at 1)
keep_logical: whether to keep the logical expressions "as is" in case the input is a logical operation.
ex: if True, for `input1 > input2`, returned fake expression is "str(input1) > str(input2)"
- if False, for `input1 > input2`, returned fake exp is "str(input1) > str(input2) ? 1 : 0".
+ if False, for `input1 > input2`, returned fake exp is "str(input1) > str(input2) ? 1 : 0"]
Default False
Returns:
@@ -1288,24 +1288,27 @@ class Output(RasterInterface):
self.make_parent_dirs()
@property
- def key_output_image(self):
+ def key_output_image(self) -> str:
"""Force the right key to be used when accessing the RasterInterface."""
return self.param_key
def exists(self) -> bool:
"""Check file exist."""
- assert self.filepath, "Filepath not set"
+ if self.filepath is None:
+ raise ValueError("Filepath is not set")
return self.filepath.exists()
def make_parent_dirs(self):
"""Create missing parent directories."""
- assert self.filepath, "Filepath not set"
- if not self.filepath.parent.exists():
- self.filepath.parent.mkdir(parents=True)
+ if self.filepath is None:
+ raise ValueError("Filepath is not set")
+ self.filepath.parent.mkdir(parents=True, exist_ok=True)
- def write(self, filepath: str = "", **kwargs):
+ def write(self, filepath: None | str | Path = None, **kwargs):
"""Write output to disk, filepath is not required if it was provided to parent App during init."""
- if not filepath and self.filepath:
+ if filepath is None and self.filepath:
+ if self.filepath.exists():
+ logger.warning("Overwriting file %s", self.filepath)
return self.parent_app.write({self.key_output_image: self.filepath}, **kwargs)
return self.parent_app.write({self.key_output_image: filepath}, **kwargs)
--
GitLab
From 43d18833d897fb80eb358af5c813d183cbc0e0cd Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 11:12:08 +0100
Subject: [PATCH 076/399] STYLE: linter
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 9278f8f..19a23f9 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -857,7 +857,7 @@ class App(RasterInterface):
"""
return id(self)
- def __getitem__(self, key) -> Any | list[int | float] | int | float | Slicer :
+ def __getitem__(self, key) -> Any | list[int | float] | int | float | Slicer:
"""Override the default __getitem__ behaviour.
This function enables 2 things :
--
GitLab
From a133ed8bf2828b01b5b42303af34a2b173bffc3d Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 11:27:46 +0100
Subject: [PATCH 077/399] ENH: raise warning when overwriting file with
App.write()
---
pyotb/core.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 19a23f9..acf0541 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -759,6 +759,8 @@ class App(RasterInterface):
# Set parameters and flush to disk
for key, output_filename in kwargs.items():
+ if Path(output_filename).exists():
+ logger.warning("%s: overwriting file %s", self.name, output_filename)
if key in dtypes:
self.propagate_dtype(key, dtypes[key])
self.set_parameters({key: output_filename})
@@ -1307,8 +1309,6 @@ class Output(RasterInterface):
def write(self, filepath: None | str | Path = None, **kwargs):
"""Write output to disk, filepath is not required if it was provided to parent App during init."""
if filepath is None and self.filepath:
- if self.filepath.exists():
- logger.warning("Overwriting file %s", self.filepath)
return self.parent_app.write({self.key_output_image: self.filepath}, **kwargs)
return self.parent_app.write({self.key_output_image: filepath}, **kwargs)
--
GitLab
From eb735137d4739a4bc7cf2a5fcdab85a47d2d551f Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 11:31:45 +0100
Subject: [PATCH 078/399] ENH: fix raise error and print available values if a
bad key is passed to set_parameters
---
pyotb/core.py | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index acf0541..b9cc5a1 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -607,7 +607,9 @@ class App(RasterInterface):
# Going through all arguments
for key, obj in parameters.items():
if key not in self.parameters_keys:
- raise KeyError(f'{self.name}: unknown parameter name "{key}"')
+ raise KeyError(
+ f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}"
+ )
# When the parameter expects a list, if needed, change the value to list
if is_key_list(self, key) and not isinstance(obj, (list, tuple)):
obj = [obj]
@@ -821,10 +823,6 @@ class App(RasterInterface):
if obj is None or (isinstance(obj, (list, tuple)) and not obj):
self.app.ClearValue(key)
return
- if key not in self.parameters_keys:
- raise KeyError(
- f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}"
- )
# Single-parameter cases
if isinstance(obj, App):
self.app.ConnectImage(key, obj.app, obj.key_output_image)
--
GitLab
From beb168dd000e505ed9a8eebbcc49388f9621def9 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 11:58:59 +0100
Subject: [PATCH 079/399] ENH: back to Output attr parent_pyotb_app to avoid
confusion with bare OTB app
---
pyotb/core.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index b9cc5a1..1f727ff 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1276,7 +1276,7 @@ class Output(RasterInterface):
"""
self.name = f"Output {param_key} from {pyotb_app.name}"
- self.parent_app = pyotb_app # keep trace of parent app
+ self.parent_pyotb_app = pyotb_app # keep trace of parent app
self.app = pyotb_app.app
self.param_key = param_key
self.filepath = None
@@ -1307,8 +1307,8 @@ class Output(RasterInterface):
def write(self, filepath: None | str | Path = None, **kwargs):
"""Write output to disk, filepath is not required if it was provided to parent App during init."""
if filepath is None and self.filepath:
- return self.parent_app.write({self.key_output_image: self.filepath}, **kwargs)
- return self.parent_app.write({self.key_output_image: filepath}, **kwargs)
+ return self.parent_pyotb_app.write({self.key_output_image: self.filepath}, **kwargs)
+ return self.parent_pyotb_app.write({self.key_output_image: filepath}, **kwargs)
def __str__(self) -> str:
"""Return a nice string representation with source app name and object id."""
--
GitLab
From c3aa56231b014f6bfcfeed32a7f9c31865426bee Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 14:25:15 +0100
Subject: [PATCH 080/399] ENH: search OTB on MacOS default app dir
---
pyotb/helpers.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index ebccc48..5d9267e 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -259,15 +259,15 @@ def __find_otb_root(scan_userdir: bool = False):
logger.info("Found %s", path.parent)
prefix = path.parent.absolute()
elif sys.platform == "darwin":
- # TODO: find OTB in macOS
- pass
-
+ for path in (Path.home() / "Applications").glob("**/OTB-*/lib"):
+ logger.info("Found %s", path.parent)
+ prefix = path.parent.absolute()
# If possible, use OTB found in user's HOME tree (this may take some time)
if scan_userdir:
- for path in Path().home().glob("**/OTB-*/lib"):
+ for path in Path.home().glob("**/OTB-*/lib"):
logger.info("Found %s", path.parent)
prefix = path.parent.absolute()
-
+ # Return latest found prefix (and version), see precedence in function def find_otb()
return prefix
--
GitLab
From fbc0360dd8a47d62163f22f44bc21e779e0e9477 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 14:34:28 +0100
Subject: [PATCH 081/399] STYLE: auto format and save some space
---
pyotb/core.py | 105 +++++++++++++++++++++--------------------------
pyotb/helpers.py | 13 +++---
2 files changed, 54 insertions(+), 64 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 1f727ff..b50b7ff 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -16,6 +16,7 @@ from .helpers import logger
class RasterInterface(ABC):
"""Abstraction of an image object."""
+
app: otb.Application
exports_dic: dict
@@ -175,14 +176,13 @@ class RasterInterface(ABC):
"""
array = self.to_numpy(preserve_dtype=True, copy=False)
- array = np.moveaxis(array, 2, 0)
+ height, width, count = array.shape
proj = self.app.GetImageProjection(self.key_output_image)
profile = {
- 'crs': proj, 'dtype': array.dtype,
- 'count': array.shape[0], 'height': array.shape[1], 'width': array.shape[2],
- 'transform': self.transform
+ 'crs': proj, 'dtype': array.dtype, 'transform': self.transform,
+ 'count': count, 'height': height, 'width': width,
}
- return array, profile
+ return np.moveaxis(array, 2, 0), profile
def xy_to_rowcol(self, x: float, y: float) -> tuple[int, int]:
"""Find (row, col) index using (x, y) projected coordinates - image CRS is expected.
@@ -514,12 +514,14 @@ class App(RasterInterface):
e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif'
"""
- self.parameters = {}
self.name = name
self.frozen = frozen
self.quiet = quiet
self.image_dic = image_dic
+ self.time_start, self.time_end = 0, 0
self.exports_dic = {}
+ self.parameters = {}
+ # Initialize app, set parameters and execute if not frozen
create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
self.app = create(name)
self.parameters_keys = tuple(self.app.GetParametersKeys())
@@ -546,12 +548,9 @@ class App(RasterInterface):
@property
def key_input(self) -> str:
"""Get the name of first input parameter, raster > vector > file."""
- return self.get_first_key(param_types=[otb.ParameterType_InputImage,
- otb.ParameterType_InputImageList]) \
- or self.get_first_key(param_types=[otb.ParameterType_InputVectorData,
- otb.ParameterType_InputVectorDataList]) \
- or self.get_first_key(param_types=[otb.ParameterType_InputFilename,
- otb.ParameterType_InputFilenameList])
+ return self.get_first_key([otb.ParameterType_InputImage, otb.ParameterType_InputImageList]) \
+ or self.get_first_key([otb.ParameterType_InputVectorData, otb.ParameterType_InputVectorDataList]) \
+ or self.get_first_key([otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList])
@property
def key_input_image(self) -> str:
@@ -576,14 +575,14 @@ class App(RasterInterface):
@property
def data(self) -> dict[str, float, list[float]]:
"""Expose app's output data values in a dictionary."""
- skip_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem")
- skip_keys = skip_keys + tuple(self.out_param_types) + tuple(self.parameters)
- keys = (k for k in self.parameters_keys if k not in skip_keys)
-
- def _check(v):
- return not isinstance(v, otb.ApplicationProxy) and v not in ("", None, [], ())
-
- return {str(k): self[k] for k in keys if _check(self[k])}
+ known_bad_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem")
+ skip_keys = known_bad_keys + tuple(self._out_param_types) + tuple(self.parameters)
+ data_dict = {}
+ for key in filter(lambda k: k not in skip_keys, self.parameters_keys):
+ value = self.__dict__.get(key)
+ if not isinstance(value, otb.ApplicationProxy) and value not in (None, "", [], ()):
+ data_dict[str(key)] = value
+ return data_dict
def set_parameters(self, *args, **kwargs):
"""Set some parameters of the app.
@@ -1053,8 +1052,7 @@ class Operation(App):
else:
nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))]
# check that all inputs have the same nb of bands
- if len(nb_bands_list) > 1:
- if not all(x == nb_bands_list[0] for x in nb_bands_list):
+ if len(nb_bands_list) > 1 and not all(x == nb_bands_list[0] for x in nb_bands_list):
raise ValueError("All images do not have the same number of bands")
nb_bands = nb_bands_list[0]
@@ -1067,10 +1065,7 @@ class Operation(App):
# this is a special case for the condition of the ternary operator `cond ? x : y`
if len(inputs) == 3 and k == 0:
# When cond is monoband whereas the result is multiband, we expand the cond to multiband
- if nb_bands != inp.shape[2]:
- cond_band = 1
- else:
- cond_band = band
+ cond_band = 1 if nb_bands != inp.shape[2] else band
fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp(inp, cond_band, keep_logical=True)
else:
# Any other input
@@ -1139,38 +1134,31 @@ class Operation(App):
if isinstance(x, Slicer) and hasattr(x, "one_band_sliced"):
if keep_logical and isinstance(x.input, LogicalOperation):
fake_exp = x.input.logical_fake_exp_bands[x.one_band_sliced - 1]
- inputs = x.input.inputs
- nb_channels = x.input.nb_channels
+ inputs, nb_channels = x.input.inputs, x.input.nb_channels
elif isinstance(x.input, Operation):
# Keep only one band of the expression
fake_exp = x.input.fake_exp_bands[x.one_band_sliced - 1]
- inputs = x.input.inputs
- nb_channels = x.input.nb_channels
+ inputs, nb_channels = x.input.inputs, x.input.nb_channels
else:
# Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1')
fake_exp = f"{x.input}b{x.one_band_sliced}"
- inputs = [x.input]
- nb_channels = {x.input: 1}
+ inputs, nb_channels = [x.input], {x.input: 1}
# For LogicalOperation, we save almost the same attributes as an Operation
elif keep_logical and isinstance(x, LogicalOperation):
fake_exp = x.logical_fake_exp_bands[band - 1]
- inputs = x.inputs
- nb_channels = x.nb_channels
+ inputs, nb_channels = x.inputs, x.nb_channels
elif isinstance(x, Operation):
fake_exp = x.fake_exp_bands[band - 1]
- inputs = x.inputs
- nb_channels = x.nb_channels
+ inputs, nb_channels = x.inputs, x.nb_channels
# For int or float input, we just need to save their value
elif isinstance(x, (int, float)):
fake_exp = str(x)
- inputs = None
- nb_channels = None
+ inputs, nb_channels = None, None
# We go on with other inputs, i.e. pyotb objects, filepaths...
else:
- nb_channels = {x: get_nbchannels(x)}
- inputs = [x]
# Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1')
fake_exp = f"{x}b{band}"
+ inputs, nb_channels = [x], {x: get_nbchannels(x)}
return fake_exp, inputs, nb_channels
def __str__(self) -> str:
@@ -1216,8 +1204,7 @@ class LogicalOperation(Operation):
else:
nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))]
# check that all inputs have the same nb of bands
- if len(nb_bands_list) > 1:
- if not all(x == nb_bands_list[0] for x in nb_bands_list):
+ if len(nb_bands_list) > 1 and not all(x == nb_bands_list[0] for x in nb_bands_list):
raise ValueError("All images do not have the same number of bands")
nb_bands = nb_bands_list[0]
@@ -1281,8 +1268,8 @@ class Output(RasterInterface):
self.param_key = param_key
self.filepath = None
if filepath:
- if '?' in filepath:
- filepath = filepath.split('?')[0]
+ if "?" in filepath:
+ filepath = filepath.split("?")[0]
self.filepath = Path(filepath)
if mkdir:
self.make_parent_dirs()
@@ -1333,7 +1320,7 @@ def get_nbchannels(inp: str | App) -> int:
info = App("ReadImageInfo", inp, quiet=True)
nb_channels = info.app.GetParameterInt("numberbands")
except Exception as e: # this happens when we pass a str that is not a filepath
- raise TypeError(f'Could not get the number of channels of `{inp}`. Not a filepath or wrong filepath') from e
+ raise TypeError(f"Could not get the number of channels of '{inp}'. Not a filepath or wrong filepath") from e
return nb_channels
@@ -1357,23 +1344,23 @@ def get_pixel_type(inp: str | App) -> str:
if not datatype:
raise TypeError(f"Unable to read pixel type of image {inp}")
datatype_to_pixeltype = {
- 'unsigned_char': 'uint8',
- 'short': 'int16',
- 'unsigned_short': 'uint16',
- 'int': 'int32',
- 'unsigned_int': 'uint32',
- 'long': 'int32',
- 'ulong': 'uint32',
- 'float': 'float',
- 'double': 'double'
+ "unsigned_char": "uint8",
+ "short": "int16",
+ "unsigned_short": "uint16",
+ "int": "int32",
+ "unsigned_int": "uint32",
+ "long": "int32",
+ "ulong": "uint32",
+ "float": "float",
+ "double": "double",
}
if datatype not in datatype_to_pixeltype:
raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}")
- pixel_type = getattr(otb, f'ImagePixelType_{datatype_to_pixeltype[datatype]}')
+ pixel_type = getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[datatype]}")
elif isinstance(inp, App):
pixel_type = inp.app.GetParameterOutputImagePixelType(inp.key_output_image)
else:
- raise TypeError(f'Could not get the pixel type of {type(inp)} object {inp}')
+ raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}")
return pixel_type
@@ -1388,10 +1375,10 @@ def parse_pixel_type(pixel_type: str | int) -> int:
"""
if isinstance(pixel_type, str): # this correspond to 'uint8' etc...
- return getattr(otb, f'ImagePixelType_{pixel_type}')
+ return getattr(otb, f"ImagePixelType_{pixel_type}")
if isinstance(pixel_type, int):
return pixel_type
- raise ValueError(f'Bad pixel type specification ({pixel_type})')
+ raise ValueError(f"Bad pixel type specification ({pixel_type})")
def is_key_list(pyotb_app: App, key: str) -> bool:
@@ -1409,7 +1396,7 @@ def is_key_images_list(pyotb_app: App, key: str) -> bool:
"""Check if a key of the App is an input parameter image list."""
return pyotb_app.app.GetParameterType(key) in (
otb.ParameterType_InputImageList,
- otb.ParameterType_InputFilenameList
+ otb.ParameterType_InputFilenameList,
)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 5d9267e..fee8d00 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-"""This module provides some helpers to properly initialize pyotb."""
+"""This module helps to ensure we properly initialize pyotb: only in case OTB is found and apps are available."""
import os
import sys
import logging
@@ -41,8 +41,9 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru
Path precedence : OTB_ROOT > python bindings directory
OR search for releases installations : HOME
- OR (for linux) : /opt/otbtf > /opt/otb > /usr/local > /usr
- OR (for windows) : C:/Program Files
+ OR (for Linux) : /opt/otbtf > /opt/otb > /usr/local > /usr
+ OR (for MacOS) : ~/Applications
+ OR (for Windows) : C:/Program Files
Args:
prefix: prefix to search OTB in (Default value = OTB_ROOT)
@@ -135,9 +136,9 @@ def set_environment(prefix: str):
os.environ["OTB_APPLICATION_PATH"] = apps_path
else:
raise EnvironmentError("Can't find OTB applications directory")
-
os.environ["LC_NUMERIC"] = "C"
os.environ["GDAL_DRIVER_PATH"] = "disable"
+
if (prefix / "share/gdal").exists():
# Local GDAL (OTB Superbuild, .run, .exe)
gdal_data = str(prefix / "share/gdal")
@@ -151,7 +152,6 @@ def set_environment(prefix: str):
proj_lib = str(prefix / "share/proj")
else:
raise EnvironmentError(f"Can't find GDAL location with current OTB prefix '{prefix}' or in /usr")
-
os.environ["GDAL_DATA"] = gdal_data
os.environ["PROJ_LIB"] = proj_lib
@@ -303,3 +303,6 @@ def __suggest_fix_import(error_message: str, prefix: str):
" first use 'call otbenv.bat' then try to import pyotb once again")
docs_link = "https://www.orfeo-toolbox.org/CookBook/Installation.html"
logger.critical("You can verify installation requirements for your OS at %s", docs_link)
+
+# Since helpers is the first module to be inititialized, this will prevent pyotb to run if OTB is not found
+find_otb()
--
GitLab
From 2f0fb7e5b27ee165c5d5d365624ef181e299bca8 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 14:34:35 +0100
Subject: [PATCH 082/399] ENH: move find_otb call to helpers module
---
pyotb/__init__.py | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index e3bc23a..ac6264c 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -2,12 +2,8 @@
"""This module provides convenient python wrapping of otbApplications."""
__version__ = "1.6.0"
-from .helpers import find_otb, logger, set_logger_level
-
-otb = find_otb()
-
+from .helpers import logger, set_logger_level
from .apps import *
-
from .core import (
App,
Input,
@@ -15,7 +11,6 @@ from .core import (
get_nbchannels,
get_pixel_type
)
-
from .functions import ( # pylint: disable=redefined-builtin
all,
any,
--
GitLab
From ea9e744555f36c72d077a4b5cf33ecb66d8a8ec5 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 14:35:08 +0100
Subject: [PATCH 083/399] ENH: make out_param_types and all_param_types private
---
pyotb/core.py | 23 ++++++++++-------------
1 file changed, 10 insertions(+), 13 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index b50b7ff..40e809b 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -525,22 +525,19 @@ class App(RasterInterface):
create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
self.app = create(name)
self.parameters_keys = tuple(self.app.GetParametersKeys())
- self.all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys}
- self.out_param_types = {k: v for k, v in self.all_param_types.items()
- if v in (otb.ParameterType_OutputImage,
- otb.ParameterType_OutputVectorData,
- otb.ParameterType_OutputFilename)}
- self.time_start, self.time_end = 0, 0
+ self._all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys}
+ otypes = (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename)
+ self._out_param_types = {k: v for k, v in self._all_param_types.items() if v in otypes}
if args or kwargs:
self.set_parameters(*args, **kwargs)
if not self.frozen:
self.execute()
- if any(key in self.parameters for key in self.out_param_types):
+ if any(key in self.parameters for key in self._out_param_types):
self.flush() # auto flush if any output param was provided during app init
def get_first_key(self, param_types: list[int]) -> str:
"""Get the first output param key for specific file types."""
- for key, param_type in sorted(self.all_param_types.items()):
+ for key, param_type in sorted(self._all_param_types.items()):
if param_type in param_types:
return key
return None
@@ -570,7 +567,7 @@ class App(RasterInterface):
@property
def used_outputs(self) -> list[str]:
"""List of used application outputs."""
- return [getattr(self, key) for key in self.out_param_types if key in self.parameters]
+ return [getattr(self, key) for key in self._out_param_types if key in self.parameters]
@property
def data(self) -> dict[str, float, list[float]]:
@@ -654,7 +651,7 @@ class App(RasterInterface):
if target_key:
keys = [target_key]
else:
- keys = [k for k, v in self.out_param_types.items() if v == otb.ParameterType_OutputImage]
+ keys = [k for k, v in self._out_param_types.items() if v == otb.ParameterType_OutputImage]
for key in keys:
self.app.SetParameterOutputImagePixelType(key, dtype)
@@ -673,7 +670,7 @@ class App(RasterInterface):
except RuntimeError:
continue # this is when there is no value for key
# Convert output param path to Output object
- if key in self.out_param_types:
+ if key in self._out_param_types:
value = Output(self, key, value)
elif isinstance(value, str):
try:
@@ -741,7 +738,7 @@ class App(RasterInterface):
if not filename_extension.startswith("?"):
filename_extension = "?" + filename_extension
for key, value in kwargs.items():
- if self.out_param_types[key] == otb.ParameterType_OutputImage and '?' not in value:
+ if self._out_param_types[key] == otb.ParameterType_OutputImage and "?" not in value:
kwargs[key] = value + filename_extension
# Manage output pixel types
@@ -751,7 +748,7 @@ class App(RasterInterface):
type_name = self.app.ConvertPixelTypeToNumpy(parse_pixel_type(pixel_type))
logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name)
for key in kwargs:
- if self.out_param_types[key] == otb.ParameterType_OutputImage:
+ if self._out_param_types[key] == otb.ParameterType_OutputImage:
dtypes[key] = parse_pixel_type(pixel_type)
elif isinstance(pixel_type, dict):
dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()}
--
GitLab
From 9dd47e300e2aec377dc2f3ed0b2b4f34a4676d4d Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 14:35:58 +0100
Subject: [PATCH 084/399] ENH: ensure to raise AttributeError when unknown
attribute is accessed
---
pyotb/core.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 40e809b..4d6d8a5 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -872,7 +872,7 @@ class App(RasterInterface):
"""
# Accessing string attributes
if isinstance(key, str):
- return self.__dict__.get(key)
+ return getattr(self, key)
# Accessing pixel value(s) using Y/X coordinates
if isinstance(key, tuple) and len(key) >= 2:
row, col = key[0], key[1]
@@ -1050,7 +1050,7 @@ class Operation(App):
nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))]
# check that all inputs have the same nb of bands
if len(nb_bands_list) > 1 and not all(x == nb_bands_list[0] for x in nb_bands_list):
- raise ValueError("All images do not have the same number of bands")
+ raise ValueError("All images do not have the same number of bands")
nb_bands = nb_bands_list[0]
# Create a list of fake expressions, each item of the list corresponding to one band
@@ -1202,7 +1202,7 @@ class LogicalOperation(Operation):
nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))]
# check that all inputs have the same nb of bands
if len(nb_bands_list) > 1 and not all(x == nb_bands_list[0] for x in nb_bands_list):
- raise ValueError("All images do not have the same number of bands")
+ raise ValueError("All images do not have the same number of bands")
nb_bands = nb_bands_list[0]
# Create a list of fake exp, each item of the list corresponding to one band
--
GitLab
From e8f376f796524c175c70963a2a1274253ef66f64 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 14:37:19 +0100
Subject: [PATCH 085/399] STYLE: remove useless pylint exceptions + linting
---
pyotb/helpers.py | 1 +
pyproject.toml | 4 +---
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index fee8d00..69aac02 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -304,5 +304,6 @@ def __suggest_fix_import(error_message: str, prefix: str):
docs_link = "https://www.orfeo-toolbox.org/CookBook/Installation.html"
logger.critical("You can verify installation requirements for your OS at %s", docs_link)
+
# Since helpers is the first module to be inititialized, this will prevent pyotb to run if OTB is not found
find_otb()
diff --git a/pyproject.toml b/pyproject.toml
index 9538fc1..10c15b9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -48,9 +48,7 @@ disable = [
"too-many-locals",
"too-many-branches",
"too-many-statements",
- "too-many-public-methods",
- "too-many-instance-attributes",
- "wrong-import-position",
+ "too-many-instance-attributes"
]
[tool.pydocstyle]
--
GitLab
From cd750a192c8ed1f1fc58ad460a111b4072a2ce32 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 15:30:38 +0100
Subject: [PATCH 086/399] STYLE: better pylint config + save some space
---
pyotb/apps.py | 4 ++--
pyotb/core.py | 32 +++++++++++++-------------
pyotb/functions.py | 56 +++++++++++++---------------------------------
pyotb/helpers.py | 4 ++--
pyproject.toml | 4 ++--
5 files changed, 37 insertions(+), 63 deletions(-)
diff --git a/pyotb/apps.py b/pyotb/apps.py
index 9421b28..bad5089 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -36,9 +36,9 @@ def get_available_applications(as_subprocess: bool = False) -> list[str]:
cmd_args = [sys.executable, "-c", pycmd]
try:
params = {"env": env, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE}
- with subprocess.Popen(cmd_args, **params) as p:
+ with subprocess.Popen(cmd_args, **params) as process:
logger.debug('Exec "%s \'%s\'"', ' '.join(cmd_args[:-1]), pycmd)
- stdout, stderr = p.communicate()
+ stdout, stderr = process.communicate()
stdout, stderr = stdout.decode(), stderr.decode()
# ast.literal_eval is secure and will raise more handy Exceptions than eval
from ast import literal_eval # pylint: disable=import-outside-toplevel
diff --git a/pyotb/core.py b/pyotb/core.py
index 4d6d8a5..b488a38 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -526,8 +526,8 @@ class App(RasterInterface):
self.app = create(name)
self.parameters_keys = tuple(self.app.GetParametersKeys())
self._all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys}
- otypes = (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename)
- self._out_param_types = {k: v for k, v in self._all_param_types.items() if v in otypes}
+ types = (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename)
+ self._out_param_types = {k: v for k, v in self._all_param_types.items() if v in types}
if args or kwargs:
self.set_parameters(*args, **kwargs)
if not self.frozen:
@@ -786,14 +786,14 @@ class App(RasterInterface):
nested dictionary summarizing the pipeline
"""
- params = self.parameters
- for k, p in params.items():
+ parameters = self.parameters.copy()
+ for key, param in parameters.items():
# In the following, we replace each parameter which is an App, with its summary.
- if isinstance(p, App): # single parameter
- params[k] = p.summarize()
- elif isinstance(p, list): # parameter list
- params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p]
- return {"name": self.app.GetName(), "parameters": params}
+ if isinstance(param, App): # single parameter
+ parameters[key] = param.summarize()
+ elif isinstance(param, list): # parameter list
+ parameters[key] = [p.summarize() if isinstance(p, App) else p for p in param]
+ return {"name": self.app.GetName(), "parameters": parameters}
# Private functions
def __parse_args(self, args: list[str | App | dict | list]) -> dict[str, Any]:
@@ -1104,8 +1104,7 @@ class Operation(App):
one_band_exp = one_band_exp.replace(str(inp), self.im_dic[str(inp)])
exp_bands.append(one_band_exp)
# Form the final expression (e.g. 'im1b1 + 1; im1b2 + 1')
- exp = ";".join(exp_bands)
- return exp_bands, exp
+ return exp_bands, ";".join(exp_bands)
@staticmethod
def make_fake_exp(x: App | str, band: int, keep_logical: bool = False) -> tuple[str, list[App], int]:
@@ -1156,6 +1155,7 @@ class Operation(App):
# Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1')
fake_exp = f"{x}b{band}"
inputs, nb_channels = [x], {x: get_nbchannels(x)}
+
return fake_exp, inputs, nb_channels
def __str__(self) -> str:
@@ -1204,7 +1204,6 @@ class LogicalOperation(Operation):
if len(nb_bands_list) > 1 and not all(x == nb_bands_list[0] for x in nb_bands_list):
raise ValueError("All images do not have the same number of bands")
nb_bands = nb_bands_list[0]
-
# Create a list of fake exp, each item of the list corresponding to one band
for i, band in enumerate(range(1, nb_bands + 1)):
expressions = []
@@ -1380,21 +1379,20 @@ def parse_pixel_type(pixel_type: str | int) -> int:
def is_key_list(pyotb_app: App, key: str) -> bool:
"""Check if a key of the App is an input parameter list."""
- return pyotb_app.app.GetParameterType(key) in (
+ types = (
otb.ParameterType_InputImageList,
otb.ParameterType_StringList,
otb.ParameterType_InputFilenameList,
otb.ParameterType_ListView,
otb.ParameterType_InputVectorDataList,
)
+ return pyotb_app.app.GetParameterType(key) in types
def is_key_images_list(pyotb_app: App, key: str) -> bool:
"""Check if a key of the App is an input parameter image list."""
- return pyotb_app.app.GetParameterType(key) in (
- otb.ParameterType_InputImageList,
- otb.ParameterType_InputFilenameList,
- )
+ types = (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList)
+ return pyotb_app.app.GetParameterType(key) in types
def get_out_images_param_keys(app: App) -> list[str]:
diff --git a/pyotb/functions.py b/pyotb/functions.py
index 250f79e..d782a0e 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -34,7 +34,6 @@ def where(cond: App | str, x: App | str | int | float, y: App | str | int | floa
x_nb_channels = get_nbchannels(x)
if not isinstance(y, (int, float)):
y_nb_channels = get_nbchannels(y)
-
if x_nb_channels and y_nb_channels:
if x_nb_channels != y_nb_channels:
raise ValueError('X and Y images do not have the same number of bands. '
@@ -42,44 +41,38 @@ def where(cond: App | str, x: App | str | int | float, y: App | str | int | floa
x_or_y_nb_channels = x_nb_channels if x_nb_channels else y_nb_channels
cond_nb_channels = get_nbchannels(cond)
-
- # Get the number of bands of the result
- if x_or_y_nb_channels: # if X or Y is a raster
- out_nb_channels = x_or_y_nb_channels
- else: # if only cond is a raster
- out_nb_channels = cond_nb_channels
-
if cond_nb_channels != 1 and x_or_y_nb_channels and cond_nb_channels != x_or_y_nb_channels:
raise ValueError('Condition and X&Y do not have the same number of bands. Condition has '
f'{cond_nb_channels} bands whereas X&Y have {x_or_y_nb_channels} bands')
-
# If needed, duplicate the single band binary mask to multiband to match the dimensions of x & y
if cond_nb_channels == 1 and x_or_y_nb_channels and x_or_y_nb_channels != 1:
logger.info('The condition has one channel whereas X/Y has/have %s channels. Expanding number'
' of channels of condition to match the number of channels of X/Y', x_or_y_nb_channels)
- operation = Operation('?', cond, x, y, nb_bands=out_nb_channels)
+ # Get the number of bands of the result
+ if x_or_y_nb_channels: # if X or Y is a raster
+ out_nb_channels = x_or_y_nb_channels
+ else: # if only cond is a raster
+ out_nb_channels = cond_nb_channels
- return operation
+ return Operation('?', cond, x, y, nb_bands=out_nb_channels)
-def clip(a: App | str, a_min: App | str | int | float, a_max: App | str | int | float):
+def clip(image: App | str, v_min: App | str | int | float, v_max: App | str | int | float):
"""Clip values of image in a range of values.
Args:
- a: input raster, can be filepath or any pyotb object
- a_min: minimum value of the range
- a_max: maximum value of the range
+ image: input raster, can be filepath or any pyotb object
+ v_min: minimum value of the range
+ v_max: maximum value of the range
Returns:
raster whose values are clipped in the range
"""
- if isinstance(a, str):
- a = Input(a)
-
- res = where(a <= a_min, a_min,
- where(a >= a_max, a_max, a))
+ if isinstance(image, str):
+ image = Input(image)
+ res = where(image <= v_min, v_min, where(image >= v_max, v_max, image))
return res
@@ -102,11 +95,9 @@ def all(*inputs): # pylint: disable=redefined-builtin
# If necessary, flatten inputs
if len(inputs) == 1 and isinstance(inputs[0], (list, tuple)):
inputs = inputs[0]
-
# Add support for generator inputs (to have the same behavior as built-in `all` function)
if isinstance(inputs, tuple) and len(inputs) == 1 and inspect.isgenerator(inputs[0]):
inputs = list(inputs[0])
-
# Transforming potential filepaths to pyotb objects
inputs = [Input(inp) if isinstance(inp, str) else inp for inp in inputs]
@@ -117,13 +108,11 @@ def all(*inputs): # pylint: disable=redefined-builtin
res = inp[:, :, 0]
else:
res = (inp[:, :, 0] != 0)
-
for band in range(1, inp.shape[-1]):
if isinstance(inp, LogicalOperation):
res = res & inp[:, :, band]
else:
res = res & (inp[:, :, band] != 0)
-
# Checking that all images are True
else:
if isinstance(inputs[0], LogicalOperation):
@@ -157,11 +146,9 @@ def any(*inputs): # pylint: disable=redefined-builtin
# If necessary, flatten inputs
if len(inputs) == 1 and isinstance(inputs[0], (list, tuple)):
inputs = inputs[0]
-
# Add support for generator inputs (to have the same behavior as built-in `any` function)
if isinstance(inputs, tuple) and len(inputs) == 1 and inspect.isgenerator(inputs[0]):
inputs = list(inputs[0])
-
# Transforming potential filepaths to pyotb objects
inputs = [Input(inp) if isinstance(inp, str) else inp for inp in inputs]
@@ -240,7 +227,6 @@ def run_tf_function(func):
func_name = func.__name__
create_and_save_model_str = func_def_str
-
# Adding the instructions to create the model and save it to output dir
create_and_save_model_str += textwrap.dedent(f"""
import tensorflow as tf
@@ -348,7 +334,6 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
inputs.extend(arg)
else:
inputs.append(arg)
-
# Getting metadatas of inputs
metadatas = {}
for inp in inputs:
@@ -362,7 +347,6 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
# Get a metadata of an arbitrary image. This is just to compare later with other images
any_metadata = next(iter(metadatas.values()))
-
# Checking if all images have the same projection
if not all(metadata['ProjectionRef'] == any_metadata['ProjectionRef']
for metadata in metadatas.values()):
@@ -403,9 +387,8 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
# TODO : it is when the user wants the final bounding box to be the union of all bounding box
# It should replace any 'outside' pixel by some NoData -> add `fillvalue` argument in the function
- logger.info('Cropping all images to extent Upper Left (%s, %s), Lower Right (%s, %s)', ulx, uly, lrx, lry)
-
# Applying this bounding box to all inputs
+ logger.info('Cropping all images to extent Upper Left (%s, %s), Lower Right (%s, %s)', ulx, uly, lrx, lry)
new_inputs = []
for inp in inputs:
try:
@@ -425,13 +408,11 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
logger.error('Cannot define the processing area for input %s: %s', inp, e)
raise
inputs = new_inputs
-
# Update metadatas
metadatas = {input: input.app.GetImageMetaData('out') for input in inputs}
# Get a metadata of an arbitrary image. This is just to compare later with other images
any_metadata = next(iter(metadatas.values()))
-
# Handling different pixel sizes
if not all(metadata['GeoTransform'][1] == any_metadata['GeoTransform'][1]
and metadata['GeoTransform'][5] == any_metadata['GeoTransform'][5]
@@ -449,9 +430,9 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
pass
# TODO : when the user explicitly specify the pixel size -> add argument inside the function
pixel_size = metadatas[reference_input]['GeoTransform'][1]
- logger.info('Resampling all inputs to resolution: %s', pixel_size)
# Perform resampling on inputs that do not comply with the target pixel size
+ logger.info('Resampling all inputs to resolution: %s', pixel_size)
new_inputs = []
for inp in inputs:
if metadatas[inp]['GeoTransform'][1] != pixel_size:
@@ -460,18 +441,14 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
else:
new_inputs.append(inp)
inputs = new_inputs
-
- # Update metadatas
metadatas = {inp: inp.app.GetImageMetaData('out') for inp in inputs}
# Final superimposition to be sure to have the exact same image sizes
- # Getting the sizes of images
image_sizes = {}
for inp in inputs:
if isinstance(inp, str):
inp = Input(inp)
image_sizes[inp] = inp.shape[:2]
-
# Selecting the most frequent image size. It will be used as reference.
most_common_image_size, _ = Counter(image_sizes.values()).most_common(1)[0]
same_size_images = [inp for inp, image_size in image_sizes.items() if image_size == most_common_image_size]
@@ -484,6 +461,5 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
new_inputs.append(superimposed)
else:
new_inputs.append(inp)
- inputs = new_inputs
- return inputs
+ return new_inputs
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 69aac02..03200a2 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -77,9 +77,9 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru
otb.Registry.SetApplicationPath(apps_path)
return otb
except ImportError as e:
- PYTHONPATH = os.environ.get("PYTHONPATH")
+ pythonpath = os.environ.get("PYTHONPATH")
if not scan:
- raise SystemExit(f"Failed to import OTB with env PYTHONPATH={PYTHONPATH}") from e
+ raise SystemExit(f"Failed to import OTB with env PYTHONPATH={pythonpath}") from e
# Else search system
logger.info("Failed to import OTB. Searching for it...")
prefix = __find_otb_root(scan_userdir)
diff --git a/pyproject.toml b/pyproject.toml
index 10c15b9..845e218 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -41,10 +41,10 @@ version = {attr = "pyotb.__version__"}
[tool.pylint]
max-line-length = 120
+max-module-lines = 2000
+good-names = ["x", "y", "i", "j", "k", "e"]
disable = [
"fixme",
- "invalid-name",
- "too-many-lines",
"too-many-locals",
"too-many-branches",
"too-many-statements",
--
GitLab
From a9217fc79e5fcb55bbe17706fb037744881a309a Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 15:34:26 +0100
Subject: [PATCH 087/399] ENH: make time_start and time_end private
---
pyotb/core.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index b488a38..180562b 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -518,7 +518,7 @@ class App(RasterInterface):
self.frozen = frozen
self.quiet = quiet
self.image_dic = image_dic
- self.time_start, self.time_end = 0, 0
+ self._time_start, self._time_end = 0, 0
self.exports_dic = {}
self.parameters = {}
# Initialize app, set parameters and execute if not frozen
@@ -562,7 +562,7 @@ class App(RasterInterface):
@property
def elapsed_time(self) -> float:
"""Get elapsed time between app init and end of exec or file writing."""
- return self.time_end - self.time_start
+ return self._time_end - self._time_start
@property
def used_outputs(self) -> list[str]:
@@ -683,13 +683,13 @@ class App(RasterInterface):
def execute(self):
"""Execute and write to disk if any output parameter has been set during init."""
logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters)
- self.time_start = perf_counter()
+ self._time_start = perf_counter()
try:
self.app.Execute()
except (RuntimeError, FileNotFoundError) as e:
raise Exception(f"{self.name}: error during during app execution") from e
self.frozen = False
- self.time_end = perf_counter()
+ self._time_end = perf_counter()
logger.debug("%s: execution ended", self.name)
self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics
@@ -701,7 +701,7 @@ class App(RasterInterface):
except RuntimeError:
logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name)
self.app.ExecuteAndWriteOutput()
- self.time_end = perf_counter()
+ self._time_end = perf_counter()
def write(self, *args, filename_extension: str = "", pixel_type: dict[str, str] | str = None,
preserve_dtype: bool = False, **kwargs):
--
GitLab
From 86b8cbf9637f35fac6dd3dc4e5880bd244fcd8b0 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 21:16:53 +0100
Subject: [PATCH 088/399] ENH: Input use Path and filepath attr as in Output
---
pyotb/core.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 180562b..e881c48 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1234,15 +1234,15 @@ class Input(App):
path: Anything supported by GDAL (local file on the filesystem, remote resource e.g. /vsicurl/.., etc.)
"""
- self.path = path
- self.name = f"Input from {path}"
super().__init__("ExtractROI", {"in": path}, frozen=True)
+ self.name = f"Input from {path}"
+ self.filepath = Path(path)
self.propagate_dtype()
self.execute()
def __str__(self) -> str:
"""Return a nice string representation with file path."""
- return f"<pyotb.Input object from {self.path}>"
+ return f"<pyotb.Input object from {self.filepath}>"
class Output(RasterInterface):
--
GitLab
From edefdc1f14889c38b764a4fbf4b411f3f3987594 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 26 Jan 2023 21:24:48 +0100
Subject: [PATCH 089/399] FIX: missing exports_dic in Output, add tests
---
pyotb/core.py | 1 +
tests/test_core.py | 7 ++++++-
tests/test_numpy.py | 9 +++++++--
3 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index e881c48..2f7a50c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1261,6 +1261,7 @@ class Output(RasterInterface):
self.name = f"Output {param_key} from {pyotb_app.name}"
self.parent_pyotb_app = pyotb_app # keep trace of parent app
self.app = pyotb_app.app
+ self.exports_dic = pyotb_app.exports_dic
self.param_key = param_key
self.filepath = None
if filepath:
diff --git a/tests/test_core.py b/tests/test_core.py
index ccd386f..848fe15 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -1,4 +1,3 @@
-import os
import pytest
import pyotb
@@ -80,6 +79,12 @@ def test_write():
INPUT.out.filepath.unlink()
+def test_output_write():
+ INPUT.out.write("/tmp/test_output_write.tif")
+ assert INPUT.out.exists()
+ INPUT.out.filepath.unlink()
+
+
# Slicer
def test_slicer_shape():
extract = INPUT[:50, :60, :3]
diff --git a/tests/test_numpy.py b/tests/test_numpy.py
index 682fe2a..d36dfb7 100644
--- a/tests/test_numpy.py
+++ b/tests/test_numpy.py
@@ -5,10 +5,15 @@ from tests_data import INPUT
def test_export():
INPUT.export()
- assert "out" in INPUT.exports_dic
- array = INPUT.exports_dic["out"]["array"]
+ array = INPUT.exports_dic[INPUT.key_output_image]["array"]
assert isinstance(array, np.ndarray)
assert array.dtype == "uint8"
+ del INPUT.exports_dic["out"]
+
+
+def test_output_export():
+ INPUT.out.export()
+ assert INPUT.out.key_output_image in INPUT.out.exports_dic
def test_to_numpy():
--
GitLab
From 35832c6d74acde9f088294c831db57af7131334e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr>
Date: Thu, 2 Feb 2023 09:08:18 +0000
Subject: [PATCH 090/399] WIP: refac classes
---
pyotb/apps.py | 37 +-
pyotb/core.py | 920 +++++++++++++++++++-----------------
pyotb/functions.py | 22 +-
tests/test_core.py | 11 +-
tests/test_numpy.py | 10 +-
tests/test_pipeline.py | 5 +-
tests/test_serialization.py | 14 +-
tests/tests_data.py | 3 +
8 files changed, 505 insertions(+), 517 deletions(-)
create mode 100644 tests/tests_data.py
diff --git a/pyotb/apps.py b/pyotb/apps.py
index ad71f54..9421b28 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -7,7 +7,7 @@ import subprocess
from pathlib import Path
import otbApplication as otb # pylint: disable=import-error
-from .core import OTBObject
+from .core import App
from .helpers import logger
@@ -59,41 +59,6 @@ def get_available_applications(as_subprocess: bool = False) -> list[str]:
return app_list
-class App(OTBObject):
- """Base class for UI related functions, will be subclassed using app name as class name, see CODE_TEMPLATE."""
- _name = ""
-
- def __init__(self, *args, **kwargs):
- """Default App constructor, adds UI specific attributes and functions."""
- super().__init__(*args, **kwargs)
- self.description = self.app.GetDocLongDescription()
-
- @property
- def elapsed_time(self):
- """Get elapsed time between app init and end of exec or file writing."""
- return self.time_end - self.time_start
-
- @property
- def used_outputs(self) -> list[str]:
- """List of used application outputs."""
- return [getattr(self, key) for key in self.out_param_types if key in self.parameters]
-
- def find_outputs(self) -> tuple[str]:
- """Find output files on disk using path found in parameters.
-
- Returns:
- list of files found on disk
-
- """
- files, missing = [], []
- for out in self.used_outputs:
- dest = files if out.exists() else missing
- dest.append(str(out.filepath.absolute()))
- for filename in missing:
- logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
- return tuple(files)
-
-
class OTBTFApp(App):
"""Helper for OTBTF."""
@staticmethod
diff --git a/pyotb/core.py b/pyotb/core.py
index 6f0a4dc..fb07a7b 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1,10 +1,11 @@
# -*- coding: utf-8 -*-
"""This module is the core of pyotb."""
from __future__ import annotations
-from typing import Any
-from pathlib import Path
+
from ast import literal_eval
+from pathlib import Path
from time import perf_counter
+from typing import Any
import numpy as np
import otbApplication as otb # pylint: disable=import-error
@@ -13,50 +14,32 @@ from .helpers import logger
class OTBObject:
- """Base class that gathers common operations for any OTB application."""
+ """Base class for all pyotb objects."""
- def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, **kwargs):
- """Common constructor for OTB applications. Handles in-memory connection between apps.
+ def __init__(self, name: str, app: otb.Application, image_dic: dict = None):
+ """Constructor for an OTBObject.
Args:
- name: name of the app, e.g. 'BandMath'
- *args: used for passing application parameters. Can be :
- - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
- (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
- - string, App or Output, useful when the user wants to specify the input "in"
- - list, useful when the user wants to specify the input list 'il'
- frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___
- quiet: whether to print logs of the OTB app
+ name: name of the object (e.g. "Slicer")
+ app: OTB application instance
image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as
- the result of app.ExportImage(). Use it when the app takes a numpy array as input.
+ the result of self.app.ExportImage(). Use it when the app takes a numpy array as input.
See this related issue for why it is necessary to keep reference of object:
https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824
- **kwargs: used for passing application parameters.
- e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif'
-
"""
- self.parameters = {}
self.name = name
- self.frozen = frozen
- self.quiet = quiet
+ self.app = app
self.image_dic = image_dic
- self.exports_dic = {}
- create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
- self.app = create(name)
+
self.parameters_keys = tuple(self.app.GetParametersKeys())
- self.time_start, self.time_end = 0, 0
self.all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys}
self.out_param_types = {k: v for k, v in self.all_param_types.items()
if v in (otb.ParameterType_OutputImage,
otb.ParameterType_OutputVectorData,
otb.ParameterType_OutputFilename)}
- if args or kwargs:
- self.set_parameters(*args, **kwargs)
- if not self.frozen:
- self.execute()
- if any(key in self.parameters for key in self.out_param_types):
- self.flush() # auto flush if any output param was provided during app init
+
+ self.exports_dic = {}
def get_first_key(self, param_types: list[str]) -> str:
"""Get the first output param key for specific file types."""
@@ -85,17 +68,6 @@ class OTBObject:
"""Get the name of first output image parameter."""
return self.get_first_key(param_types=[otb.ParameterType_OutputImage])
- @property
- def data(self):
- """Expose app's output data values in a dictionary."""
- skip_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem")
- skip_keys = skip_keys + tuple(self.out_param_types) + tuple(self.parameters)
- keys = (k for k in self.parameters_keys if k not in skip_keys)
-
- def _check(v):
- return not isinstance(v, otb.ApplicationProxy) and v not in ("", None, [], ())
- return {str(k): self[k] for k in keys if _check(self[k])}
-
@property
def metadata(self):
"""Return first output image metadata dictionary."""
@@ -125,7 +97,7 @@ class OTBObject:
"""
if not self.key_output_image:
- raise TypeError(f"{self.name}: this application has no raster output")
+ raise TypeError(f"\"{self.name}\" has no raster output")
width, height = self.app.GetImageSize(self.key_output_image)
bands = self.app.GetImageNbBands(self.key_output_image)
return height, width, bands
@@ -145,196 +117,17 @@ class OTBObject:
origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2
return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y
- def set_parameters(self, *args, **kwargs):
- """Set some parameters of the app.
-
- When useful, e.g. for images list, this function appends the parameters
- instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths
-
- Args:
- *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
- (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
- - string, App or Output, useful when the user implicitly wants to set the param "in"
- - list, useful when the user implicitly wants to set the param "il"
- **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif'
-
- Raises:
- Exception: when the setting of a parameter failed
-
- """
- parameters = kwargs
- parameters.update(self.__parse_args(args))
- # Going through all arguments
- for key, obj in parameters.items():
- if key not in self.parameters_keys:
- raise KeyError(f'{self.name}: unknown parameter name "{key}"')
- # When the parameter expects a list, if needed, change the value to list
- if is_key_list(self, key) and not isinstance(obj, (list, tuple)):
- obj = [obj]
- logger.info('%s: argument for parameter "%s" was converted to list', self.name, key)
- try:
- # This is when we actually call self.app.SetParameter*
- self.__set_param(key, obj)
- except (RuntimeError, TypeError, ValueError, KeyError) as e:
- raise Exception(
- f"{self.name}: something went wrong before execution "
- f"(while setting parameter '{key}' to '{obj}')"
- ) from e
- # Update _parameters using values from OtbApplication object
- otb_params = self.app.GetParameters().items()
- otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params}
- # Update param dict and save values as object attributes
- self.parameters.update({**parameters, **otb_params})
- self.save_objects()
-
- def save_objects(self):
- """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`.
-
- This is useful when the key contains reserved characters such as a point eg "io.out"
- """
- for key in self.parameters_keys:
- if key in dir(self.__class__):
- continue # skip forbidden attribute since it is already used by the class
- value = self.parameters.get(key) # basic parameters
- if value is None:
- try:
- value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results)
- except RuntimeError:
- continue # this is when there is no value for key
- # Convert output param path to Output object
- if key in self.out_param_types:
- value = Output(self, key, value)
- elif isinstance(value, str):
- try:
- value = literal_eval(value)
- except (ValueError, SyntaxError):
- pass
- # Save attribute
- setattr(self, key, value)
-
- def execute(self):
- """Execute and write to disk if any output parameter has been set during init."""
- logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters)
- self.time_start = perf_counter()
- try:
- self.app.Execute()
- except (RuntimeError, FileNotFoundError) as e:
- raise Exception(f"{self.name}: error during during app execution") from e
- self.frozen = False
- self.time_end = perf_counter()
- logger.debug("%s: execution ended", self.name)
- self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics
-
- def flush(self):
- """Flush data to disk, this is when WriteOutput is actually called."""
- try:
- logger.debug("%s: flushing data to disk", self.name)
- self.app.WriteOutput()
- except RuntimeError:
- logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name)
- self.app.ExecuteAndWriteOutput()
- self.time_end = perf_counter()
-
- def write(self, *args, filename_extension: str = "", pixel_type: dict[str, str] | str = None,
- preserve_dtype: bool = False, **kwargs):
- """Set output pixel type and write the output raster files.
-
- Args:
- *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key contains
- non-standard characters such as a point, e.g. {'io.out':'output.tif'}
- - string, useful when there is only one output, e.g. 'output.tif'
- - None if output file was passed during App init
- filename_extension: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES")
- Will be used for all outputs (Default value = "")
- pixel_type: Can be : - dictionary {output_parameter_key: pixeltype} when specifying for several outputs
- - str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several
- outputs, all outputs are written with this unique type.
- Valid pixel types are uint8, uint16, uint32, int16, int32, float, double,
- cint16, cint32, cfloat, cdouble. (Default value = None)
- preserve_dtype: propagate main input pixel type to outputs, in case pixel_type is None
- **kwargs: keyword arguments e.g. out='output.tif'
-
- """
- # Gather all input arguments in kwargs dict
- for arg in args:
- if isinstance(arg, dict):
- kwargs.update(arg)
- elif isinstance(arg, str) and kwargs:
- logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg)
- elif isinstance(arg, str) and self.key_output_image:
- kwargs.update({self.key_output_image: arg})
-
- # Append filename extension to filenames
- if filename_extension:
- logger.debug("%s: using extended filename for outputs: %s", self.name, filename_extension)
- if not filename_extension.startswith("?"):
- filename_extension = "?" + filename_extension
- for key, value in kwargs.items():
- if self.out_param_types[key] == otb.ParameterType_OutputImage and '?' not in value:
- kwargs[key] = value + filename_extension
-
- # Manage output pixel types
- dtypes = {}
- if pixel_type:
- if isinstance(pixel_type, str):
- type_name = self.app.ConvertPixelTypeToNumpy(parse_pixel_type(pixel_type))
- logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name)
- for key in kwargs:
- if self.out_param_types[key] == otb.ParameterType_OutputImage:
- dtypes[key] = parse_pixel_type(pixel_type)
- elif isinstance(pixel_type, dict):
- dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()}
- elif preserve_dtype:
- self.propagate_dtype() # all outputs will have the same type as the main input raster
-
- # Set parameters and flush to disk
- for key, output_filename in kwargs.items():
- if key in dtypes:
- self.propagate_dtype(key, dtypes[key])
- self.set_parameters({key: output_filename})
- self.flush()
-
- def propagate_dtype(self, target_key: str = None, dtype: int = None):
- """Propagate a pixel type from main input to every outputs, or to a target output key only.
-
- With multiple inputs (if dtype is not provided), the type of the first input is considered.
- With multiple outputs (if target_key is not provided), all outputs will be converted to the same pixel type.
-
- Args:
- target_key: output param key to change pixel type
- dtype: data type to use
-
- """
- if not dtype:
- param = self.parameters.get(self.key_input_image)
- if not param:
- logger.warning("%s: could not propagate pixel type from inputs to output", self.name)
- return
- if isinstance(param, (list, tuple)):
- param = param[0] # first image in "il"
- try:
- dtype = get_pixel_type(param)
- except (TypeError, RuntimeError):
- logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param)
- return
- if target_key:
- keys = [target_key]
- else:
- keys = [k for k, v in self.out_param_types.items() if v == otb.ParameterType_OutputImage]
- for key in keys:
- self.app.SetParameterOutputImagePixelType(key, dtype)
-
def get_infos(self):
"""Return a dict output of ReadImageInfo for the first image output."""
if not self.key_output_image:
raise TypeError(f"{self.name}: this application has no raster output")
- return OTBObject("ReadImageInfo", self, quiet=True).data
+ return App("ReadImageInfo", self, quiet=True).data
def get_statistics(self):
"""Return a dict output of ComputeImagesStatistics for the first image output."""
if not self.key_output_image:
raise TypeError(f"{self.name}: this application has no raster output")
- return OTBObject("ComputeImagesStatistics", self, quiet=True).data
+ return App("ComputeImagesStatistics", self, quiet=True).data
def read_values_at_coords(self, row: int, col: int, bands: int = None) -> list[int | float] | int | float:
"""Get pixel value(s) at a given YX coordinates.
@@ -349,7 +142,7 @@ class OTBObject:
"""
channels = []
- app = OTBObject("PixelValue", self, coordx=col, coordy=row, frozen=True, quiet=True)
+ app = App("PixelValue", self, coordx=col, coordy=row, frozen=True, quiet=True)
if bands is not None:
if isinstance(bands, int):
if bands < 0:
@@ -385,22 +178,6 @@ class OTBObject:
return list(range(0, nb_channels, step))
raise ValueError(f"{self.name}: '{bands}' cannot be interpreted as valid slicing.")
- def summarize(self) -> dict:
- """Serialize an object and its pipeline into a dictionary.
-
- Returns:
- nested dictionary summarizing the pipeline
-
- """
- params = self.parameters
- for k, p in params.items():
- # In the following, we replace each parameter which is an OTBObject, with its summary.
- if isinstance(p, OTBObject): # single parameter
- params[k] = p.summarize()
- elif isinstance(p, list): # parameter list
- params[k] = [pi.summarize() if isinstance(pi, OTBObject) else pi for pi in p]
- return {"name": self.name, "parameters": params}
-
def export(self, key: str = None, preserve_dtype: bool = True) -> dict[str, dict[str, np.ndarray]]:
"""Export a specific output image as numpy array and store it in object exports_dic.
@@ -473,58 +250,6 @@ class OTBObject:
row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x
return abs(int(row)), int(col)
- # Private functions
- def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]:
- """Gather all input arguments in kwargs dict.
-
- Args:
- args: the list of arguments passed to set_parameters()
-
- Returns:
- a dictionary with the right keyword depending on the object
-
- """
- kwargs = {}
- for arg in args:
- if isinstance(arg, dict):
- kwargs.update(arg)
- elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.key_input):
- kwargs.update({self.key_input: arg})
- return kwargs
-
- def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]):
- """Set one parameter, decide which otb.Application method to use depending on target object."""
- if obj is None or (isinstance(obj, (list, tuple)) and not obj):
- self.app.ClearValue(key)
- return
- if key not in self.parameters_keys:
- raise KeyError(
- f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}"
- )
- # Single-parameter cases
- if isinstance(obj, OTBObject):
- self.app.ConnectImage(key, obj.app, obj.key_output_image)
- elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB
- self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0])
- elif key == "ram": # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200
- self.app.SetParameterInt("ram", int(obj))
- elif not isinstance(obj, list): # any other parameters (str, int...)
- self.app.SetParameterValue(key, obj)
- # Images list
- elif is_key_images_list(self, key):
- # To enable possible in-memory connections, we go through the list and set the parameters one by one
- for inp in obj:
- if isinstance(inp, OTBObject):
- self.app.ConnectImage(key, inp.app, inp.key_output_image)
- elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB
- self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0])
- else: # here `input` should be an image filepath
- # Append `input` to the list, do not overwrite any previously set element of the image list
- self.app.AddParameterStringList(key, inp)
- # List of any other types (str, int...)
- else:
- self.app.SetParameterValue(key, obj)
-
# Special functions
def __hash__(self):
"""Override the default behaviour of the hash function.
@@ -535,31 +260,6 @@ class OTBObject:
"""
return id(self)
- def __str__(self):
- """Return a nice string representation with object id."""
- return f"<pyotb.App {self.name} object id {id(self)}>"
-
- def __getattr__(self, name):
- """This method is called when the default attribute access fails.
-
- We choose to access the attribute `name` of self.app.
- Thus, any method of otbApplication can be used transparently on OTBObject objects,
- e.g. SetParameterOutputImagePixelType() or ExportImage() work
-
- Args:
- name: attribute name
-
- Returns:
- attribute
-
- Raises:
- AttributeError: when `name` is not an attribute of self.app
-
- """
- if name in dir(self.app):
- return getattr(self.app, name)
- raise AttributeError(f"{self.name}: could not find attribute `{name}`")
-
def __getitem__(self, key):
"""Override the default __getitem__ behaviour.
@@ -598,7 +298,7 @@ class OTBObject:
key = key + (slice(None, None, None),)
return Slicer(self, *key)
- def __add__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __add__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default addition and flavours it with BandMathX.
Args:
@@ -612,7 +312,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("+", self, other)
- def __sub__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __sub__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default subtraction and flavours it with BandMathX.
Args:
@@ -626,7 +326,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("-", self, other)
- def __mul__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __mul__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default subtraction and flavours it with BandMathX.
Args:
@@ -640,7 +340,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("*", self, other)
- def __truediv__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __truediv__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default subtraction and flavours it with BandMathX.
Args:
@@ -654,7 +354,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("/", self, other)
- def __radd__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __radd__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default reverse addition and flavours it with BandMathX.
Args:
@@ -668,7 +368,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("+", other, self)
- def __rsub__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __rsub__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default subtraction and flavours it with BandMathX.
Args:
@@ -682,7 +382,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("-", other, self)
- def __rmul__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __rmul__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default multiplication and flavours it with BandMathX.
Args:
@@ -696,7 +396,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return Operation("*", other, self)
- def __rtruediv__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __rtruediv__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default division and flavours it with BandMathX.
Args:
@@ -719,7 +419,7 @@ class OTBObject:
"""
return Operation("abs", self)
- def __ge__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __ge__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default greater or equal and flavours it with BandMathX.
Args:
@@ -733,7 +433,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation(">=", self, other)
- def __le__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __le__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default less or equal and flavours it with BandMathX.
Args:
@@ -747,7 +447,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation("<=", self, other)
- def __gt__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __gt__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default greater operator and flavours it with BandMathX.
Args:
@@ -761,7 +461,7 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation(">", self, other)
- def __lt__(self, other: OTBObject | Output | str | int | float) -> Operation:
+ def __lt__(self, other: OTBObject | str | int | float) -> Operation:
"""Overrides the default less operator and flavours it with BandMathX.
Args:
@@ -775,125 +475,457 @@ class OTBObject:
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return LogicalOperation("<", self, other)
- def __eq__(self, other: OTBObject | Output | str | int | float) -> Operation:
- """Overrides the default eq operator and flavours it with BandMathX.
+ def __eq__(self, other: OTBObject | str | int | float) -> Operation:
+ """Overrides the default eq operator and flavours it with BandMathX.
+
+ Args:
+ other: the other member of the operation
+
+ Returns:
+ self == other
+
+ """
+ if isinstance(other, (np.ndarray, np.generic)):
+ return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
+ return LogicalOperation("==", self, other)
+
+ def __ne__(self, other: OTBObject | str | int | float) -> Operation:
+ """Overrides the default different operator and flavours it with BandMathX.
+
+ Args:
+ other: the other member of the operation
+
+ Returns:
+ self != other
+
+ """
+ if isinstance(other, (np.ndarray, np.generic)):
+ return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
+ return LogicalOperation("!=", self, other)
+
+ def __or__(self, other: OTBObject | str | int | float) -> Operation:
+ """Overrides the default or operator and flavours it with BandMathX.
+
+ Args:
+ other: the other member of the operation
+
+ Returns:
+ self || other
+
+ """
+ if isinstance(other, (np.ndarray, np.generic)):
+ return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
+ return LogicalOperation("||", self, other)
+
+ def __and__(self, other: OTBObject | str | int | float) -> Operation:
+ """Overrides the default and operator and flavours it with BandMathX.
+
+ Args:
+ other: the other member of the operation
+
+ Returns:
+ self && other
+
+ """
+ if isinstance(other, (np.ndarray, np.generic)):
+ return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
+ return LogicalOperation("&&", self, other)
+
+ # TODO: other operations ?
+ # e.g. __pow__... cf https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types
+
+ def __array__(self) -> np.ndarray:
+ """This is called when running np.asarray(pyotb_object).
+
+ Returns:
+ a numpy array
+
+ """
+ return self.to_numpy()
+
+ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs) -> OTBObject:
+ """This is called whenever a numpy function is called on a pyotb object.
+
+ Operation is performed in numpy, then imported back to pyotb with the same georeference as input.
+
+ Args:
+ ufunc: numpy function
+ method: an internal numpy argument
+ inputs: inputs, at least one being pyotb object. If there are several pyotb objects, they must all have
+ the same georeference and pixel size.
+ **kwargs: kwargs of the numpy function
+
+ Returns:
+ a pyotb object
+
+ """
+ if method == "__call__":
+ # Converting potential pyotb inputs to arrays
+ arrays = []
+ image_dic = None
+ for inp in inputs:
+ if isinstance(inp, (float, int, np.ndarray, np.generic)):
+ arrays.append(inp)
+ elif isinstance(inp, OTBObject):
+ if not inp.exports_dic:
+ inp.export()
+ image_dic = inp.exports_dic[inp.key_output_image]
+ array = image_dic["array"]
+ arrays.append(array)
+ else:
+ logger.debug(type(self))
+ return NotImplemented
+ # Performing the numpy operation
+ result_array = ufunc(*arrays, **kwargs)
+ result_dic = image_dic
+ result_dic["array"] = result_array
+ # Importing back to OTB, pass the result_dic just to keep reference
+ pyotb_app = App("ExtractROI", image_dic=result_dic, frozen=True, quiet=True)
+ if result_array.shape[2] == 1:
+ pyotb_app.app.ImportImage("in", result_dic)
+ else:
+ pyotb_app.app.ImportVectorImage("in", result_dic)
+ pyotb_app.execute()
+ return pyotb_app
+ return NotImplemented
+
+
+class App(OTBObject):
+ """Base class that gathers common operations for any OTB application."""
+
+ def __init__(self, otb_app_name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None,
+ name: str = None, **kwargs):
+ """Common constructor for OTB applications. Handles in-memory connection between apps.
+
+ Args:
+ otb_app_name: name of the OTB application, e.g. 'BandMath'
+ *args: used for passing application parameters. Can be :
+ - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
+ (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
+ - string or OTBObject, useful when the user wants to specify the input "in"
+ - list, useful when the user wants to specify the input list 'il'
+ frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___
+ quiet: whether to print logs of the OTB app
+ image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as
+ the result of app.ExportImage(). Use it when the app takes a numpy array as input.
+ See this related issue for why it is necessary to keep reference of object:
+ https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824
+ name: override the application name
+ **kwargs: used for passing application parameters.
+ e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif'
+
+ """
+ self.frozen = frozen
+ self.quiet = quiet
+ create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
+ super().__init__(name=name or f"OTB Application {otb_app_name}", app=create(otb_app_name), image_dic=image_dic)
+ self.description = self.app.GetDocLongDescription()
+
+ # Set parameters
+ self.parameters = {}
+ if args or kwargs:
+ self.set_parameters(*args, **kwargs)
+ if not self.frozen:
+ self.execute()
+ if any(key in self.parameters for key in self.out_param_types):
+ self.flush() # auto flush if any output param was provided during app init
+
+ # Elapsed time
+ self.time_start, self.time_end = 0, 0
+
+ @property
+ def elapsed_time(self):
+ """Get elapsed time between app init and end of exec or file writing."""
+ return self.time_end - self.time_start
+
+ @property
+ def used_outputs(self) -> list[str]:
+ """List of used application outputs."""
+ return [getattr(self, key) for key in self.out_param_types if key in self.parameters]
+
+ def find_outputs(self) -> tuple[str]:
+ """Find output files on disk using path found in parameters.
+
+ Returns:
+ list of files found on disk
+
+ """
+ files, missing = [], []
+ for out in self.used_outputs:
+ dest = files if out.exists() else missing
+ dest.append(str(out.filepath.absolute()))
+ for filename in missing:
+ logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
+ return tuple(files)
+
+ @property
+ def data(self):
+ """Expose app's output data values in a dictionary."""
+ skip_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem")
+ skip_keys = skip_keys + tuple(self.out_param_types) + tuple(self.parameters)
+ keys = (k for k in self.parameters_keys if k not in skip_keys)
+
+ def _check(v):
+ return not isinstance(v, otb.ApplicationProxy) and v not in ("", None, [], ())
+
+ return {str(k): self[k] for k in keys if _check(self[k])}
+
+ def set_parameters(self, *args, **kwargs):
+ """Set some parameters of the app.
+
+ When useful, e.g. for images list, this function appends the parameters
+ instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths
+
+ Args:
+ *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
+ (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
+ - string or OTBObject, useful when the user implicitly wants to set the param "in"
+ - list, useful when the user implicitly wants to set the param "il"
+ **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif'
+
+ Raises:
+ Exception: when the setting of a parameter failed
+
+ """
+ parameters = kwargs
+ parameters.update(self.__parse_args(args))
+ # Going through all arguments
+ for key, obj in parameters.items():
+ if key not in self.parameters_keys:
+ raise KeyError(f'{self.name}: unknown parameter name "{key}"')
+ # When the parameter expects a list, if needed, change the value to list
+ if is_key_list(self, key) and not isinstance(obj, (list, tuple)):
+ obj = [obj]
+ logger.info('%s: argument for parameter "%s" was converted to list', self.name, key)
+ try:
+ # This is when we actually call self.app.SetParameter*
+ self.__set_param(key, obj)
+ except (RuntimeError, TypeError, ValueError, KeyError) as e:
+ raise Exception(
+ f"{self.name}: something went wrong before execution "
+ f"(while setting parameter '{key}' to '{obj}')"
+ ) from e
+ # Update _parameters using values from OtbApplication object
+ otb_params = self.app.GetParameters().items()
+ otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params}
+ # Update param dict and save values as object attributes
+ self.parameters.update({**parameters, **otb_params})
+ self.save_objects()
+
+ def save_objects(self):
+ """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`.
- Args:
- other: the other member of the operation
+ This is useful when the key contains reserved characters such as a point eg "io.out"
+ """
+ for key in self.parameters_keys:
+ if key in dir(self.__class__):
+ continue # skip forbidden attribute since it is already used by the class
+ value = self.parameters.get(key) # basic parameters
+ if value is None:
+ try:
+ value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results)
+ except RuntimeError:
+ continue # this is when there is no value for key
+ # Convert output param path to Output object
+ if key in self.out_param_types:
+ value = Output(self, key, value)
+ elif isinstance(value, str):
+ try:
+ value = literal_eval(value)
+ except (ValueError, SyntaxError):
+ pass
+ # Save attribute
+ setattr(self, key, value)
- Returns:
- self == other
+ def execute(self):
+ """Execute and write to disk if any output parameter has been set during init."""
+ logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters)
+ self.time_start = perf_counter()
+ try:
+ self.app.Execute()
+ except (RuntimeError, FileNotFoundError) as e:
+ raise Exception(f"{self.name}: error during during app execution") from e
+ self.frozen = False
+ self.time_end = perf_counter()
+ logger.debug("%s: execution ended", self.name)
+ self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics
- """
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return LogicalOperation("==", self, other)
+ def flush(self):
+ """Flush data to disk, this is when WriteOutput is actually called."""
+ try:
+ logger.debug("%s: flushing data to disk", self.name)
+ self.app.WriteOutput()
+ except RuntimeError:
+ logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name)
+ self.app.ExecuteAndWriteOutput()
+ self.time_end = perf_counter()
- def __ne__(self, other: OTBObject | Output | str | int | float) -> Operation:
- """Overrides the default different operator and flavours it with BandMathX.
+ def write(self, *args, filename_extension: str = "", pixel_type: dict[str, str] | str = None,
+ preserve_dtype: bool = False, **kwargs):
+ """Set output pixel type and write the output raster files.
Args:
- other: the other member of the operation
-
- Returns:
- self != other
+ *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key contains
+ non-standard characters such as a point, e.g. {'io.out':'output.tif'}
+ - string, useful when there is only one output, e.g. 'output.tif'
+ - None if output file was passed during App init
+ filename_extension: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES")
+ Will be used for all outputs (Default value = "")
+ pixel_type: Can be : - dictionary {output_parameter_key: pixeltype} when specifying for several outputs
+ - str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several
+ outputs, all outputs are written with this unique type.
+ Valid pixel types are uint8, uint16, uint32, int16, int32, float, double,
+ cint16, cint32, cfloat, cdouble. (Default value = None)
+ preserve_dtype: propagate main input pixel type to outputs, in case pixel_type is None
+ **kwargs: keyword arguments e.g. out='output.tif'
"""
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return LogicalOperation("!=", self, other)
+ # Gather all input arguments in kwargs dict
+ for arg in args:
+ if isinstance(arg, dict):
+ kwargs.update(arg)
+ elif isinstance(arg, str) and kwargs:
+ logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg)
+ elif isinstance(arg, str) and self.key_output_image:
+ kwargs.update({self.key_output_image: arg})
- def __or__(self, other: OTBObject | Output | str | int | float) -> Operation:
- """Overrides the default or operator and flavours it with BandMathX.
+ # Append filename extension to filenames
+ if filename_extension:
+ logger.debug("%s: using extended filename for outputs: %s", self.name, filename_extension)
+ if not filename_extension.startswith("?"):
+ filename_extension = "?" + filename_extension
+ for key, value in kwargs.items():
+ if self.out_param_types[key] == otb.ParameterType_OutputImage and '?' not in value:
+ kwargs[key] = value + filename_extension
- Args:
- other: the other member of the operation
+ # Manage output pixel types
+ dtypes = {}
+ if pixel_type:
+ if isinstance(pixel_type, str):
+ type_name = self.app.ConvertPixelTypeToNumpy(parse_pixel_type(pixel_type))
+ logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name)
+ for key in kwargs:
+ if self.out_param_types[key] == otb.ParameterType_OutputImage:
+ dtypes[key] = parse_pixel_type(pixel_type)
+ elif isinstance(pixel_type, dict):
+ dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()}
+ elif preserve_dtype:
+ self.propagate_dtype() # all outputs will have the same type as the main input raster
- Returns:
- self || other
+ # Set parameters and flush to disk
+ for key, output_filename in kwargs.items():
+ if key in dtypes:
+ self.propagate_dtype(key, dtypes[key])
+ self.set_parameters({key: output_filename})
+ self.flush()
- """
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return LogicalOperation("||", self, other)
+ def propagate_dtype(self, target_key: str = None, dtype: int = None):
+ """Propagate a pixel type from main input to every outputs, or to a target output key only.
- def __and__(self, other: OTBObject | Output | str | int | float) -> Operation:
- """Overrides the default and operator and flavours it with BandMathX.
+ With multiple inputs (if dtype is not provided), the type of the first input is considered.
+ With multiple outputs (if target_key is not provided), all outputs will be converted to the same pixel type.
Args:
- other: the other member of the operation
-
- Returns:
- self && other
+ target_key: output param key to change pixel type
+ dtype: data type to use
"""
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return LogicalOperation("&&", self, other)
+ if not dtype:
+ param = self.parameters.get(self.key_input_image)
+ if not param:
+ logger.warning("%s: could not propagate pixel type from inputs to output", self.name)
+ return
+ if isinstance(param, (list, tuple)):
+ param = param[0] # first image in "il"
+ try:
+ dtype = get_pixel_type(param)
+ except (TypeError, RuntimeError):
+ logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param)
+ return
+ if target_key:
+ keys = [target_key]
+ else:
+ keys = [k for k, v in self.out_param_types.items() if v == otb.ParameterType_OutputImage]
+ for key in keys:
+ self.app.SetParameterOutputImagePixelType(key, dtype)
- # TODO: other operations ?
- # e.g. __pow__... cf https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types
+ # Private functions
+ def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]:
+ """Gather all input arguments in kwargs dict.
- def __array__(self) -> np.ndarray:
- """This is called when running np.asarray(pyotb_object).
+ Args:
+ args: the list of arguments passed to set_parameters()
Returns:
- a numpy array
+ a dictionary with the right keyword depending on the object
"""
- return self.to_numpy()
-
- def __array_ufunc__(self, ufunc, method, *inputs, **kwargs) -> OTBObject:
- """This is called whenever a numpy function is called on a pyotb object.
+ kwargs = {}
+ for arg in args:
+ if isinstance(arg, dict):
+ kwargs.update(arg)
+ elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.key_input):
+ kwargs.update({self.key_input: arg})
+ return kwargs
- Operation is performed in numpy, then imported back to pyotb with the same georeference as input.
+ def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]):
+ """Set one parameter, decide which otb.Application method to use depending on target object."""
+ if obj is None or (isinstance(obj, (list, tuple)) and not obj):
+ self.app.ClearValue(key)
+ return
+ if key not in self.parameters_keys:
+ raise KeyError(
+ f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}"
+ )
+ # Single-parameter cases
+ if isinstance(obj, OTBObject):
+ self.app.ConnectImage(key, obj.app, obj.key_output_image)
+ elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB
+ self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0])
+ elif key == "ram": # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200
+ self.app.SetParameterInt("ram", int(obj))
+ elif not isinstance(obj, list): # any other parameters (str, int...)
+ self.app.SetParameterValue(key, obj)
+ # Images list
+ elif is_key_images_list(self, key):
+ # To enable possible in-memory connections, we go through the list and set the parameters one by one
+ for inp in obj:
+ if isinstance(inp, OTBObject):
+ self.app.ConnectImage(key, inp.app, inp.key_output_image)
+ elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB
+ self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0])
+ else: # here `input` should be an image filepath
+ # Append `input` to the list, do not overwrite any previously set element of the image list
+ self.app.AddParameterStringList(key, inp)
+ # List of any other types (str, int...)
+ else:
+ self.app.SetParameterValue(key, obj)
- Args:
- ufunc: numpy function
- method: an internal numpy argument
- inputs: inputs, at least one being pyotb object. If there are several pyotb objects, they must all have
- the same georeference and pixel size.
- **kwargs: kwargs of the numpy function
+ def summarize(self) -> dict:
+ """Serialize an object and its pipeline into a dictionary.
Returns:
- a pyotb object
+ nested dictionary summarizing the pipeline
"""
- if method == "__call__":
- # Converting potential pyotb inputs to arrays
- arrays = []
- image_dic = None
- for inp in inputs:
- if isinstance(inp, (float, int, np.ndarray, np.generic)):
- arrays.append(inp)
- elif isinstance(inp, OTBObject):
- if not inp.exports_dic:
- inp.export()
- image_dic = inp.exports_dic[inp.key_output_image]
- array = image_dic["array"]
- arrays.append(array)
- else:
- logger.debug(type(self))
- return NotImplemented
- # Performing the numpy operation
- result_array = ufunc(*arrays, **kwargs)
- result_dic = image_dic
- result_dic["array"] = result_array
- # Importing back to OTB, pass the result_dic just to keep reference
- app = OTBObject("ExtractROI", image_dic=result_dic, frozen=True, quiet=True)
- if result_array.shape[2] == 1:
- app.ImportImage("in", result_dic)
- else:
- app.ImportVectorImage("in", result_dic)
- app.execute()
- return app
- return NotImplemented
+ params = self.parameters
+ for k, p in params.items():
+ # In the following, we replace each parameter which is an OTBObject, with its summary.
+ if isinstance(p, App): # single parameter
+ params[k] = p.summarize()
+ elif isinstance(p, list): # parameter list
+ params[k] = [pi.summarize() if isinstance(pi, App) else pi for pi in p]
+ return {"name": self.app.GetName(), "parameters": params}
+
+ def __str__(self):
+ """Return a nice string representation with object id."""
+ return f"<pyotb.App {self.name} object id {id(self)}>"
-class Slicer(OTBObject):
+class Slicer(App):
"""Slicer objects i.e. when we call something like raster[:, :, 2] from Python."""
- def __init__(self, obj: OTBObject | Output | str, rows: int, cols: int, channels: int):
+ def __init__(self, obj: OTBObject | str, rows: int, cols: int, channels: int):
"""Create a slicer object, that can be used directly for writing or inside a BandMath.
It contains :
@@ -907,8 +939,7 @@ class Slicer(OTBObject):
channels: channels, can be slicing, list or int
"""
- super().__init__("ExtractROI", {"in": obj, "mode": "extent"}, quiet=True, frozen=True)
- self.name = "Slicer"
+ super().__init__("ExtractROI", {"in": obj, "mode": "extent"}, quiet=True, frozen=True, name="Slicer")
self.rows, self.cols = rows, cols
parameters = {}
@@ -957,7 +988,7 @@ class Slicer(OTBObject):
self.execute()
-class Operation(OTBObject):
+class Operation(App):
"""Class for arithmetic/math operations done in Python.
Example:
@@ -978,7 +1009,7 @@ class Operation(OTBObject):
"""
- def __init__(self, operator: str, *inputs, nb_bands: int = None):
+ def __init__(self, operator: str, *inputs, nb_bands: int = None, name: str = None):
"""Given some inputs and an operator, this function enables to transform this into an OTB application.
Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator.
@@ -986,8 +1017,9 @@ class Operation(OTBObject):
Args:
operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
- *inputs: inputs. Can be App, Output, Input, Operation, Slicer, filepath, int or float
+ *inputs: inputs. Can be OTBObject, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
+ name: override the Operation name
"""
self.operator = operator
@@ -1014,11 +1046,10 @@ class Operation(OTBObject):
self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)]
self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands)
# Execute app
- name = "BandMath" if len(self.exp_bands) == 1 else "BandMathX"
- super().__init__(name, il=self.unique_inputs, exp=self.exp, quiet=True)
- self.name = f'Operation exp="{self.exp}"'
+ super().__init__("BandMath" if len(self.exp_bands) == 1 else "BandMathX", il=self.unique_inputs,
+ exp=self.exp, quiet=True, name=name or f'Operation exp="{self.exp}"')
- def create_fake_exp(self, operator: str, inputs: list[OTBObject | Output | str | int | float],
+ def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float],
nb_bands: int = None):
"""Create a 'fake' expression.
@@ -1026,7 +1057,7 @@ class Operation(OTBObject):
Args:
operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
- inputs: inputs. Can be App, Output, Input, Operation, Slicer, filepath, int or float
+ inputs: inputs. Can be OTBObject, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
@@ -1110,7 +1141,7 @@ class Operation(OTBObject):
return exp_bands, exp
@staticmethod
- def create_one_input_fake_exp(x: OTBObject | Output | str,
+ def create_one_input_fake_exp(x: OTBObject | str,
band: int, keep_logical: bool = False) -> tuple(str, list[OTBObject], int):
"""This an internal function, only to be used by `create_fake_exp`.
@@ -1190,19 +1221,18 @@ class LogicalOperation(Operation):
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
- super().__init__(operator, *inputs, nb_bands=nb_bands)
+ super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation")
self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands)
- def create_fake_exp(self, operator: str, inputs: list[OTBObject | Output | str | int | float],
- nb_bands: int = None):
+ def create_fake_exp(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None):
"""Create a 'fake' expression.
- E.g for the operation input1 > input2, we create a fake expression that is like
+ e.g for the operation input1 > input2, we create a fake expression that is like
"str(input1) > str(input2) ? 1 : 0" and a logical fake expression that is like "str(input1) > str(input2)"
Args:
operator: str (one of >, <, >=, <=, ==, !=, &, |)
- inputs: Can be App, Output, Input, Operation, Slicer, filepath, int or float
+ inputs: Can be OTBObject, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
@@ -1237,7 +1267,7 @@ class LogicalOperation(Operation):
self.fake_exp_bands.append(fake_exp)
-class Input(OTBObject):
+class Input(App):
"""Class for transforming a filepath to pyOTB object."""
def __init__(self, path: str):
@@ -1248,8 +1278,7 @@ class Input(OTBObject):
"""
self.path = path
- super().__init__("ExtractROI", {"in": path}, frozen=True)
- self.name = f"Input from {path}"
+ super().__init__("ExtractROI", {"in": path}, frozen=True, name=f"Input from {path}")
self.propagate_dtype()
self.execute()
@@ -1262,7 +1291,7 @@ class Output(OTBObject):
"""Object that behave like a pointer to a specific application output file."""
def __init__(self, pyotb_app: OTBObject, # pylint: disable=super-init-not-called
- param_key: str, filepath: str = None, mkdir: bool = True):
+ param_key: str = None, filepath: str = None, mkdir: bool = True):
"""Constructor for an Output object.
Args:
@@ -1272,9 +1301,9 @@ class Output(OTBObject):
mkdir: create missing parent directories
"""
- self.pyotb_app, self.app = pyotb_app, pyotb_app.app
- self.parameters = pyotb_app.parameters
- self.param_key = param_key
+ super().__init__(name=f"Output {param_key} from {pyotb_app.name}", app=pyotb_app.app)
+ self.parent_pyotb_app = pyotb_app # keep trace of parent app
+ self.param_key = param_key or super().key_output_image
self.filepath = None
if filepath:
if '?' in filepath:
@@ -1282,7 +1311,6 @@ class Output(OTBObject):
self.filepath = Path(filepath)
if mkdir:
self.make_parent_dirs()
- self.name = f"Output {param_key} from {self.pyotb_app.name}"
@property
def key_output_image(self):
@@ -1291,16 +1319,18 @@ class Output(OTBObject):
def exists(self) -> bool:
"""Check file exist."""
+ assert self.filepath, "Filepath not set"
return self.filepath.exists()
def make_parent_dirs(self):
"""Create missing parent directories."""
+ assert self.filepath, "Filepath not set"
if not self.filepath.parent.exists():
self.filepath.parent.mkdir(parents=True)
def __str__(self) -> str:
"""Return a nice string representation with source app name and object id."""
- return f"<pyotb.Output {self.source_app.name} object, id {id(self)}>"
+ return f"<pyotb.Output {self.name} object, id {id(self)}>"
def get_nbchannels(inp: str | OTBObject) -> int:
@@ -1318,8 +1348,8 @@ def get_nbchannels(inp: str | OTBObject) -> int:
else:
# Executing the app, without printing its log
try:
- info = OTBObject("ReadImageInfo", inp, quiet=True)
- nb_channels = info.GetParameterInt("numberbands")
+ info = App("ReadImageInfo", inp, quiet=True)
+ nb_channels = info.app.GetParameterInt("numberbands")
except Exception as e: # this happens when we pass a str that is not a filepath
raise TypeError(f'Could not get the number of channels of `{inp}`. Not a filepath or wrong filepath') from e
return nb_channels
@@ -1338,10 +1368,10 @@ def get_pixel_type(inp: str | OTBObject) -> str:
"""
if isinstance(inp, str):
try:
- info = OTBObject("ReadImageInfo", inp, quiet=True)
+ info = App("ReadImageInfo", inp, quiet=True)
except Exception as info_err: # this happens when we pass a str that is not a filepath
raise TypeError(f"Could not get the pixel type of `{inp}`. Not a filepath or wrong filepath") from info_err
- datatype = info.GetParameterString("datatype") # which is such as short, float...
+ datatype = info.app.GetParameterString("datatype") # which is such as short, float...
if not datatype:
raise TypeError(f"Unable to read pixel type of image {inp}")
datatype_to_pixeltype = {
@@ -1359,7 +1389,7 @@ def get_pixel_type(inp: str | OTBObject) -> str:
raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}")
pixel_type = getattr(otb, f'ImagePixelType_{datatype_to_pixeltype[datatype]}')
elif isinstance(inp, OTBObject):
- pixel_type = inp.GetParameterOutputImagePixelType(inp.key_output_image)
+ pixel_type = inp.app.GetParameterOutputImagePixelType(inp.key_output_image)
else:
raise TypeError(f'Could not get the pixel type of {type(inp)} object {inp}')
return pixel_type
diff --git a/pyotb/functions.py b/pyotb/functions.py
index 71c298b..d25610a 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -9,12 +9,11 @@ import textwrap
import subprocess
from collections import Counter
-from .core import OTBObject, Input, Operation, LogicalOperation, get_nbchannels, Output
+from .core import OTBObject, App, Operation, LogicalOperation, Input, get_nbchannels
from .helpers import logger
-def where(cond: OTBObject | Output | str, x: OTBObject | Output | str | int | float,
- y: OTBObject | Output | str | int | float) -> Operation:
+def where(cond: OTBObject | str, x: OTBObject | str | int | float, y: OTBObject | str | int | float) -> Operation:
"""Functionally similar to numpy.where. Where cond is True (!=0), returns x. Else returns y.
Args:
@@ -64,8 +63,7 @@ def where(cond: OTBObject | Output | str, x: OTBObject | Output | str | int | fl
return operation
-def clip(a: OTBObject | Output | str, a_min: OTBObject | Output | str | int | float,
- a_max: OTBObject | Output | str | int | float):
+def clip(a: OTBObject | str, a_min: OTBObject | str | int | float, a_max: OTBObject | str | int | float):
"""Clip values of image in a range of values.
Args:
@@ -355,9 +353,9 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
metadatas = {}
for inp in inputs:
if isinstance(inp, str): # this is for filepaths
- metadata = Input(inp).GetImageMetaData('out')
+ metadata = Input(inp).app.GetImageMetaData('out')
elif isinstance(inp, OTBObject):
- metadata = inp.GetImageMetaData(inp.output_param)
+ metadata = inp.app.GetImageMetaData(inp.output_param)
else:
raise TypeError(f"Wrong input : {inp}")
metadatas[inp] = metadata
@@ -416,7 +414,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
'mode.extent.ulx': ulx, 'mode.extent.uly': lry, # bug in OTB <= 7.3 :
'mode.extent.lrx': lrx, 'mode.extent.lry': uly, # ULY/LRY are inverted
}
- new_input = OTBObject('ExtractROI', params)
+ new_input = App('ExtractROI', params)
# TODO: OTB 7.4 fixes this bug, how to handle different versions of OTB?
new_inputs.append(new_input)
# Potentially update the reference inputs for later resampling
@@ -429,7 +427,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
inputs = new_inputs
# Update metadatas
- metadatas = {input: input.GetImageMetaData('out') for input in inputs}
+ metadatas = {input: input.app.GetImageMetaData('out') for input in inputs}
# Get a metadata of an arbitrary image. This is just to compare later with other images
any_metadata = next(iter(metadatas.values()))
@@ -457,14 +455,14 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
new_inputs = []
for inp in inputs:
if metadatas[inp]['GeoTransform'][1] != pixel_size:
- superimposed = OTBObject('Superimpose', inr=reference_input, inm=inp, interpolator=interpolator)
+ superimposed = App('Superimpose', inr=reference_input, inm=inp, interpolator=interpolator)
new_inputs.append(superimposed)
else:
new_inputs.append(inp)
inputs = new_inputs
# Update metadatas
- metadatas = {inp: inp.GetImageMetaData('out') for inp in inputs}
+ metadatas = {inp: inp.app.GetImageMetaData('out') for inp in inputs}
# Final superimposition to be sure to have the exact same image sizes
# Getting the sizes of images
@@ -482,7 +480,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
new_inputs = []
for inp in inputs:
if image_sizes[inp] != most_common_image_size:
- superimposed = OTBObject('Superimpose', inr=same_size_images[0], inm=inp, interpolator=interpolator)
+ superimposed = App('Superimpose', inr=same_size_images[0], inm=inp, interpolator=interpolator)
new_inputs.append(superimposed)
else:
new_inputs.append(inp)
diff --git a/tests/test_core.py b/tests/test_core.py
index 436f99e..086c26e 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -1,13 +1,8 @@
-import os
-import pyotb
-from ast import literal_eval
-from pathlib import Path
-
import pytest
+import pyotb
+from tests_data import INPUT
-FILEPATH = os.environ["TEST_INPUT_IMAGE"]
-INPUT = pyotb.Input(FILEPATH)
TEST_IMAGE_STATS = {
'out.mean': [79.5505, 109.225, 115.456, 249.349],
'out.min': [33, 64, 91, 47],
@@ -59,9 +54,11 @@ def test_nonraster_property():
with pytest.raises(TypeError):
pyotb.ReadImageInfo(INPUT).dtype
+
def test_elapsed_time():
assert pyotb.ReadImageInfo(INPUT).elapsed_time < 1
+
# Other functions
def test_get_infos():
infos = INPUT.get_infos()
diff --git a/tests/test_numpy.py b/tests/test_numpy.py
index dd33425..0f42435 100644
--- a/tests/test_numpy.py
+++ b/tests/test_numpy.py
@@ -1,10 +1,10 @@
-import os
import numpy as np
import pyotb
+from tests_data import INPUT
+import numpy as np
-
-FILEPATH = os.environ["TEST_INPUT_IMAGE"]
-INPUT = pyotb.Input(FILEPATH)
+import pyotb
+from tests_data import INPUT
def test_export():
@@ -37,7 +37,7 @@ def test_convert_to_array():
def test_pixel_coords_otb_equals_numpy():
- assert INPUT[19,7] == list(INPUT.to_numpy()[19,7])
+ assert INPUT[19, 7] == list(INPUT.to_numpy()[19, 7])
def test_add_noise_array():
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index 3c88153..3acc354 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -1,9 +1,8 @@
-import sys
import os
import itertools
import pytest
import pyotb
-from pyotb.helpers import logger
+from tests_data import INPUT, FILEPATH
# List of buildings blocks, we can add other pyotb objects here
@@ -23,8 +22,6 @@ PYOTB_BLOCKS = [
PIPELINES_LENGTH = [1, 2, 3]
ALL_BLOCKS = PYOTB_BLOCKS + OTBAPPS_BLOCKS
-FILEPATH = os.environ["TEST_INPUT_IMAGE"]
-INPUT = pyotb.Input(FILEPATH)
def generate_pipeline(inp, building_blocks):
diff --git a/tests/test_serialization.py b/tests/test_serialization.py
index e64f2f1..b56e447 100644
--- a/tests/test_serialization.py
+++ b/tests/test_serialization.py
@@ -1,18 +1,16 @@
-import os
import pyotb
-
-filepath = os.environ["TEST_INPUT_IMAGE"]
+from tests_data import FILEPATH
def test_pipeline_simple():
# BandMath -> OrthoRectification -> ManageNoData
- app1 = pyotb.BandMath({'il': [filepath], 'exp': 'im1b1'})
+ app1 = pyotb.BandMath({'il': [FILEPATH], 'exp': 'im1b1'})
app2 = pyotb.OrthoRectification({'io.in': app1})
app3 = pyotb.ManageNoData({'in': app2})
summary = app3.summarize()
reference = {'name': 'ManageNoData', 'parameters': {'in': {
'name': 'OrthoRectification', 'parameters': {'io.in': {
- 'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}},
+ 'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}},
'map': 'utm',
'outputs.isotropic': True}},
'mode': 'buildmask'}}
@@ -21,19 +19,19 @@ def test_pipeline_simple():
def test_pipeline_diamond():
# Diamond graph
- app1 = pyotb.BandMath({'il': [filepath], 'exp': 'im1b1'})
+ app1 = pyotb.BandMath({'il': [FILEPATH], 'exp': 'im1b1'})
app2 = pyotb.OrthoRectification({'io.in': app1})
app3 = pyotb.ManageNoData({'in': app2})
app4 = pyotb.BandMathX({'il': [app2, app3], 'exp': 'im1+im2'})
summary = app4.summarize()
reference = {'name': 'BandMathX', 'parameters': {'il': [
{'name': 'OrthoRectification', 'parameters': {'io.in': {
- 'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}},
+ 'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}},
'map': 'utm',
'outputs.isotropic': True}},
{'name': 'ManageNoData', 'parameters': {'in': {
'name': 'OrthoRectification', 'parameters': {
- 'io.in': {'name': 'BandMath', 'parameters': {'il': (filepath,), 'exp': 'im1b1'}},
+ 'io.in': {'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}},
'map': 'utm',
'outputs.isotropic': True}},
'mode': 'buildmask'}}
diff --git a/tests/tests_data.py b/tests/tests_data.py
new file mode 100644
index 0000000..b190f40
--- /dev/null
+++ b/tests/tests_data.py
@@ -0,0 +1,3 @@
+import pyotb
+FILEPATH = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false"
+INPUT = pyotb.Input(FILEPATH)
--
GitLab
From 6a4ba70c3de182ff88db74ae60cb92aba3b8bfe3 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 2 Feb 2023 10:34:39 +0100
Subject: [PATCH 091/399] STYLE: linting
---
pyotb/core.py | 4 ++--
pyotb/functions.py | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index cc3c96e..a582159 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -614,7 +614,7 @@ class App(RasterInterface):
# This is when we actually call self.app.SetParameter*
self.__set_param(key, obj)
except (RuntimeError, TypeError, ValueError, KeyError) as e:
- raise Exception(
+ raise RuntimeError(
f"{self.name}: something went wrong before execution "
f"(while setting parameter '{key}' to '{obj}')"
) from e
@@ -687,7 +687,7 @@ class App(RasterInterface):
try:
self.app.Execute()
except (RuntimeError, FileNotFoundError) as e:
- raise Exception(f"{self.name}: error during during app execution") from e
+ raise RuntimeError(f"{self.name}: error during during app execution") from e
self.frozen = False
self._time_end = perf_counter()
logger.debug("%s: execution ended", self.name)
diff --git a/pyotb/functions.py b/pyotb/functions.py
index d782a0e..6cc7889 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -118,7 +118,7 @@ def all(*inputs): # pylint: disable=redefined-builtin
if isinstance(inputs[0], LogicalOperation):
res = inputs[0]
else:
- res = (inputs[0] != 0)
+ res = inputs[0] != 0
for inp in inputs[1:]:
if isinstance(inp, LogicalOperation):
res = res & inp
@@ -171,7 +171,7 @@ def any(*inputs): # pylint: disable=redefined-builtin
if isinstance(inputs[0], LogicalOperation):
res = inputs[0]
else:
- res = (inputs[0] != 0)
+ res = inputs[0] != 0
for inp in inputs[1:]:
if isinstance(inp, LogicalOperation):
res = res | inp
--
GitLab
From b674ff86850e48c7ddb85412f97fc14e2b4663a9 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 2 Feb 2023 21:34:05 +0100
Subject: [PATCH 092/399] FIX: bug with output
---
pyotb/core.py | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index a582159..e8761c1 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -590,7 +590,7 @@ class App(RasterInterface):
Args:
*args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
(e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
- - string or App, useful when the user implicitly wants to set the param "in"
+ - string or RasterInterface, useful when the user implicitly wants to set the param "in"
- list, useful when the user implicitly wants to set the param "il"
**kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif'
@@ -788,15 +788,15 @@ class App(RasterInterface):
"""
parameters = self.parameters.copy()
for key, param in parameters.items():
- # In the following, we replace each parameter which is an App, with its summary.
- if isinstance(param, App): # single parameter
+ # In the following, we replace each parameter which is an RasterInterface, with its summary.
+ if isinstance(param, RasterInterface): # single parameter
parameters[key] = param.summarize()
elif isinstance(param, list): # parameter list
- parameters[key] = [p.summarize() if isinstance(p, App) else p for p in param]
+ parameters[key] = [p.summarize() if isinstance(p, RasterInterface) else p for p in param]
return {"name": self.app.GetName(), "parameters": parameters}
# Private functions
- def __parse_args(self, args: list[str | App | dict | list]) -> dict[str, Any]:
+ def __parse_args(self, args: list[str | RasterInterface | dict | list]) -> dict[str, Any]:
"""Gather all input arguments in kwargs dict.
Args:
@@ -810,17 +810,17 @@ class App(RasterInterface):
for arg in args:
if isinstance(arg, dict):
kwargs.update(arg)
- elif isinstance(arg, (str, App)) or isinstance(arg, list) and is_key_list(self, self.key_input):
+ elif isinstance(arg, (str, RasterInterface)) or isinstance(arg, list) and is_key_list(self, self.key_input):
kwargs.update({self.key_input: arg})
return kwargs
- def __set_param(self, key: str, obj: list | tuple | App | otb.Application | list[Any]):
+ def __set_param(self, key: str, obj: list | tuple | RasterInterface | otb.Application | list[Any]):
"""Set one parameter, decide which otb.Application method to use depending on target object."""
if obj is None or (isinstance(obj, (list, tuple)) and not obj):
self.app.ClearValue(key)
return
# Single-parameter cases
- if isinstance(obj, App):
+ if isinstance(obj, RasterInterface):
self.app.ConnectImage(key, obj.app, obj.key_output_image)
elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB
self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0])
@@ -832,7 +832,7 @@ class App(RasterInterface):
elif is_key_images_list(self, key):
# To enable possible in-memory connections, we go through the list and set the parameters one by one
for inp in obj:
- if isinstance(inp, App):
+ if isinstance(inp, RasterInterface):
self.app.ConnectImage(key, inp.app, inp.key_output_image)
elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB
self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0])
--
GitLab
From b71950898619af15f2b3976a1a6339cf6a7ac136 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 2 Feb 2023 22:22:12 +0100
Subject: [PATCH 093/399] REFAC: Operators
---
pyotb/core.py | 240 +++++++-------------------------------------------
1 file changed, 32 insertions(+), 208 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index e8761c1..a9ea536 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -198,241 +198,65 @@ class RasterInterface(ABC):
row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x
return abs(int(row)), int(col)
- def __add__(self, other: App | str | int | float) -> Operation:
- """Overrides the default addition and flavours it with BandMathX.
-
- Args:
- other: the other member of the operation
-
- Returns:
- self + other
-
- """
- if isinstance(other, (np.ndarray, np.generic)):
+ @staticmethod
+ def _create_operator(op_cls, name, a, b):
+ if isinstance(b, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return Operation("+", self, other)
+ return op_cls(name, a, b)
- def __sub__(self, other: App | str | int | float) -> Operation:
- """Overrides the default subtraction and flavours it with BandMathX.
-
- Args:
- other: the other member of the operation
-
- Returns:
- self - other
+ def __add__(self, other: App | str | int | float) -> Operation:
+ return self._create_operator(Operation, "+", self, other)
- """
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return Operation("-", self, other)
+ def __sub__(self, other: App | str | int | float) -> Operation:
+ return self._create_operator(Operation, "-", self, other)
def __mul__(self, other: App | str | int | float) -> Operation:
- """Overrides the default subtraction and flavours it with BandMathX.
-
- Args:
- other: the other member of the operation
-
- Returns:
- self * other
-
- """
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return Operation("*", self, other)
+ return self._create_operator(Operation, "*", self, other)
def __truediv__(self, other: App | str | int | float) -> Operation:
- """Overrides the default subtraction and flavours it with BandMathX.
-
- Args:
- other: the other member of the operation
-
- Returns:
- self / other
-
- """
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return Operation("/", self, other)
+ return self._create_operator(Operation, "/", self, other)
def __radd__(self, other: App | str | int | float) -> Operation:
- """Overrides the default reverse addition and flavours it with BandMathX.
-
- Args:
- other: the other member of the operation
-
- Returns:
- other + self
-
- """
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return Operation("+", other, self)
+ return self._create_operator(Operation, "+", other, self)
def __rsub__(self, other: App | str | int | float) -> Operation:
- """Overrides the default subtraction and flavours it with BandMathX.
-
- Args:
- other: the other member of the operation
-
- Returns:
- other - self
-
- """
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return Operation("-", other, self)
+ return self._create_operator(Operation, "-", other, self)
def __rmul__(self, other: App | str | int | float) -> Operation:
- """Overrides the default multiplication and flavours it with BandMathX.
-
- Args:
- other: the other member of the operation
-
- Returns:
- other * self
-
- """
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return Operation("*", other, self)
+ return self._create_operator(Operation, "*", other, self)
def __rtruediv__(self, other: App | str | int | float) -> Operation:
- """Overrides the default division and flavours it with BandMathX.
-
- Args:
- other: the other member of the operation
-
- Returns:
- other / self
-
- """
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return Operation("/", other, self)
+ return self._create_operator(Operation, "/", other, self)
def __abs__(self) -> Operation:
- """Overrides the default abs operator and flavours it with BandMathX.
-
- Returns:
- abs(self)
-
- """
return Operation("abs", self)
- def __ge__(self, other: App | str | int | float) -> Operation:
- """Overrides the default greater or equal and flavours it with BandMathX.
-
- Args:
- other: the other member of the operation
-
- Returns:
- self >= other
-
- """
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return LogicalOperation(">=", self, other)
-
- def __le__(self, other: App | str | int | float) -> Operation:
- """Overrides the default less or equal and flavours it with BandMathX.
-
- Args:
- other: the other member of the operation
-
- Returns:
- self <= other
-
- """
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return LogicalOperation("<=", self, other)
+ def __ge__(self, other: App | str | int | float) -> LogicalOperation:
+ return self._create_operator(LogicalOperation, ">=", self, other)
- def __gt__(self, other: App | str | int | float) -> Operation:
- """Overrides the default greater operator and flavours it with BandMathX.
+ def __le__(self, other: App | str | int | float) -> LogicalOperation:
+ return self._create_operator(LogicalOperation, "<=", self, other)
- Args:
- other: the other member of the operation
+ def __gt__(self, other: App | str | int | float) -> LogicalOperation:
+ return self._create_operator(LogicalOperation, ">", self, other)
- Returns:
- self > other
+ def __lt__(self, other: App | str | int | float) -> LogicalOperation:
+ return self._create_operator(LogicalOperation, "<", self, other)
- """
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return LogicalOperation(">", self, other)
+ def __eq__(self, other: App | str | int | float) -> LogicalOperation:
+ return self._create_operator(LogicalOperation, "==", self, other)
- def __lt__(self, other: App | str | int | float) -> Operation:
- """Overrides the default less operator and flavours it with BandMathX.
+ def __ne__(self, other: App | str | int | float) -> LogicalOperation:
+ return self._create_operator(LogicalOperation, "!=", self, other)
- Args:
- other: the other member of the operation
+ def __or__(self, other: App | str | int | float) -> LogicalOperation:
+ return self._create_operator(LogicalOperation, "||", self, other)
- Returns:
- self < other
-
- """
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return LogicalOperation("<", self, other)
-
- def __eq__(self, other: App | str | int | float) -> Operation:
- """Overrides the default eq operator and flavours it with BandMathX.
-
- Args:
- other: the other member of the operation
-
- Returns:
- self == other
-
- """
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return LogicalOperation("==", self, other)
-
- def __ne__(self, other: App | str | int | float) -> Operation:
- """Overrides the default different operator and flavours it with BandMathX.
-
- Args:
- other: the other member of the operation
-
- Returns:
- self != other
-
- """
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return LogicalOperation("!=", self, other)
-
- def __or__(self, other: App | str | int | float) -> Operation:
- """Overrides the default or operator and flavours it with BandMathX.
-
- Args:
- other: the other member of the operation
-
- Returns:
- self || other
-
- """
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return LogicalOperation("||", self, other)
-
- def __and__(self, other: App | str | int | float) -> Operation:
- """Overrides the default and operator and flavours it with BandMathX.
-
- Args:
- other: the other member of the operation
-
- Returns:
- self && other
-
- """
- if isinstance(other, (np.ndarray, np.generic)):
- return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return LogicalOperation("&&", self, other)
+ def __and__(self, other: App | str | int | float) -> LogicalOperation:
+ return self._create_operator(LogicalOperation, "&&", self, other)
- # TODO: other operations ?
- # e.g. __pow__... cf https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types
+ # Some other operations could be implemented with the same pattern
+ # e.g. __pow__... cf https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types
def __array__(self) -> np.ndarray:
"""This is called when running np.asarray(pyotb_object).
--
GitLab
From 805f6b9e3ba347860c43c4d65bc05ed01e119267 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 2 Feb 2023 22:36:09 +0100
Subject: [PATCH 094/399] DOC: docstring
---
pyotb/core.py | 35 ++++++++++++++++++++++++++++++++---
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index a9ea536..2499aa6 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -199,60 +199,89 @@ class RasterInterface(ABC):
return abs(int(row)), int(col)
@staticmethod
- def _create_operator(op_cls, name, a, b):
- if isinstance(b, (np.ndarray, np.generic)):
+ def _create_operator(op_cls, name, x, y) -> Operation:
+ """Create an operator.
+
+ Args:
+ op_cls: Operator class
+ name: operator expression
+ x: first element
+ y: second element
+
+ Return:
+ operator
+
+ """
+ if isinstance(y, (np.ndarray, np.generic)):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
- return op_cls(name, a, b)
+ return op_cls(name, x, y)
def __add__(self, other: App | str | int | float) -> Operation:
+ """Addition."""
return self._create_operator(Operation, "+", self, other)
def __sub__(self, other: App | str | int | float) -> Operation:
+ """Subtraction."""
return self._create_operator(Operation, "-", self, other)
def __mul__(self, other: App | str | int | float) -> Operation:
+ """Multiplication."""
return self._create_operator(Operation, "*", self, other)
def __truediv__(self, other: App | str | int | float) -> Operation:
+ """Division."""
return self._create_operator(Operation, "/", self, other)
def __radd__(self, other: App | str | int | float) -> Operation:
+ """Right addition."""
return self._create_operator(Operation, "+", other, self)
def __rsub__(self, other: App | str | int | float) -> Operation:
+ """Right subtraction."""
return self._create_operator(Operation, "-", other, self)
def __rmul__(self, other: App | str | int | float) -> Operation:
+ """Right multiplication."""
return self._create_operator(Operation, "*", other, self)
def __rtruediv__(self, other: App | str | int | float) -> Operation:
+ """Right division."""
return self._create_operator(Operation, "/", other, self)
def __abs__(self) -> Operation:
+ """Absolute value."""
return Operation("abs", self)
def __ge__(self, other: App | str | int | float) -> LogicalOperation:
+ """Greater of equal than."""
return self._create_operator(LogicalOperation, ">=", self, other)
def __le__(self, other: App | str | int | float) -> LogicalOperation:
+ """Lower of equal than."""
return self._create_operator(LogicalOperation, "<=", self, other)
def __gt__(self, other: App | str | int | float) -> LogicalOperation:
+ """Greater than."""
return self._create_operator(LogicalOperation, ">", self, other)
def __lt__(self, other: App | str | int | float) -> LogicalOperation:
+ """Lower than."""
return self._create_operator(LogicalOperation, "<", self, other)
def __eq__(self, other: App | str | int | float) -> LogicalOperation:
+ """Equality."""
return self._create_operator(LogicalOperation, "==", self, other)
def __ne__(self, other: App | str | int | float) -> LogicalOperation:
+ """Inequality."""
return self._create_operator(LogicalOperation, "!=", self, other)
def __or__(self, other: App | str | int | float) -> LogicalOperation:
+ """Logical or."""
return self._create_operator(LogicalOperation, "||", self, other)
def __and__(self, other: App | str | int | float) -> LogicalOperation:
+ """Logical and."""
return self._create_operator(LogicalOperation, "&&", self, other)
# Some other operations could be implemented with the same pattern
--
GitLab
From ac5f66d82a9362606846fb2466ae0b9102f552e6 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 3 Feb 2023 10:25:39 +0100
Subject: [PATCH 095/399] DOC: ops can deal with RasterInterface objects
---
pyotb/core.py | 32 ++++++++++++++++----------------
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 2499aa6..1781d18 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -216,35 +216,35 @@ class RasterInterface(ABC):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return op_cls(name, x, y)
- def __add__(self, other: App | str | int | float) -> Operation:
+ def __add__(self, other: RasterInterface | str | int | float) -> Operation:
"""Addition."""
return self._create_operator(Operation, "+", self, other)
- def __sub__(self, other: App | str | int | float) -> Operation:
+ def __sub__(self, other: RasterInterface | str | int | float) -> Operation:
"""Subtraction."""
return self._create_operator(Operation, "-", self, other)
- def __mul__(self, other: App | str | int | float) -> Operation:
+ def __mul__(self, other: RasterInterface | str | int | float) -> Operation:
"""Multiplication."""
return self._create_operator(Operation, "*", self, other)
- def __truediv__(self, other: App | str | int | float) -> Operation:
+ def __truediv__(self, other: RasterInterface | str | int | float) -> Operation:
"""Division."""
return self._create_operator(Operation, "/", self, other)
- def __radd__(self, other: App | str | int | float) -> Operation:
+ def __radd__(self, other: RasterInterface | str | int | float) -> Operation:
"""Right addition."""
return self._create_operator(Operation, "+", other, self)
- def __rsub__(self, other: App | str | int | float) -> Operation:
+ def __rsub__(self, other: RasterInterface | str | int | float) -> Operation:
"""Right subtraction."""
return self._create_operator(Operation, "-", other, self)
- def __rmul__(self, other: App | str | int | float) -> Operation:
+ def __rmul__(self, other: RasterInterface | str | int | float) -> Operation:
"""Right multiplication."""
return self._create_operator(Operation, "*", other, self)
- def __rtruediv__(self, other: App | str | int | float) -> Operation:
+ def __rtruediv__(self, other: RasterInterface | str | int | float) -> Operation:
"""Right division."""
return self._create_operator(Operation, "/", other, self)
@@ -252,35 +252,35 @@ class RasterInterface(ABC):
"""Absolute value."""
return Operation("abs", self)
- def __ge__(self, other: App | str | int | float) -> LogicalOperation:
+ def __ge__(self, other: RasterInterface | str | int | float) -> LogicalOperation:
"""Greater of equal than."""
return self._create_operator(LogicalOperation, ">=", self, other)
- def __le__(self, other: App | str | int | float) -> LogicalOperation:
+ def __le__(self, other: RasterInterface | str | int | float) -> LogicalOperation:
"""Lower of equal than."""
return self._create_operator(LogicalOperation, "<=", self, other)
- def __gt__(self, other: App | str | int | float) -> LogicalOperation:
+ def __gt__(self, other: RasterInterface | str | int | float) -> LogicalOperation:
"""Greater than."""
return self._create_operator(LogicalOperation, ">", self, other)
- def __lt__(self, other: App | str | int | float) -> LogicalOperation:
+ def __lt__(self, other: RasterInterface | str | int | float) -> LogicalOperation:
"""Lower than."""
return self._create_operator(LogicalOperation, "<", self, other)
- def __eq__(self, other: App | str | int | float) -> LogicalOperation:
+ def __eq__(self, other: RasterInterface | str | int | float) -> LogicalOperation:
"""Equality."""
return self._create_operator(LogicalOperation, "==", self, other)
- def __ne__(self, other: App | str | int | float) -> LogicalOperation:
+ def __ne__(self, other: RasterInterface | str | int | float) -> LogicalOperation:
"""Inequality."""
return self._create_operator(LogicalOperation, "!=", self, other)
- def __or__(self, other: App | str | int | float) -> LogicalOperation:
+ def __or__(self, other: RasterInterface | str | int | float) -> LogicalOperation:
"""Logical or."""
return self._create_operator(LogicalOperation, "||", self, other)
- def __and__(self, other: App | str | int | float) -> LogicalOperation:
+ def __and__(self, other: RasterInterface | str | int | float) -> LogicalOperation:
"""Logical and."""
return self._create_operator(LogicalOperation, "&&", self, other)
--
GitLab
From 0dbe151708b3a38731f256a0a803b3b2c60f0af1 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 3 Feb 2023 11:19:11 +0100
Subject: [PATCH 096/399] REFAC: self.logical_fake_exp_bands only in
LogicalOperation
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 1781d18..6b7d58c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -856,7 +856,6 @@ class Operation(App):
self.inputs = []
self.nb_channels = {}
self.fake_exp_bands = []
- self.logical_fake_exp_bands = []
self.build_fake_expressions(operator, inputs, nb_bands=nb_bands)
# Transforming images to the adequate im#, e.g. `input1` to "im1"
# creating a dictionary that is like {str(input1): 'im1', 'image2.tif': 'im2', ...}.
@@ -1033,6 +1032,7 @@ class LogicalOperation(Operation):
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
+ self.logical_fake_exp_bands = []
super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation")
self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands)
--
GitLab
From c7d2a0370411b05ba799ec3ef650290cfee31a9a Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 3 Feb 2023 13:19:57 +0100
Subject: [PATCH 097/399] REFAC: rename attrs/funcs
---
pyotb/core.py | 112 +++++++++++++++++++++-----------------------
tests/test_core.py | 6 +--
tests/test_numpy.py | 4 +-
3 files changed, 59 insertions(+), 63 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 6b7d58c..daa4082 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -14,7 +14,7 @@ import otbApplication as otb # pylint: disable=import-error
from .helpers import logger
-class RasterInterface(ABC):
+class ImageObject(ABC):
"""Abstraction of an image object."""
app: otb.Application
@@ -22,7 +22,7 @@ class RasterInterface(ABC):
@property
@abstractmethod
- def key_output_image(self):
+ def output_image_key(self):
"""Returns the name of a parameter associated to an image. Property defined in App and Output."""
@abstractmethod
@@ -32,7 +32,7 @@ class RasterInterface(ABC):
@property
def metadata(self) -> dict[str, (str, float, list[float])]:
"""Return first output image metadata dictionary."""
- return dict(self.app.GetMetadataDictionary(self.key_output_image))
+ return dict(self.app.GetMetadataDictionary(self.output_image_key))
@property
def dtype(self) -> np.dtype:
@@ -42,7 +42,7 @@ class RasterInterface(ABC):
dtype: pixel type of the output image
"""
- enum = self.app.GetParameterOutputImagePixelType(self.key_output_image)
+ enum = self.app.GetParameterOutputImagePixelType(self.output_image_key)
return self.app.ConvertPixelTypeToNumpy(enum)
@property
@@ -53,8 +53,8 @@ class RasterInterface(ABC):
shape: (height, width, bands)
"""
- width, height = self.app.GetImageSize(self.key_output_image)
- bands = self.app.GetImageNbBands(self.key_output_image)
+ width, height = self.app.GetImageSize(self.output_image_key)
+ bands = self.app.GetImageNbBands(self.output_image_key)
return height, width, bands
@property
@@ -64,13 +64,13 @@ class RasterInterface(ABC):
Returns:
transform: (X spacing, X offset, X origin, Y offset, Y spacing, Y origin)
"""
- spacing_x, spacing_y = self.app.GetImageSpacing(self.key_output_image)
- origin_x, origin_y = self.app.GetImageOrigin(self.key_output_image)
+ spacing_x, spacing_y = self.app.GetImageSpacing(self.output_image_key)
+ origin_x, origin_y = self.app.GetImageOrigin(self.output_image_key)
# Shift image origin since OTB is giving coordinates of pixel center instead of corners
origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2
return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y
- def get_infos(self) -> dict[str, (str, float, list[float])]:
+ def get_info(self) -> dict[str, (str, float, list[float])]:
"""Return a dict output of ReadImageInfo for the first image output."""
return App("ReadImageInfo", self, quiet=True).data
@@ -78,7 +78,7 @@ class RasterInterface(ABC):
"""Return a dict output of ComputeImagesStatistics for the first image output."""
return App("ComputeImagesStatistics", self, quiet=True).data
- def read_values_at_coords(self, row: int, col: int, bands: int = None) -> list[int | float] | int | float:
+ def get_values_at_coords(self, row: int, col: int, bands: int = None) -> list[int | float] | int | float:
"""Get pixel value(s) at a given YX coordinates.
Args:
@@ -106,9 +106,7 @@ class RasterInterface(ABC):
app.set_parameters({"cl": [f"Channel{n + 1}" for n in channels]})
app.execute()
data = literal_eval(app.app.GetParameterString("value"))
- if len(channels) == 1:
- return data[0]
- return data
+ return data[0] if len(channels) == 1 else data
def channels_list_from_slice(self, bands: int) -> list[int]:
"""Get list of channels to read values at, from a slice."""
@@ -140,7 +138,7 @@ class RasterInterface(ABC):
"""
if key is None:
- key = self.key_output_image
+ key = self.output_image_key
if key not in self.exports_dic:
self.exports_dic[key] = self.app.ExportImage(key)
if preserve_dtype:
@@ -163,9 +161,7 @@ class RasterInterface(ABC):
"""
data = self.export(key, preserve_dtype)
array = data["array"]
- if copy:
- return array.copy()
- return array
+ return array.copy() if copy else array
def to_rasterio(self) -> tuple[np.ndarray, dict[str, Any]]:
"""Export image as a numpy array and its metadata compatible with rasterio.
@@ -177,14 +173,14 @@ class RasterInterface(ABC):
"""
array = self.to_numpy(preserve_dtype=True, copy=False)
height, width, count = array.shape
- proj = self.app.GetImageProjection(self.key_output_image)
+ proj = self.app.GetImageProjection(self.output_image_key)
profile = {
'crs': proj, 'dtype': array.dtype, 'transform': self.transform,
'count': count, 'height': height, 'width': width,
}
return np.moveaxis(array, 2, 0), profile
- def xy_to_rowcol(self, x: float, y: float) -> tuple[int, int]:
+ def get_rowcol_from_xy(self, x: float, y: float) -> tuple[int, int]:
"""Find (row, col) index using (x, y) projected coordinates - image CRS is expected.
Args:
@@ -216,35 +212,35 @@ class RasterInterface(ABC):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return op_cls(name, x, y)
- def __add__(self, other: RasterInterface | str | int | float) -> Operation:
+ def __add__(self, other: ImageObject | str | int | float) -> Operation:
"""Addition."""
return self._create_operator(Operation, "+", self, other)
- def __sub__(self, other: RasterInterface | str | int | float) -> Operation:
+ def __sub__(self, other: ImageObject | str | int | float) -> Operation:
"""Subtraction."""
return self._create_operator(Operation, "-", self, other)
- def __mul__(self, other: RasterInterface | str | int | float) -> Operation:
+ def __mul__(self, other: ImageObject | str | int | float) -> Operation:
"""Multiplication."""
return self._create_operator(Operation, "*", self, other)
- def __truediv__(self, other: RasterInterface | str | int | float) -> Operation:
+ def __truediv__(self, other: ImageObject | str | int | float) -> Operation:
"""Division."""
return self._create_operator(Operation, "/", self, other)
- def __radd__(self, other: RasterInterface | str | int | float) -> Operation:
+ def __radd__(self, other: ImageObject | str | int | float) -> Operation:
"""Right addition."""
return self._create_operator(Operation, "+", other, self)
- def __rsub__(self, other: RasterInterface | str | int | float) -> Operation:
+ def __rsub__(self, other: ImageObject | str | int | float) -> Operation:
"""Right subtraction."""
return self._create_operator(Operation, "-", other, self)
- def __rmul__(self, other: RasterInterface | str | int | float) -> Operation:
+ def __rmul__(self, other: ImageObject | str | int | float) -> Operation:
"""Right multiplication."""
return self._create_operator(Operation, "*", other, self)
- def __rtruediv__(self, other: RasterInterface | str | int | float) -> Operation:
+ def __rtruediv__(self, other: ImageObject | str | int | float) -> Operation:
"""Right division."""
return self._create_operator(Operation, "/", other, self)
@@ -252,35 +248,35 @@ class RasterInterface(ABC):
"""Absolute value."""
return Operation("abs", self)
- def __ge__(self, other: RasterInterface | str | int | float) -> LogicalOperation:
+ def __ge__(self, other: ImageObject | str | int | float) -> LogicalOperation:
"""Greater of equal than."""
return self._create_operator(LogicalOperation, ">=", self, other)
- def __le__(self, other: RasterInterface | str | int | float) -> LogicalOperation:
+ def __le__(self, other: ImageObject | str | int | float) -> LogicalOperation:
"""Lower of equal than."""
return self._create_operator(LogicalOperation, "<=", self, other)
- def __gt__(self, other: RasterInterface | str | int | float) -> LogicalOperation:
+ def __gt__(self, other: ImageObject | str | int | float) -> LogicalOperation:
"""Greater than."""
return self._create_operator(LogicalOperation, ">", self, other)
- def __lt__(self, other: RasterInterface | str | int | float) -> LogicalOperation:
+ def __lt__(self, other: ImageObject | str | int | float) -> LogicalOperation:
"""Lower than."""
return self._create_operator(LogicalOperation, "<", self, other)
- def __eq__(self, other: RasterInterface | str | int | float) -> LogicalOperation:
+ def __eq__(self, other: ImageObject | str | int | float) -> LogicalOperation:
"""Equality."""
return self._create_operator(LogicalOperation, "==", self, other)
- def __ne__(self, other: RasterInterface | str | int | float) -> LogicalOperation:
+ def __ne__(self, other: ImageObject | str | int | float) -> LogicalOperation:
"""Inequality."""
return self._create_operator(LogicalOperation, "!=", self, other)
- def __or__(self, other: RasterInterface | str | int | float) -> LogicalOperation:
+ def __or__(self, other: ImageObject | str | int | float) -> LogicalOperation:
"""Logical or."""
return self._create_operator(LogicalOperation, "||", self, other)
- def __and__(self, other: RasterInterface | str | int | float) -> LogicalOperation:
+ def __and__(self, other: ImageObject | str | int | float) -> LogicalOperation:
"""Logical and."""
return self._create_operator(LogicalOperation, "&&", self, other)
@@ -322,7 +318,7 @@ class RasterInterface(ABC):
elif isinstance(inp, App):
if not inp.exports_dic:
inp.export()
- image_dic = inp.exports_dic[inp.key_output_image]
+ image_dic = inp.exports_dic[inp.output_image_key]
array = image_dic["array"]
arrays.append(array)
else:
@@ -343,7 +339,7 @@ class RasterInterface(ABC):
return NotImplemented
-class App(RasterInterface):
+class App(ImageObject):
"""Base class that gathers common operations for any OTB application."""
def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, **kwargs):
@@ -408,7 +404,7 @@ class App(RasterInterface):
return self.get_first_key(param_types=[otb.ParameterType_InputImage, otb.ParameterType_InputImageList])
@property
- def key_output_image(self) -> str:
+ def output_image_key(self) -> str:
"""Get the name of first output image parameter."""
return self.get_first_key(param_types=[otb.ParameterType_OutputImage])
@@ -443,7 +439,7 @@ class App(RasterInterface):
Args:
*args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
(e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
- - string or RasterInterface, useful when the user implicitly wants to set the param "in"
+ - string or ImageObject, useful when the user implicitly wants to set the param "in"
- list, useful when the user implicitly wants to set the param "il"
**kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif'
@@ -582,8 +578,8 @@ class App(RasterInterface):
kwargs.update(arg)
elif isinstance(arg, str) and kwargs:
logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg)
- elif isinstance(arg, (str, Path)) and self.key_output_image:
- kwargs.update({self.key_output_image: str(arg)})
+ elif isinstance(arg, (str, Path)) and self.output_image_key:
+ kwargs.update({self.output_image_key: str(arg)})
# Append filename extension to filenames
if filename_extension:
@@ -641,15 +637,15 @@ class App(RasterInterface):
"""
parameters = self.parameters.copy()
for key, param in parameters.items():
- # In the following, we replace each parameter which is an RasterInterface, with its summary.
- if isinstance(param, RasterInterface): # single parameter
+ # In the following, we replace each parameter which is an ImageObject, with its summary.
+ if isinstance(param, ImageObject): # single parameter
parameters[key] = param.summarize()
elif isinstance(param, list): # parameter list
- parameters[key] = [p.summarize() if isinstance(p, RasterInterface) else p for p in param]
+ parameters[key] = [p.summarize() if isinstance(p, ImageObject) else p for p in param]
return {"name": self.app.GetName(), "parameters": parameters}
# Private functions
- def __parse_args(self, args: list[str | RasterInterface | dict | list]) -> dict[str, Any]:
+ def __parse_args(self, args: list[str | ImageObject | dict | list]) -> dict[str, Any]:
"""Gather all input arguments in kwargs dict.
Args:
@@ -663,18 +659,18 @@ class App(RasterInterface):
for arg in args:
if isinstance(arg, dict):
kwargs.update(arg)
- elif isinstance(arg, (str, RasterInterface)) or isinstance(arg, list) and is_key_list(self, self.key_input):
+ elif isinstance(arg, (str, ImageObject)) or isinstance(arg, list) and is_key_list(self, self.key_input):
kwargs.update({self.key_input: arg})
return kwargs
- def __set_param(self, key: str, obj: list | tuple | RasterInterface | otb.Application | list[Any]):
+ def __set_param(self, key: str, obj: list | tuple | ImageObject | otb.Application | list[Any]):
"""Set one parameter, decide which otb.Application method to use depending on target object."""
if obj is None or (isinstance(obj, (list, tuple)) and not obj):
self.app.ClearValue(key)
return
# Single-parameter cases
- if isinstance(obj, RasterInterface):
- self.app.ConnectImage(key, obj.app, obj.key_output_image)
+ if isinstance(obj, ImageObject):
+ self.app.ConnectImage(key, obj.app, obj.output_image_key)
elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB
self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0])
elif key == "ram": # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200
@@ -685,8 +681,8 @@ class App(RasterInterface):
elif is_key_images_list(self, key):
# To enable possible in-memory connections, we go through the list and set the parameters one by one
for inp in obj:
- if isinstance(inp, RasterInterface):
- self.app.ConnectImage(key, inp.app, inp.key_output_image)
+ if isinstance(inp, ImageObject):
+ self.app.ConnectImage(key, inp.app, inp.output_image_key)
elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB
self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0])
else: # here `input` should be an image filepath
@@ -735,7 +731,7 @@ class App(RasterInterface):
channels = None
if len(key) == 3:
channels = key[2]
- return self.read_values_at_coords(row, col, channels)
+ return self.get_values_at_coords(row, col, channels)
# Slicing
if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)):
raise ValueError(f'"{key}"cannot be interpreted as valid slicing. Slicing should be 2D or 3D.')
@@ -1098,7 +1094,7 @@ class Input(App):
return f"<pyotb.Input object from {self.filepath}>"
-class Output(RasterInterface):
+class Output(ImageObject):
"""Object that behave like a pointer to a specific application output file."""
def __init__(self, pyotb_app: App, param_key: str = None, filepath: str = None, mkdir: bool = True):
@@ -1125,8 +1121,8 @@ class Output(RasterInterface):
self.make_parent_dirs()
@property
- def key_output_image(self) -> str:
- """Force the right key to be used when accessing the RasterInterface."""
+ def output_image_key(self) -> str:
+ """Force the right key to be used when accessing the ImageObject."""
return self.param_key
def exists(self) -> bool:
@@ -1144,8 +1140,8 @@ class Output(RasterInterface):
def write(self, filepath: None | str | Path = None, **kwargs):
"""Write output to disk, filepath is not required if it was provided to parent App during init."""
if filepath is None and self.filepath:
- return self.parent_pyotb_app.write({self.key_output_image: self.filepath}, **kwargs)
- return self.parent_pyotb_app.write({self.key_output_image: filepath}, **kwargs)
+ return self.parent_pyotb_app.write({self.output_image_key: self.filepath}, **kwargs)
+ return self.parent_pyotb_app.write({self.output_image_key: filepath}, **kwargs)
def __str__(self) -> str:
"""Return a nice string representation with source app name and object id."""
@@ -1208,7 +1204,7 @@ def get_pixel_type(inp: str | App) -> str:
raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}")
pixel_type = getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[datatype]}")
elif isinstance(inp, App):
- pixel_type = inp.app.GetParameterOutputImagePixelType(inp.key_output_image)
+ pixel_type = inp.app.GetParameterOutputImagePixelType(inp.output_image_key)
else:
raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}")
return pixel_type
diff --git a/tests/test_core.py b/tests/test_core.py
index 75dca37..58ddeac 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -27,7 +27,7 @@ def test_key_input():
def test_key_output():
- assert INPUT.key_output_image == "out"
+ assert INPUT.output_image_key == "out"
def test_dtype():
@@ -62,7 +62,7 @@ def test_elapsed_time():
# Other functions
def test_get_infos():
- infos = INPUT.get_infos()
+ infos = INPUT.get_info()
assert (infos["sizex"], infos["sizey"]) == (251, 304)
@@ -71,7 +71,7 @@ def test_get_statistics():
def test_xy_to_rowcol():
- assert INPUT.xy_to_rowcol(760101, 6945977) == (19, 7)
+ assert INPUT.get_rowcol_from_xy(760101, 6945977) == (19, 7)
def test_write():
diff --git a/tests/test_numpy.py b/tests/test_numpy.py
index d36dfb7..e62ffbb 100644
--- a/tests/test_numpy.py
+++ b/tests/test_numpy.py
@@ -5,7 +5,7 @@ from tests_data import INPUT
def test_export():
INPUT.export()
- array = INPUT.exports_dic[INPUT.key_output_image]["array"]
+ array = INPUT.exports_dic[INPUT.output_image_key]["array"]
assert isinstance(array, np.ndarray)
assert array.dtype == "uint8"
del INPUT.exports_dic["out"]
@@ -13,7 +13,7 @@ def test_export():
def test_output_export():
INPUT.out.export()
- assert INPUT.out.key_output_image in INPUT.out.exports_dic
+ assert INPUT.out.output_image_key in INPUT.out.exports_dic
def test_to_numpy():
--
GitLab
From 97159e767a4b5aa7afe6ece1ddd56d9caea0379b Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 3 Feb 2023 13:52:05 +0100
Subject: [PATCH 098/399] ADD: version --> 2.0.0
---
pyotb/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index ac6264c..8e22979 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
"""This module provides convenient python wrapping of otbApplications."""
-__version__ = "1.6.0"
+__version__ = "2.0.0"
from .helpers import logger, set_logger_level
from .apps import *
--
GitLab
From 824a87d4c55cb5677f49c788ec7496ea1ffb815c Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 3 Feb 2023 11:42:10 +0100
Subject: [PATCH 099/399] REFAC: Use RasterInterface when possible
---
pyotb/core.py | 42 ++++++++++++++++++++++--------------------
1 file changed, 22 insertions(+), 20 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index daa4082..8239e3a 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -315,7 +315,7 @@ class ImageObject(ABC):
for inp in inputs:
if isinstance(inp, (float, int, np.ndarray, np.generic)):
arrays.append(inp)
- elif isinstance(inp, App):
+ elif isinstance(inp, RasterInterface):
if not inp.exports_dic:
inp.export()
image_dic = inp.exports_dic[inp.output_image_key]
@@ -841,7 +841,7 @@ class Operation(App):
Args:
operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
- *inputs: inputs. Can be App, filepath, int or float
+ *inputs: inputs. Can be RasterInterface, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
name: override the Operation name
@@ -873,14 +873,15 @@ class Operation(App):
super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True)
self.name = f'Operation exp="{self.exp}"'
- def build_fake_expressions(self, operator: str, inputs: list[App | str | int | float], nb_bands: int = None):
+ def build_fake_expressions(self, operator: str, inputs: list[RasterInterface | str | int | float],
+ nb_bands: int = None):
"""Create a list of 'fake' expressions, one for each band.
E.g for the operation input1 + input2, we create a fake expression that is like "str(input1) + str(input2)"
Args:
operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
- inputs: inputs. Can be App, filepath, int or float
+ inputs: inputs. Can be RasterInterface, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
@@ -955,7 +956,8 @@ class Operation(App):
return exp_bands, ";".join(exp_bands)
@staticmethod
- def make_fake_exp(x: App | str, band: int, keep_logical: bool = False) -> tuple[str, list[App], int]:
+ def make_fake_exp(x: RasterInterface | str, band: int, keep_logical: bool = False) \
+ -> tuple[str, list[RasterInterface], int]:
"""This an internal function, only to be used by `build_fake_expressions`.
Enable to create a fake expression just for one input and one band.
@@ -1032,7 +1034,7 @@ class LogicalOperation(Operation):
super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation")
self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands)
- def build_fake_expressions(self, operator: str, inputs: list[App | str | int | float], nb_bands: int = None):
+ def build_fake_expressions(self, operator: str, inputs: list[RasterInterface | str | int | float], nb_bands: int = None):
"""Create a list of 'fake' expressions, one for each band.
e.g for the operation input1 > input2, we create a fake expression that is like
@@ -1040,7 +1042,7 @@ class LogicalOperation(Operation):
Args:
operator: str (one of >, <, >=, <=, ==, !=, &, |)
- inputs: Can be App, filepath, int or float
+ inputs: Can be RasterInterface, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
@@ -1148,29 +1150,29 @@ class Output(ImageObject):
return f"<pyotb.Output {self.name} object, id {id(self)}>"
-def get_nbchannels(inp: str | App) -> int:
+def get_nbchannels(inp: str | RasterInterface) -> int:
"""Get the nb of bands of input image.
Args:
- inp: can be filepath or pyotb object
+ inp: can be filepath or RasterInterface object
Returns:
number of bands in image
"""
- if isinstance(inp, App):
+ if isinstance(inp, RasterInterface):
nb_channels = inp.shape[-1]
else:
# Executing the app, without printing its log
try:
- info = App("ReadImageInfo", inp, quiet=True)
+ info = RasterInterface("ReadImageInfo", inp, quiet=True)
nb_channels = info.app.GetParameterInt("numberbands")
except Exception as e: # this happens when we pass a str that is not a filepath
raise TypeError(f"Could not get the number of channels of '{inp}'. Not a filepath or wrong filepath") from e
return nb_channels
-def get_pixel_type(inp: str | App) -> str:
+def get_pixel_type(inp: str | RasterInterface) -> str:
"""Get the encoding of input image pixels.
Args:
@@ -1178,7 +1180,7 @@ def get_pixel_type(inp: str | App) -> str:
Returns:
pixel_type: OTB enum e.g. `otbApplication.ImagePixelType_uint8', which actually is an int.
- For an App with several outputs, only the pixel type of the first output is returned
+ For an RasterInterface with several outputs, only the pixel type of the first output is returned
"""
if isinstance(inp, str):
@@ -1203,8 +1205,8 @@ def get_pixel_type(inp: str | App) -> str:
if datatype not in datatype_to_pixeltype:
raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}")
pixel_type = getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[datatype]}")
- elif isinstance(inp, App):
- pixel_type = inp.app.GetParameterOutputImagePixelType(inp.output_image_key)
+ elif isinstance(inp, RasterInterface):
+ pixel_type = inp.app.GetParameterOutputImagePixelType(inp.key_output_image)
else:
raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}")
return pixel_type
@@ -1227,8 +1229,8 @@ def parse_pixel_type(pixel_type: str | int) -> int:
raise ValueError(f"Bad pixel type specification ({pixel_type})")
-def is_key_list(pyotb_app: App, key: str) -> bool:
- """Check if a key of the App is an input parameter list."""
+def is_key_list(pyotb_app: RasterInterface, key: str) -> bool:
+ """Check if a key of the RasterInterface is an input parameter list."""
types = (
otb.ParameterType_InputImageList,
otb.ParameterType_StringList,
@@ -1239,12 +1241,12 @@ def is_key_list(pyotb_app: App, key: str) -> bool:
return pyotb_app.app.GetParameterType(key) in types
-def is_key_images_list(pyotb_app: App, key: str) -> bool:
- """Check if a key of the App is an input parameter image list."""
+def is_key_images_list(pyotb_app: RasterInterface, key: str) -> bool:
+ """Check if a key of the RasterInterface is an input parameter image list."""
types = (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList)
return pyotb_app.app.GetParameterType(key) in types
-def get_out_images_param_keys(app: App) -> list[str]:
+def get_out_images_param_keys(app: RasterInterface) -> list[str]:
"""Return every output parameter keys of an OTB app."""
return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage]
--
GitLab
From 89158d2ebf754153da2d8b2538071fa84c851290 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 3 Feb 2023 11:47:39 +0100
Subject: [PATCH 100/399] REFAC: Use RasterInterface when possible
---
pyotb/core.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 8239e3a..81fb51c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1034,7 +1034,8 @@ class LogicalOperation(Operation):
super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation")
self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands)
- def build_fake_expressions(self, operator: str, inputs: list[RasterInterface | str | int | float], nb_bands: int = None):
+ def build_fake_expressions(self, operator: str, inputs: list[RasterInterface | str | int | float],
+ nb_bands: int = None):
"""Create a list of 'fake' expressions, one for each band.
e.g for the operation input1 > input2, we create a fake expression that is like
@@ -1165,7 +1166,7 @@ def get_nbchannels(inp: str | RasterInterface) -> int:
else:
# Executing the app, without printing its log
try:
- info = RasterInterface("ReadImageInfo", inp, quiet=True)
+ info = App("ReadImageInfo", inp, quiet=True)
nb_channels = info.app.GetParameterInt("numberbands")
except Exception as e: # this happens when we pass a str that is not a filepath
raise TypeError(f"Could not get the number of channels of '{inp}'. Not a filepath or wrong filepath") from e
--
GitLab
From 847f0fadd11648e1eee5d4d8e7d6f1944b853621 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 3 Feb 2023 13:48:28 +0100
Subject: [PATCH 101/399] FIX: sync with renamed attrs/cls
---
pyotb/core.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 81fb51c..594fd1f 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1206,8 +1206,8 @@ def get_pixel_type(inp: str | RasterInterface) -> str:
if datatype not in datatype_to_pixeltype:
raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}")
pixel_type = getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[datatype]}")
- elif isinstance(inp, RasterInterface):
- pixel_type = inp.app.GetParameterOutputImagePixelType(inp.key_output_image)
+ elif isinstance(inp, ImageObject):
+ pixel_type = inp.app.GetParameterOutputImagePixelType(inp.output_image_key)
else:
raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}")
return pixel_type
--
GitLab
From 249a8a3958613f0a6bdaa24bd38d6ee7a7e3a808 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Feb 2023 19:35:33 +0100
Subject: [PATCH 102/399] FIX: merge conflicts
---
pyotb/core.py | 36 ++++++++++++++++++------------------
1 file changed, 18 insertions(+), 18 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 594fd1f..3705961 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -315,7 +315,7 @@ class ImageObject(ABC):
for inp in inputs:
if isinstance(inp, (float, int, np.ndarray, np.generic)):
arrays.append(inp)
- elif isinstance(inp, RasterInterface):
+ elif isinstance(inp, ImageObject):
if not inp.exports_dic:
inp.export()
image_dic = inp.exports_dic[inp.output_image_key]
@@ -841,7 +841,7 @@ class Operation(App):
Args:
operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
- *inputs: inputs. Can be RasterInterface, filepath, int or float
+ *inputs: inputs. Can be ImageObject, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
name: override the Operation name
@@ -873,7 +873,7 @@ class Operation(App):
super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True)
self.name = f'Operation exp="{self.exp}"'
- def build_fake_expressions(self, operator: str, inputs: list[RasterInterface | str | int | float],
+ def build_fake_expressions(self, operator: str, inputs: list[ImageObject | str | int | float],
nb_bands: int = None):
"""Create a list of 'fake' expressions, one for each band.
@@ -881,7 +881,7 @@ class Operation(App):
Args:
operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
- inputs: inputs. Can be RasterInterface, filepath, int or float
+ inputs: inputs. Can be ImageObject, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
@@ -956,8 +956,8 @@ class Operation(App):
return exp_bands, ";".join(exp_bands)
@staticmethod
- def make_fake_exp(x: RasterInterface | str, band: int, keep_logical: bool = False) \
- -> tuple[str, list[RasterInterface], int]:
+ def make_fake_exp(x: ImageObject | str, band: int, keep_logical: bool = False) \
+ -> tuple[str, list[ImageObject], int]:
"""This an internal function, only to be used by `build_fake_expressions`.
Enable to create a fake expression just for one input and one band.
@@ -1034,7 +1034,7 @@ class LogicalOperation(Operation):
super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation")
self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands)
- def build_fake_expressions(self, operator: str, inputs: list[RasterInterface | str | int | float],
+ def build_fake_expressions(self, operator: str, inputs: list[ImageObject | str | int | float],
nb_bands: int = None):
"""Create a list of 'fake' expressions, one for each band.
@@ -1043,7 +1043,7 @@ class LogicalOperation(Operation):
Args:
operator: str (one of >, <, >=, <=, ==, !=, &, |)
- inputs: Can be RasterInterface, filepath, int or float
+ inputs: Can be ImageObject, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
@@ -1151,17 +1151,17 @@ class Output(ImageObject):
return f"<pyotb.Output {self.name} object, id {id(self)}>"
-def get_nbchannels(inp: str | RasterInterface) -> int:
+def get_nbchannels(inp: str | ImageObject) -> int:
"""Get the nb of bands of input image.
Args:
- inp: can be filepath or RasterInterface object
+ inp: can be filepath or ImageObject object
Returns:
number of bands in image
"""
- if isinstance(inp, RasterInterface):
+ if isinstance(inp, ImageObject):
nb_channels = inp.shape[-1]
else:
# Executing the app, without printing its log
@@ -1173,7 +1173,7 @@ def get_nbchannels(inp: str | RasterInterface) -> int:
return nb_channels
-def get_pixel_type(inp: str | RasterInterface) -> str:
+def get_pixel_type(inp: str | ImageObject) -> str:
"""Get the encoding of input image pixels.
Args:
@@ -1181,7 +1181,7 @@ def get_pixel_type(inp: str | RasterInterface) -> str:
Returns:
pixel_type: OTB enum e.g. `otbApplication.ImagePixelType_uint8', which actually is an int.
- For an RasterInterface with several outputs, only the pixel type of the first output is returned
+ For an ImageObject with several outputs, only the pixel type of the first output is returned
"""
if isinstance(inp, str):
@@ -1230,8 +1230,8 @@ def parse_pixel_type(pixel_type: str | int) -> int:
raise ValueError(f"Bad pixel type specification ({pixel_type})")
-def is_key_list(pyotb_app: RasterInterface, key: str) -> bool:
- """Check if a key of the RasterInterface is an input parameter list."""
+def is_key_list(pyotb_app: ImageObject, key: str) -> bool:
+ """Check if a key of the ImageObject is an input parameter list."""
types = (
otb.ParameterType_InputImageList,
otb.ParameterType_StringList,
@@ -1242,12 +1242,12 @@ def is_key_list(pyotb_app: RasterInterface, key: str) -> bool:
return pyotb_app.app.GetParameterType(key) in types
-def is_key_images_list(pyotb_app: RasterInterface, key: str) -> bool:
- """Check if a key of the RasterInterface is an input parameter image list."""
+def is_key_images_list(pyotb_app: ImageObject, key: str) -> bool:
+ """Check if a key of the ImageObject is an input parameter image list."""
types = (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList)
return pyotb_app.app.GetParameterType(key) in types
-def get_out_images_param_keys(app: RasterInterface) -> list[str]:
+def get_out_images_param_keys(app: ImageObject) -> list[str]:
"""Return every output parameter keys of an OTB app."""
return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage]
--
GitLab
From 4c3c72961a4f3cc53acba9fde70779c1b6b8ab00 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 2 Feb 2023 21:14:51 +0100
Subject: [PATCH 103/399] FIX: summarize() for RasterInterface
---
pyotb/core.py | 51 ++++++++++++++++++++++++++-------------------------
1 file changed, 26 insertions(+), 25 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 3705961..bdf4eb5 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -338,6 +338,22 @@ class ImageObject(ABC):
return pyotb_app
return NotImplemented
+ def summarize(self) -> dict[str, str | dict[str, Any]]:
+ """Serialize an object and its pipeline into a dictionary.
+
+ Returns:
+ nested dictionary summarizing the pipeline
+
+ """
+ parameters = self.parameters.copy()
+ for key, param in parameters.items():
+ # In the following, we replace each parameter which is an ImageObject, with its summary.
+ if isinstance(param, ImageObject): # single parameter
+ parameters[key] = param.summarize()
+ elif isinstance(param, list): # parameter list
+ parameters[key] = [p.summarize() if isinstance(p, ImageObject) else p for p in param]
+ return {"name": self.app.GetName(), "parameters": parameters}
+
class App(ImageObject):
"""Base class that gathers common operations for any OTB application."""
@@ -465,7 +481,7 @@ class App(ImageObject):
except (RuntimeError, TypeError, ValueError, KeyError) as e:
raise RuntimeError(
f"{self.name}: something went wrong before execution "
- f"(while setting parameter '{key}' to '{obj}')"
+ f"(while setting parameter '{key}' to '{obj}': {e})"
) from e
# Update _parameters using values from OtbApplication object
otb_params = self.app.GetParameters().items()
@@ -536,7 +552,7 @@ class App(ImageObject):
try:
self.app.Execute()
except (RuntimeError, FileNotFoundError) as e:
- raise RuntimeError(f"{self.name}: error during during app execution") from e
+ raise RuntimeError(f"{self.name}: error during during app execution ({e}") from e
self.frozen = False
self._time_end = perf_counter()
logger.debug("%s: execution ended", self.name)
@@ -628,22 +644,6 @@ class App(ImageObject):
logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
return tuple(files)
- def summarize(self) -> dict[str, str | dict[str, Any]]:
- """Serialize an object and its pipeline into a dictionary.
-
- Returns:
- nested dictionary summarizing the pipeline
-
- """
- parameters = self.parameters.copy()
- for key, param in parameters.items():
- # In the following, we replace each parameter which is an ImageObject, with its summary.
- if isinstance(param, ImageObject): # single parameter
- parameters[key] = param.summarize()
- elif isinstance(param, list): # parameter list
- parameters[key] = [p.summarize() if isinstance(p, ImageObject) else p for p in param]
- return {"name": self.app.GetName(), "parameters": parameters}
-
# Private functions
def __parse_args(self, args: list[str | ImageObject | dict | list]) -> dict[str, Any]:
"""Gather all input arguments in kwargs dict.
@@ -1079,16 +1079,16 @@ class LogicalOperation(Operation):
class Input(App):
"""Class for transforming a filepath to pyOTB object."""
- def __init__(self, path: str):
+ def __init__(self, filepath: str):
"""Default constructor.
Args:
- path: Anything supported by GDAL (local file on the filesystem, remote resource e.g. /vsicurl/.., etc.)
+ filepath: Anything supported by GDAL (local file on the filesystem, remote resource e.g. /vsicurl/.., etc.)
"""
- super().__init__("ExtractROI", {"in": path}, frozen=True)
- self.name = f"Input from {path}"
- self.filepath = Path(path)
+ super().__init__("ExtractROI", {"in": filepath}, frozen=True)
+ self.name = f"Input from {filepath}"
+ self.filepath = Path(filepath)
self.propagate_dtype()
self.execute()
@@ -1112,6 +1112,7 @@ class Output(ImageObject):
"""
self.name = f"Output {param_key} from {pyotb_app.name}"
self.parent_pyotb_app = pyotb_app # keep trace of parent app
+ self.parameters = pyotb_app.parameters
self.app = pyotb_app.app
self.exports_dic = pyotb_app.exports_dic
self.param_key = param_key
@@ -1169,7 +1170,7 @@ def get_nbchannels(inp: str | ImageObject) -> int:
info = App("ReadImageInfo", inp, quiet=True)
nb_channels = info.app.GetParameterInt("numberbands")
except Exception as e: # this happens when we pass a str that is not a filepath
- raise TypeError(f"Could not get the number of channels of '{inp}'. Not a filepath or wrong filepath") from e
+ raise TypeError(f"Could not get the number of channels of '{inp}' ({e})") from e
return nb_channels
@@ -1188,7 +1189,7 @@ def get_pixel_type(inp: str | ImageObject) -> str:
try:
info = App("ReadImageInfo", inp, quiet=True)
except Exception as info_err: # this happens when we pass a str that is not a filepath
- raise TypeError(f"Could not get the pixel type of `{inp}`. Not a filepath or wrong filepath") from info_err
+ raise TypeError(f"Could not get the pixel type of `{inp}` ({info_err})") from info_err
datatype = info.app.GetParameterString("datatype") # which is such as short, float...
if not datatype:
raise TypeError(f"Unable to read pixel type of image {inp}")
--
GitLab
From d6de016596440d1facb081b2fa9b8059199456cd Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 2 Feb 2023 22:05:16 +0100
Subject: [PATCH 104/399] ADD: parameters property to RasterInterface
---
pyotb/core.py | 17 ++++++++++++++---
1 file changed, 14 insertions(+), 3 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index bdf4eb5..37a56bc 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -354,6 +354,10 @@ class ImageObject(ABC):
parameters[key] = [p.summarize() if isinstance(p, ImageObject) else p for p in param]
return {"name": self.app.GetName(), "parameters": parameters}
+ @abstractmethod
+ def parameters(self):
+ """Parameters"""
+
class App(ImageObject):
"""Base class that gathers common operations for any OTB application."""
@@ -385,7 +389,7 @@ class App(ImageObject):
self.image_dic = image_dic
self._time_start, self._time_end = 0, 0
self.exports_dic = {}
- self.parameters = {}
+ self._parameters = {}
# Initialize app, set parameters and execute if not frozen
create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
self.app = create(name)
@@ -446,6 +450,10 @@ class App(ImageObject):
data_dict[str(key)] = value
return data_dict
+ @property
+ def parameters(self):
+ return self._parameters
+
def set_parameters(self, *args, **kwargs):
"""Set some parameters of the app.
@@ -487,7 +495,7 @@ class App(ImageObject):
otb_params = self.app.GetParameters().items()
otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params}
# Update param dict and save values as object attributes
- self.parameters.update({**parameters, **otb_params})
+ self._parameters.update({**parameters, **otb_params})
self.save_objects()
def propagate_dtype(self, target_key: str = None, dtype: int = None):
@@ -1112,7 +1120,6 @@ class Output(ImageObject):
"""
self.name = f"Output {param_key} from {pyotb_app.name}"
self.parent_pyotb_app = pyotb_app # keep trace of parent app
- self.parameters = pyotb_app.parameters
self.app = pyotb_app.app
self.exports_dic = pyotb_app.exports_dic
self.param_key = param_key
@@ -1129,6 +1136,10 @@ class Output(ImageObject):
"""Force the right key to be used when accessing the ImageObject."""
return self.param_key
+ @property
+ def parameters(self):
+ return self.parent_pyotb_app.parameters
+
def exists(self) -> bool:
"""Check file exist."""
if self.filepath is None:
--
GitLab
From ff71b801656f56db1f7976f34a98f066c7c0f71d Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 2 Feb 2023 22:15:09 +0100
Subject: [PATCH 105/399] ADD: new test to prevent bug
---
tests/test_core.py | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/tests/test_core.py b/tests/test_core.py
index 58ddeac..7bed905 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -166,3 +166,14 @@ def test_ndvi_comparison():
thresholded_bandmath = pyotb.where(ndvi_bandmath >= 0.3, 1, 0)
assert thresholded_bandmath.exp == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)"
+
+
+def test_output_in_arg():
+ o = pyotb.Output(INPUT, "out")
+ t = pyotb.ReadImageInfo(o)
+ assert t
+
+
+def test_output_summary():
+ o = pyotb.Output(INPUT, "out")
+ assert o.summarize()
--
GitLab
From c329fc1bed3a92f25132ada7202e7d07d3d45e81 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 2 Feb 2023 22:15:23 +0100
Subject: [PATCH 106/399] STYLE: linter
---
pyotb/core.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 37a56bc..b6e6228 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -356,7 +356,7 @@ class ImageObject(ABC):
@abstractmethod
def parameters(self):
- """Parameters"""
+ """Parameters."""
class App(ImageObject):
@@ -452,6 +452,7 @@ class App(ImageObject):
@property
def parameters(self):
+ """Parameters."""
return self._parameters
def set_parameters(self, *args, **kwargs):
@@ -1138,6 +1139,7 @@ class Output(ImageObject):
@property
def parameters(self):
+ """Parameters."""
return self.parent_pyotb_app.parameters
def exists(self) -> bool:
--
GitLab
From 83c1d7a0db7b6aafbafe6421e5c389bf058f08bb Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 2 Feb 2023 22:16:53 +0100
Subject: [PATCH 107/399] REFAC: parameters abstract property in
RasterInterface
---
pyotb/core.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/pyotb/core.py b/pyotb/core.py
index b6e6228..3a969f1 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -355,6 +355,7 @@ class ImageObject(ABC):
return {"name": self.app.GetName(), "parameters": parameters}
@abstractmethod
+ @property
def parameters(self):
"""Parameters."""
--
GitLab
From 3ab4fe2c0dc09e8b35fd7cb7e45394f4e4c71a63 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 2 Feb 2023 22:26:09 +0100
Subject: [PATCH 108/399] REFAC: parameters abstract property in
RasterInterface
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 3a969f1..e363caf 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -354,8 +354,8 @@ class ImageObject(ABC):
parameters[key] = [p.summarize() if isinstance(p, ImageObject) else p for p in param]
return {"name": self.app.GetName(), "parameters": parameters}
- @abstractmethod
@property
+ @abstractmethod
def parameters(self):
"""Parameters."""
--
GitLab
From 5083e62f2a89f00b6e8b015916db1b3b6cb6e64f Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 3 Feb 2023 13:11:10 +0100
Subject: [PATCH 109/399] ADD: name in RasterInterface
---
pyotb/core.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/pyotb/core.py b/pyotb/core.py
index e363caf..f035979 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -17,6 +17,7 @@ from .helpers import logger
class ImageObject(ABC):
"""Abstraction of an image object."""
+ name: str
app: otb.Application
exports_dic: dict
--
GitLab
From 733e032c4b3a2ba31c1a9ee0c338c219fc82a5ae Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 3 Feb 2023 13:35:29 +0100
Subject: [PATCH 110/399] REFAC: add parameters as attribute rather than
property
---
pyotb/core.py | 24 ++++++------------------
1 file changed, 6 insertions(+), 18 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index f035979..01f1a91 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -19,6 +19,7 @@ class ImageObject(ABC):
name: str
app: otb.Application
+ parameters: dict
exports_dic: dict
@property
@@ -355,10 +356,6 @@ class ImageObject(ABC):
parameters[key] = [p.summarize() if isinstance(p, ImageObject) else p for p in param]
return {"name": self.app.GetName(), "parameters": parameters}
- @property
- @abstractmethod
- def parameters(self):
- """Parameters."""
class App(ImageObject):
@@ -391,7 +388,7 @@ class App(ImageObject):
self.image_dic = image_dic
self._time_start, self._time_end = 0, 0
self.exports_dic = {}
- self._parameters = {}
+ self.parameters = {}
# Initialize app, set parameters and execute if not frozen
create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
self.app = create(name)
@@ -422,7 +419,7 @@ class App(ImageObject):
@property
def key_input_image(self) -> str:
- """Get the name of first input image parameter."""
+ """Name of the first input image parameter."""
return self.get_first_key(param_types=[otb.ParameterType_InputImage, otb.ParameterType_InputImageList])
@property
@@ -452,11 +449,6 @@ class App(ImageObject):
data_dict[str(key)] = value
return data_dict
- @property
- def parameters(self):
- """Parameters."""
- return self._parameters
-
def set_parameters(self, *args, **kwargs):
"""Set some parameters of the app.
@@ -498,7 +490,7 @@ class App(ImageObject):
otb_params = self.app.GetParameters().items()
otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params}
# Update param dict and save values as object attributes
- self._parameters.update({**parameters, **otb_params})
+ self.parameters.update({**parameters, **otb_params})
self.save_objects()
def propagate_dtype(self, target_key: str = None, dtype: int = None):
@@ -844,7 +836,7 @@ class Operation(App):
"""
- def __init__(self, operator: str, *inputs, nb_bands: int = None, name: str = None):
+ def __init__(self, operator: str, *inputs, nb_bands: int = None):
"""Given some inputs and an operator, this function enables to transform this into an OTB application.
Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator.
@@ -1126,6 +1118,7 @@ class Output(ImageObject):
self.app = pyotb_app.app
self.exports_dic = pyotb_app.exports_dic
self.param_key = param_key
+ self.parameters = self.parent_pyotb_app.parameters
self.filepath = None
if filepath:
if "?" in filepath:
@@ -1139,11 +1132,6 @@ class Output(ImageObject):
"""Force the right key to be used when accessing the ImageObject."""
return self.param_key
- @property
- def parameters(self):
- """Parameters."""
- return self.parent_pyotb_app.parameters
-
def exists(self) -> bool:
"""Check file exist."""
if self.filepath is None:
--
GitLab
From 683b75cfa40b48ed6ef5fd5af68f9008102a68ee Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 3 Feb 2023 13:46:33 +0100
Subject: [PATCH 111/399] FIX: sync with renamed attrs/cls
---
pyotb/core.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 01f1a91..a174e9c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -836,7 +836,7 @@ class Operation(App):
"""
- def __init__(self, operator: str, *inputs, nb_bands: int = None):
+ def __init__(self, operator: str, *inputs, nb_bands: int = None, name: str = None):
"""Given some inputs and an operator, this function enables to transform this into an OTB application.
Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator.
@@ -874,7 +874,7 @@ class Operation(App):
appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX"
# Execute app
super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True)
- self.name = f'Operation exp="{self.exp}"'
+ self.name = name or f'Operation exp="{self.exp}"'
def build_fake_expressions(self, operator: str, inputs: list[ImageObject | str | int | float],
nb_bands: int = None):
--
GitLab
From bb9ce48bf66c7476c4d6d78081803396c462d49a Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Feb 2023 17:32:55 +0000
Subject: [PATCH 112/399] BUG: App init error will raise RuntimeError
---
pyotb/core.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index a174e9c..319dc64 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1172,7 +1172,7 @@ def get_nbchannels(inp: str | ImageObject) -> int:
try:
info = App("ReadImageInfo", inp, quiet=True)
nb_channels = info.app.GetParameterInt("numberbands")
- except Exception as e: # this happens when we pass a str that is not a filepath
+ except RuntimeError as e: # this happens when we pass a str that is not a filepath
raise TypeError(f"Could not get the number of channels of '{inp}' ({e})") from e
return nb_channels
@@ -1191,7 +1191,7 @@ def get_pixel_type(inp: str | ImageObject) -> str:
if isinstance(inp, str):
try:
info = App("ReadImageInfo", inp, quiet=True)
- except Exception as info_err: # this happens when we pass a str that is not a filepath
+ except RuntimeError as info_err: # this happens when we pass a str that is not a filepath
raise TypeError(f"Could not get the pixel type of `{inp}` ({info_err})") from info_err
datatype = info.app.GetParameterString("datatype") # which is such as short, float...
if not datatype:
--
GitLab
From 819c3bf681abd096872f16f7803c9c5a92a4a5cb Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Feb 2023 17:34:53 +0000
Subject: [PATCH 113/399] ENH: rename error e to info_err
---
pyotb/core.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 319dc64..b9fa376 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1172,8 +1172,8 @@ def get_nbchannels(inp: str | ImageObject) -> int:
try:
info = App("ReadImageInfo", inp, quiet=True)
nb_channels = info.app.GetParameterInt("numberbands")
- except RuntimeError as e: # this happens when we pass a str that is not a filepath
- raise TypeError(f"Could not get the number of channels of '{inp}' ({e})") from e
+ except RuntimeError as info_err: # this happens when we pass a str that is not a filepath
+ raise TypeError(f"Could not get the number of channels of '{inp}' ({info_err})") from info_err
return nb_channels
--
GitLab
From c9b25997804ef721cff3f05b7353500c23516ca3 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Feb 2023 20:09:32 +0000
Subject: [PATCH 114/399] Back to OTBObject
---
pyotb/core.py | 97 +++++++++++++++++++++++++--------------------------
1 file changed, 48 insertions(+), 49 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index b9fa376..a1bf285 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -14,7 +14,7 @@ import otbApplication as otb # pylint: disable=import-error
from .helpers import logger
-class ImageObject(ABC):
+class OTBObject(ABC):
"""Abstraction of an image object."""
name: str
@@ -214,35 +214,35 @@ class ImageObject(ABC):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return op_cls(name, x, y)
- def __add__(self, other: ImageObject | str | int | float) -> Operation:
+ def __add__(self, other: OTBObject | str | int | float) -> Operation:
"""Addition."""
return self._create_operator(Operation, "+", self, other)
- def __sub__(self, other: ImageObject | str | int | float) -> Operation:
+ def __sub__(self, other: OTBObject | str | int | float) -> Operation:
"""Subtraction."""
return self._create_operator(Operation, "-", self, other)
- def __mul__(self, other: ImageObject | str | int | float) -> Operation:
+ def __mul__(self, other: OTBObject | str | int | float) -> Operation:
"""Multiplication."""
return self._create_operator(Operation, "*", self, other)
- def __truediv__(self, other: ImageObject | str | int | float) -> Operation:
+ def __truediv__(self, other: OTBObject | str | int | float) -> Operation:
"""Division."""
return self._create_operator(Operation, "/", self, other)
- def __radd__(self, other: ImageObject | str | int | float) -> Operation:
+ def __radd__(self, other: OTBObject | str | int | float) -> Operation:
"""Right addition."""
return self._create_operator(Operation, "+", other, self)
- def __rsub__(self, other: ImageObject | str | int | float) -> Operation:
+ def __rsub__(self, other: OTBObject | str | int | float) -> Operation:
"""Right subtraction."""
return self._create_operator(Operation, "-", other, self)
- def __rmul__(self, other: ImageObject | str | int | float) -> Operation:
+ def __rmul__(self, other: OTBObject | str | int | float) -> Operation:
"""Right multiplication."""
return self._create_operator(Operation, "*", other, self)
- def __rtruediv__(self, other: ImageObject | str | int | float) -> Operation:
+ def __rtruediv__(self, other: OTBObject | str | int | float) -> Operation:
"""Right division."""
return self._create_operator(Operation, "/", other, self)
@@ -250,35 +250,35 @@ class ImageObject(ABC):
"""Absolute value."""
return Operation("abs", self)
- def __ge__(self, other: ImageObject | str | int | float) -> LogicalOperation:
+ def __ge__(self, other: OTBObject | str | int | float) -> LogicalOperation:
"""Greater of equal than."""
return self._create_operator(LogicalOperation, ">=", self, other)
- def __le__(self, other: ImageObject | str | int | float) -> LogicalOperation:
+ def __le__(self, other: OTBObject | str | int | float) -> LogicalOperation:
"""Lower of equal than."""
return self._create_operator(LogicalOperation, "<=", self, other)
- def __gt__(self, other: ImageObject | str | int | float) -> LogicalOperation:
+ def __gt__(self, other: OTBObject | str | int | float) -> LogicalOperation:
"""Greater than."""
return self._create_operator(LogicalOperation, ">", self, other)
- def __lt__(self, other: ImageObject | str | int | float) -> LogicalOperation:
+ def __lt__(self, other: OTBObject | str | int | float) -> LogicalOperation:
"""Lower than."""
return self._create_operator(LogicalOperation, "<", self, other)
- def __eq__(self, other: ImageObject | str | int | float) -> LogicalOperation:
+ def __eq__(self, other: OTBObject | str | int | float) -> LogicalOperation:
"""Equality."""
return self._create_operator(LogicalOperation, "==", self, other)
- def __ne__(self, other: ImageObject | str | int | float) -> LogicalOperation:
+ def __ne__(self, other: OTBObject | str | int | float) -> LogicalOperation:
"""Inequality."""
return self._create_operator(LogicalOperation, "!=", self, other)
- def __or__(self, other: ImageObject | str | int | float) -> LogicalOperation:
+ def __or__(self, other: OTBObject | str | int | float) -> LogicalOperation:
"""Logical or."""
return self._create_operator(LogicalOperation, "||", self, other)
- def __and__(self, other: ImageObject | str | int | float) -> LogicalOperation:
+ def __and__(self, other: OTBObject | str | int | float) -> LogicalOperation:
"""Logical and."""
return self._create_operator(LogicalOperation, "&&", self, other)
@@ -317,7 +317,7 @@ class ImageObject(ABC):
for inp in inputs:
if isinstance(inp, (float, int, np.ndarray, np.generic)):
arrays.append(inp)
- elif isinstance(inp, ImageObject):
+ elif isinstance(inp, OTBObject):
if not inp.exports_dic:
inp.export()
image_dic = inp.exports_dic[inp.output_image_key]
@@ -349,16 +349,15 @@ class ImageObject(ABC):
"""
parameters = self.parameters.copy()
for key, param in parameters.items():
- # In the following, we replace each parameter which is an ImageObject, with its summary.
- if isinstance(param, ImageObject): # single parameter
+ # In the following, we replace each parameter which is an OTBObject, with its summary.
+ if isinstance(param, OTBObject): # single parameter
parameters[key] = param.summarize()
elif isinstance(param, list): # parameter list
- parameters[key] = [p.summarize() if isinstance(p, ImageObject) else p for p in param]
+ parameters[key] = [p.summarize() if isinstance(p, OTBObject) else p for p in param]
return {"name": self.app.GetName(), "parameters": parameters}
-
-class App(ImageObject):
+class App(OTBObject):
"""Base class that gathers common operations for any OTB application."""
def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, **kwargs):
@@ -458,7 +457,7 @@ class App(ImageObject):
Args:
*args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
(e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
- - string or ImageObject, useful when the user implicitly wants to set the param "in"
+ - string or OTBObject, useful when the user implicitly wants to set the param "in"
- list, useful when the user implicitly wants to set the param "il"
**kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif'
@@ -648,7 +647,7 @@ class App(ImageObject):
return tuple(files)
# Private functions
- def __parse_args(self, args: list[str | ImageObject | dict | list]) -> dict[str, Any]:
+ def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]:
"""Gather all input arguments in kwargs dict.
Args:
@@ -662,17 +661,17 @@ class App(ImageObject):
for arg in args:
if isinstance(arg, dict):
kwargs.update(arg)
- elif isinstance(arg, (str, ImageObject)) or isinstance(arg, list) and is_key_list(self, self.key_input):
+ elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.key_input):
kwargs.update({self.key_input: arg})
return kwargs
- def __set_param(self, key: str, obj: list | tuple | ImageObject | otb.Application | list[Any]):
+ def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]):
"""Set one parameter, decide which otb.Application method to use depending on target object."""
if obj is None or (isinstance(obj, (list, tuple)) and not obj):
self.app.ClearValue(key)
return
# Single-parameter cases
- if isinstance(obj, ImageObject):
+ if isinstance(obj, OTBObject):
self.app.ConnectImage(key, obj.app, obj.output_image_key)
elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB
self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0])
@@ -684,7 +683,7 @@ class App(ImageObject):
elif is_key_images_list(self, key):
# To enable possible in-memory connections, we go through the list and set the parameters one by one
for inp in obj:
- if isinstance(inp, ImageObject):
+ if isinstance(inp, OTBObject):
self.app.ConnectImage(key, inp.app, inp.output_image_key)
elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB
self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0])
@@ -844,7 +843,7 @@ class Operation(App):
Args:
operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
- *inputs: inputs. Can be ImageObject, filepath, int or float
+ *inputs: inputs. Can be OTBObject, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
name: override the Operation name
@@ -876,7 +875,7 @@ class Operation(App):
super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True)
self.name = name or f'Operation exp="{self.exp}"'
- def build_fake_expressions(self, operator: str, inputs: list[ImageObject | str | int | float],
+ def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float],
nb_bands: int = None):
"""Create a list of 'fake' expressions, one for each band.
@@ -884,7 +883,7 @@ class Operation(App):
Args:
operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
- inputs: inputs. Can be ImageObject, filepath, int or float
+ inputs: inputs. Can be OTBObject, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
@@ -959,8 +958,8 @@ class Operation(App):
return exp_bands, ";".join(exp_bands)
@staticmethod
- def make_fake_exp(x: ImageObject | str, band: int, keep_logical: bool = False) \
- -> tuple[str, list[ImageObject], int]:
+ def make_fake_exp(x: OTBObject | str, band: int, keep_logical: bool = False) \
+ -> tuple[str, list[OTBObject], int]:
"""This an internal function, only to be used by `build_fake_expressions`.
Enable to create a fake expression just for one input and one band.
@@ -1037,7 +1036,7 @@ class LogicalOperation(Operation):
super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation")
self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands)
- def build_fake_expressions(self, operator: str, inputs: list[ImageObject | str | int | float],
+ def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float],
nb_bands: int = None):
"""Create a list of 'fake' expressions, one for each band.
@@ -1046,7 +1045,7 @@ class LogicalOperation(Operation):
Args:
operator: str (one of >, <, >=, <=, ==, !=, &, |)
- inputs: Can be ImageObject, filepath, int or float
+ inputs: Can be OTBObject, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
"""
@@ -1100,7 +1099,7 @@ class Input(App):
return f"<pyotb.Input object from {self.filepath}>"
-class Output(ImageObject):
+class Output(OTBObject):
"""Object that behave like a pointer to a specific application output file."""
def __init__(self, pyotb_app: App, param_key: str = None, filepath: str = None, mkdir: bool = True):
@@ -1129,7 +1128,7 @@ class Output(ImageObject):
@property
def output_image_key(self) -> str:
- """Force the right key to be used when accessing the ImageObject."""
+ """Force the right key to be used when accessing the OTBObject."""
return self.param_key
def exists(self) -> bool:
@@ -1155,17 +1154,17 @@ class Output(ImageObject):
return f"<pyotb.Output {self.name} object, id {id(self)}>"
-def get_nbchannels(inp: str | ImageObject) -> int:
+def get_nbchannels(inp: str | OTBObject) -> int:
"""Get the nb of bands of input image.
Args:
- inp: can be filepath or ImageObject object
+ inp: can be filepath or OTBObject object
Returns:
number of bands in image
"""
- if isinstance(inp, ImageObject):
+ if isinstance(inp, OTBObject):
nb_channels = inp.shape[-1]
else:
# Executing the app, without printing its log
@@ -1177,7 +1176,7 @@ def get_nbchannels(inp: str | ImageObject) -> int:
return nb_channels
-def get_pixel_type(inp: str | ImageObject) -> str:
+def get_pixel_type(inp: str | OTBObject) -> str:
"""Get the encoding of input image pixels.
Args:
@@ -1185,7 +1184,7 @@ def get_pixel_type(inp: str | ImageObject) -> str:
Returns:
pixel_type: OTB enum e.g. `otbApplication.ImagePixelType_uint8', which actually is an int.
- For an ImageObject with several outputs, only the pixel type of the first output is returned
+ For an OTBObject with several outputs, only the pixel type of the first output is returned
"""
if isinstance(inp, str):
@@ -1210,7 +1209,7 @@ def get_pixel_type(inp: str | ImageObject) -> str:
if datatype not in datatype_to_pixeltype:
raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}")
pixel_type = getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[datatype]}")
- elif isinstance(inp, ImageObject):
+ elif isinstance(inp, OTBObject):
pixel_type = inp.app.GetParameterOutputImagePixelType(inp.output_image_key)
else:
raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}")
@@ -1234,8 +1233,8 @@ def parse_pixel_type(pixel_type: str | int) -> int:
raise ValueError(f"Bad pixel type specification ({pixel_type})")
-def is_key_list(pyotb_app: ImageObject, key: str) -> bool:
- """Check if a key of the ImageObject is an input parameter list."""
+def is_key_list(pyotb_app: OTBObject, key: str) -> bool:
+ """Check if a key of the OTBObject is an input parameter list."""
types = (
otb.ParameterType_InputImageList,
otb.ParameterType_StringList,
@@ -1246,12 +1245,12 @@ def is_key_list(pyotb_app: ImageObject, key: str) -> bool:
return pyotb_app.app.GetParameterType(key) in types
-def is_key_images_list(pyotb_app: ImageObject, key: str) -> bool:
- """Check if a key of the ImageObject is an input parameter image list."""
+def is_key_images_list(pyotb_app: OTBObject, key: str) -> bool:
+ """Check if a key of the OTBObject is an input parameter image list."""
types = (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList)
return pyotb_app.app.GetParameterType(key) in types
-def get_out_images_param_keys(app: ImageObject) -> list[str]:
+def get_out_images_param_keys(app: OTBObject) -> list[str]:
"""Return every output parameter keys of an OTB app."""
return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage]
--
GitLab
From ecce6d984b46494e45b2882d95aa7c4132681df0 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 4 Feb 2023 00:23:17 +0100
Subject: [PATCH 115/399] FIX: __getitem__ in OTBObject
---
pyotb/core.py | 110 +++++++++++++++++++++++++-------------------------
1 file changed, 55 insertions(+), 55 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index a1bf285..b80fbcb 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -196,8 +196,24 @@ class OTBObject(ABC):
row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x
return abs(int(row)), int(col)
+ def summarize(self) -> dict[str, str | dict[str, Any]]:
+ """Serialize an object and its pipeline into a dictionary.
+
+ Returns:
+ nested dictionary summarizing the pipeline
+
+ """
+ parameters = self.parameters.copy()
+ for key, param in parameters.items():
+ # In the following, we replace each parameter which is an OTBObject, with its summary.
+ if isinstance(param, OTBObject): # single parameter
+ parameters[key] = param.summarize()
+ elif isinstance(param, list): # parameter list
+ parameters[key] = [p.summarize() if isinstance(p, OTBObject) else p for p in param]
+ return {"name": self.app.GetName(), "parameters": parameters}
+
@staticmethod
- def _create_operator(op_cls, name, x, y) -> Operation:
+ def __create_operator(op_cls, name, x, y) -> Operation:
"""Create an operator.
Args:
@@ -214,6 +230,44 @@ class OTBObject(ABC):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return op_cls(name, x, y)
+ def __getitem__(self, key) -> Any | list[int | float] | int | float | Slicer:
+ """Override the default __getitem__ behaviour.
+
+ This function enables 2 things :
+ - access attributes like that : object['any_attribute']
+ - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3]
+ selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]]
+ selecting 1000x1000 subset : object[:1000, :1000]
+ - access pixel value(s) at a specified row, col index
+
+ Args:
+ key: attribute key
+
+ Returns:
+ attribute, pixel values or Slicer
+
+ """
+ # Accessing string attributes
+ if isinstance(key, str):
+ return getattr(self, key)
+ # Accessing pixel value(s) using Y/X coordinates
+ if isinstance(key, tuple) and len(key) >= 2:
+ row, col = key[0], key[1]
+ if isinstance(row, int) and isinstance(col, int):
+ if row < 0 or col < 0:
+ raise ValueError(f"{self.name}: can't read pixel value at negative coordinates ({row}, {col})")
+ channels = None
+ if len(key) == 3:
+ channels = key[2]
+ return self.get_values_at_coords(row, col, channels)
+ # Slicing
+ if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)):
+ raise ValueError(f'"{key}"cannot be interpreted as valid slicing. Slicing should be 2D or 3D.')
+ if isinstance(key, tuple) and len(key) == 2:
+ # Adding a 3rd dimension
+ key = key + (slice(None, None, None),)
+ return Slicer(self, *key)
+
def __add__(self, other: OTBObject | str | int | float) -> Operation:
"""Addition."""
return self._create_operator(Operation, "+", self, other)
@@ -340,22 +394,6 @@ class OTBObject(ABC):
return pyotb_app
return NotImplemented
- def summarize(self) -> dict[str, str | dict[str, Any]]:
- """Serialize an object and its pipeline into a dictionary.
-
- Returns:
- nested dictionary summarizing the pipeline
-
- """
- parameters = self.parameters.copy()
- for key, param in parameters.items():
- # In the following, we replace each parameter which is an OTBObject, with its summary.
- if isinstance(param, OTBObject): # single parameter
- parameters[key] = param.summarize()
- elif isinstance(param, list): # parameter list
- parameters[key] = [p.summarize() if isinstance(p, OTBObject) else p for p in param]
- return {"name": self.app.GetName(), "parameters": parameters}
-
class App(OTBObject):
"""Base class that gathers common operations for any OTB application."""
@@ -704,44 +742,6 @@ class App(OTBObject):
"""
return id(self)
- def __getitem__(self, key) -> Any | list[int | float] | int | float | Slicer:
- """Override the default __getitem__ behaviour.
-
- This function enables 2 things :
- - access attributes like that : object['any_attribute']
- - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3]
- selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]]
- selecting 1000x1000 subset : object[:1000, :1000]
- - access pixel value(s) at a specified row, col index
-
- Args:
- key: attribute key
-
- Returns:
- attribute, pixel values or Slicer
-
- """
- # Accessing string attributes
- if isinstance(key, str):
- return getattr(self, key)
- # Accessing pixel value(s) using Y/X coordinates
- if isinstance(key, tuple) and len(key) >= 2:
- row, col = key[0], key[1]
- if isinstance(row, int) and isinstance(col, int):
- if row < 0 or col < 0:
- raise ValueError(f"{self.name}: can't read pixel value at negative coordinates ({row}, {col})")
- channels = None
- if len(key) == 3:
- channels = key[2]
- return self.get_values_at_coords(row, col, channels)
- # Slicing
- if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)):
- raise ValueError(f'"{key}"cannot be interpreted as valid slicing. Slicing should be 2D or 3D.')
- if isinstance(key, tuple) and len(key) == 2:
- # Adding a 3rd dimension
- key = key + (slice(None, None, None),)
- return Slicer(self, *key)
-
def __str__(self) -> str:
"""Return a nice string representation with object id."""
return f"<pyotb.App {self.name} object id {id(self)}>"
--
GitLab
From c4aad2f7c2c648aac54f5c34fe4dfe4c10394cff Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 4 Feb 2023 16:35:08 +0100
Subject: [PATCH 116/399] ENH: input and outpout keys
---
pyotb/core.py | 40 +++++++++++++++++++++++++---------------
1 file changed, 25 insertions(+), 15 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index b80fbcb..1395e4e 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -440,29 +440,39 @@ class App(OTBObject):
if any(key in self.parameters for key in self._out_param_types):
self.flush() # auto flush if any output param was provided during app init
- def get_first_key(self, param_types: list[int]) -> str:
- """Get the first output param key for specific file types."""
- for key, param_type in sorted(self._all_param_types.items()):
- if param_type in param_types:
- return key
+ def get_first_key(self, *args: tuple[list[int]]) -> str:
+ """Get the first param key for specific file types, try each list in args."""
+ for param_types in args:
+ for key, param_type in sorted(self._all_param_types.items()):
+ if param_type in param_types:
+ return key
return None
@property
- def key_input(self) -> str:
+ def input_key(self) -> str:
"""Get the name of first input parameter, raster > vector > file."""
- return self.get_first_key([otb.ParameterType_InputImage, otb.ParameterType_InputImageList]) \
- or self.get_first_key([otb.ParameterType_InputVectorData, otb.ParameterType_InputVectorDataList]) \
- or self.get_first_key([otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList])
+ return self.get_first_key(
+ [otb.ParameterType_InputImage, otb.ParameterType_InputImageList],
+ [otb.ParameterType_InputVectorData, otb.ParameterType_InputVectorDataList],
+ [otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList]
+ )
@property
- def key_input_image(self) -> str:
+ def input_image_key(self) -> str:
"""Name of the first input image parameter."""
- return self.get_first_key(param_types=[otb.ParameterType_InputImage, otb.ParameterType_InputImageList])
+ return self.get_first_key([otb.ParameterType_InputImage, otb.ParameterType_InputImageList])
+
+ @property
+ def output_key(self) -> str:
+ """Name of the first output parameter, raster > vector > file."""
+ return self.get_first_key(
+ [otb.ParameterType_OutputImage], [otb.ParameterType_OutputVectorData], [otb.ParameterType_OutputFilename]
+ )
@property
def output_image_key(self) -> str:
"""Get the name of first output image parameter."""
- return self.get_first_key(param_types=[otb.ParameterType_OutputImage])
+ return self.get_first_key([otb.ParameterType_OutputImage])
@property
def elapsed_time(self) -> float:
@@ -542,7 +552,7 @@ class App(OTBObject):
"""
if not dtype:
- param = self.parameters.get(self.key_input_image)
+ param = self.parameters.get(self.input_image_key)
if not param:
logger.warning("%s: could not propagate pixel type from inputs to output", self.name)
return
@@ -699,8 +709,8 @@ class App(OTBObject):
for arg in args:
if isinstance(arg, dict):
kwargs.update(arg)
- elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.key_input):
- kwargs.update({self.key_input: arg})
+ elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.input_key):
+ kwargs.update({self.input_key: arg})
return kwargs
def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]):
--
GitLab
From 39a715cafcb0c1fab2a064ae23db4a97083820b4 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 4 Feb 2023 16:35:20 +0100
Subject: [PATCH 117/399] ENH: add _ to _create_operator
---
pyotb/core.py | 32 ++++++++++++++++----------------
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 1395e4e..995b975 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -270,35 +270,35 @@ class OTBObject(ABC):
def __add__(self, other: OTBObject | str | int | float) -> Operation:
"""Addition."""
- return self._create_operator(Operation, "+", self, other)
+ return self.__create_operator(Operation, "+", self, other)
def __sub__(self, other: OTBObject | str | int | float) -> Operation:
"""Subtraction."""
- return self._create_operator(Operation, "-", self, other)
+ return self.__create_operator(Operation, "-", self, other)
def __mul__(self, other: OTBObject | str | int | float) -> Operation:
"""Multiplication."""
- return self._create_operator(Operation, "*", self, other)
+ return self.__create_operator(Operation, "*", self, other)
def __truediv__(self, other: OTBObject | str | int | float) -> Operation:
"""Division."""
- return self._create_operator(Operation, "/", self, other)
+ return self.__create_operator(Operation, "/", self, other)
def __radd__(self, other: OTBObject | str | int | float) -> Operation:
"""Right addition."""
- return self._create_operator(Operation, "+", other, self)
+ return self.__create_operator(Operation, "+", other, self)
def __rsub__(self, other: OTBObject | str | int | float) -> Operation:
"""Right subtraction."""
- return self._create_operator(Operation, "-", other, self)
+ return self.__create_operator(Operation, "-", other, self)
def __rmul__(self, other: OTBObject | str | int | float) -> Operation:
"""Right multiplication."""
- return self._create_operator(Operation, "*", other, self)
+ return self.__create_operator(Operation, "*", other, self)
def __rtruediv__(self, other: OTBObject | str | int | float) -> Operation:
"""Right division."""
- return self._create_operator(Operation, "/", other, self)
+ return self.__create_operator(Operation, "/", other, self)
def __abs__(self) -> Operation:
"""Absolute value."""
@@ -306,35 +306,35 @@ class OTBObject(ABC):
def __ge__(self, other: OTBObject | str | int | float) -> LogicalOperation:
"""Greater of equal than."""
- return self._create_operator(LogicalOperation, ">=", self, other)
+ return self.__create_operator(LogicalOperation, ">=", self, other)
def __le__(self, other: OTBObject | str | int | float) -> LogicalOperation:
"""Lower of equal than."""
- return self._create_operator(LogicalOperation, "<=", self, other)
+ return self.__create_operator(LogicalOperation, "<=", self, other)
def __gt__(self, other: OTBObject | str | int | float) -> LogicalOperation:
"""Greater than."""
- return self._create_operator(LogicalOperation, ">", self, other)
+ return self.__create_operator(LogicalOperation, ">", self, other)
def __lt__(self, other: OTBObject | str | int | float) -> LogicalOperation:
"""Lower than."""
- return self._create_operator(LogicalOperation, "<", self, other)
+ return self.__create_operator(LogicalOperation, "<", self, other)
def __eq__(self, other: OTBObject | str | int | float) -> LogicalOperation:
"""Equality."""
- return self._create_operator(LogicalOperation, "==", self, other)
+ return self.__create_operator(LogicalOperation, "==", self, other)
def __ne__(self, other: OTBObject | str | int | float) -> LogicalOperation:
"""Inequality."""
- return self._create_operator(LogicalOperation, "!=", self, other)
+ return self.__create_operator(LogicalOperation, "!=", self, other)
def __or__(self, other: OTBObject | str | int | float) -> LogicalOperation:
"""Logical or."""
- return self._create_operator(LogicalOperation, "||", self, other)
+ return self.__create_operator(LogicalOperation, "||", self, other)
def __and__(self, other: OTBObject | str | int | float) -> LogicalOperation:
"""Logical and."""
- return self._create_operator(LogicalOperation, "&&", self, other)
+ return self.__create_operator(LogicalOperation, "&&", self, other)
# Some other operations could be implemented with the same pattern
# e.g. __pow__... cf https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types
--
GitLab
From 536f24c9f4a8b2c104f0c8fc83b89cb97dbb8657 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 4 Feb 2023 19:05:54 +0100
Subject: [PATCH 118/399] ENH: refac save_objects - add __getattr__ and remove
data property
---
pyotb/core.py | 107 +++++++++++++++++++++++---------------------------
1 file changed, 50 insertions(+), 57 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 995b975..358ad1e 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -230,11 +230,32 @@ class OTBObject(ABC):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return op_cls(name, x, y)
- def __getitem__(self, key) -> Any | list[int | float] | int | float | Slicer:
+ # Special functions
+ def __hash__(self) -> int:
+ """Override the default behaviour of the hash function.
+
+ Returns:
+ self hash
+
+ """
+ return id(self)
+
+ def __getattr__(self, key: str) -> Any:
+ """Return object attribute, or if key is found in self.parameters or self.data."""
+ if key in dir(self):
+ return self.__dict__[key]
+ if key in self._out_param_types and key in self.parameters:
+ return Output(self, key, self.parameters[key])
+ if key in self.parameters:
+ return self.parameters[key]
+ if key in self.data:
+ return self.data[key]
+ raise AttributeError(f"{self.name}: unknown attribute '{key}'")
+
+ def __getitem__(self, key) -> Any | list[float] | float | Slicer:
"""Override the default __getitem__ behaviour.
This function enables 2 things :
- - access attributes like that : object['any_attribute']
- slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3]
selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]]
selecting 1000x1000 subset : object[:1000, :1000]
@@ -244,12 +265,9 @@ class OTBObject(ABC):
key: attribute key
Returns:
- attribute, pixel values or Slicer
+ list of pixel values if vector image, or pixel value, or Slicer
"""
- # Accessing string attributes
- if isinstance(key, str):
- return getattr(self, key)
# Accessing pixel value(s) using Y/X coordinates
if isinstance(key, tuple) and len(key) >= 2:
row, col = key[0], key[1]
@@ -372,11 +390,8 @@ class OTBObject(ABC):
if isinstance(inp, (float, int, np.ndarray, np.generic)):
arrays.append(inp)
elif isinstance(inp, OTBObject):
- if not inp.exports_dic:
- inp.export()
- image_dic = inp.exports_dic[inp.output_image_key]
- array = image_dic["array"]
- arrays.append(array)
+ image_dic = inp.export()
+ arrays.append(image_dic["array"])
else:
logger.debug(type(self))
return NotImplemented
@@ -424,8 +439,7 @@ class App(OTBObject):
self.quiet = quiet
self.image_dic = image_dic
self._time_start, self._time_end = 0, 0
- self.exports_dic = {}
- self.parameters = {}
+ self.data, self.parameters, self.exports_dic = {}, {}, {}
# Initialize app, set parameters and execute if not frozen
create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
self.app = create(name)
@@ -484,18 +498,6 @@ class App(OTBObject):
"""List of used application outputs."""
return [getattr(self, key) for key in self._out_param_types if key in self.parameters]
- @property
- def data(self) -> dict[str, float, list[float]]:
- """Expose app's output data values in a dictionary."""
- known_bad_keys = ("ram", "elev.default", "mapproj.utm.zone", "mapproj.utm.northhem")
- skip_keys = known_bad_keys + tuple(self._out_param_types) + tuple(self.parameters)
- data_dict = {}
- for key in filter(lambda k: k not in skip_keys, self.parameters_keys):
- value = self.__dict__.get(key)
- if not isinstance(value, otb.ApplicationProxy) and value not in (None, "", [], ()):
- data_dict[str(key)] = value
- return data_dict
-
def set_parameters(self, *args, **kwargs):
"""Set some parameters of the app.
@@ -533,11 +535,8 @@ class App(OTBObject):
f"{self.name}: something went wrong before execution "
f"(while setting parameter '{key}' to '{obj}': {e})"
) from e
- # Update _parameters using values from OtbApplication object
- otb_params = self.app.GetParameters().items()
- otb_params = {k: str(v) if isinstance(v, otb.ApplicationProxy) else v for k, v in otb_params}
# Update param dict and save values as object attributes
- self.parameters.update({**parameters, **otb_params})
+ self.parameters.update(parameters)
self.save_objects()
def propagate_dtype(self, target_key: str = None, dtype: int = None):
@@ -571,29 +570,27 @@ class App(OTBObject):
self.app.SetParameterOutputImagePixelType(key, dtype)
def save_objects(self):
- """Saving app parameters and outputs as attributes, so that they can be accessed with `obj.key`.
-
- This is useful when the key contains reserved characters such as a point eg "io.out"
- """
+ """Saving OTB app parameters and outputs in data and parameters dict."""
for key in self.parameters_keys:
- if key in dir(self.__class__):
- continue # skip forbidden attribute since it is already used by the class
- value = self.parameters.get(key) # basic parameters
+ value = self.parameters.get(key)
if value is None:
try:
value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results)
+ if isinstance(value, otb.ApplicationProxy):
+ continue
except RuntimeError:
continue # this is when there is no value for key
- # Convert output param path to Output object
- if key in self._out_param_types:
- value = Output(self, key, value)
- elif isinstance(value, str):
- try:
- value = literal_eval(value)
- except (ValueError, SyntaxError):
- pass
- # Save attribute
- setattr(self, key, value)
+ # Save app data output or update parameters dict with otb.Application values
+ if isinstance(value, OTBObject) or bool(value) or value == 0:
+ if self.app.GetParameterRole(key) == 0:
+ self.parameters[key] = value
+ else:
+ if isinstance(value, str):
+ try:
+ value = literal_eval(value)
+ except (ValueError, SyntaxError):
+ pass
+ self.data[key] = value
def execute(self):
"""Execute and write to disk if any output parameter has been set during init."""
@@ -615,6 +612,7 @@ class App(OTBObject):
self.app.WriteOutput()
except RuntimeError:
logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name)
+ self._time_start = perf_counter()
self.app.ExecuteAndWriteOutput()
self._time_end = perf_counter()
@@ -743,18 +741,13 @@ class App(OTBObject):
self.app.SetParameterValue(key, obj)
# Special functions
- def __hash__(self) -> int:
- """Override the default behaviour of the hash function.
+ def __getitem__(self, key: str) -> Any | list[int | float] | int | float | Slicer:
+ """This function is called when we use App()[...].
- Returns:
- self hash
-
- """
- return id(self)
-
- def __str__(self) -> str:
- """Return a nice string representation with object id."""
- return f"<pyotb.App {self.name} object id {id(self)}>"
+ We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer."""
+ if isinstance(key, str) and key in self.parameters_keys:
+ return getattr(self, key)
+ return super().__getitem__(key)
class Slicer(App):
--
GitLab
From 5c676a1b838c4a6b646176d871b45e5f636804d1 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 4 Feb 2023 20:30:43 +0100
Subject: [PATCH 119/399] ENH: update tests, move data to tests_data
---
tests/test_core.py | 42 +++++++++++++++++------------------
tests/test_numpy.py | 4 ++--
tests/test_serialization.py | 25 +++------------------
tests/tests_data.py | 44 ++++++++++++++++++++++++++++++++++++-
4 files changed, 69 insertions(+), 46 deletions(-)
diff --git a/tests/test_core.py b/tests/test_core.py
index 7bed905..1b5db4c 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -1,14 +1,7 @@
import pytest
import pyotb
-from tests_data import INPUT
-
-TEST_IMAGE_STATS = {
- 'out.mean': [79.5505, 109.225, 115.456, 249.349],
- 'out.min': [33, 64, 91, 47],
- 'out.max': [255, 255, 230, 255],
- 'out.std': [51.0754, 35.3152, 23.4514, 20.3827]
-}
+from tests_data import INPUT, TEST_IMAGE_STATS
# Input settings
@@ -23,7 +16,7 @@ def test_wrong_key():
# OTBObject properties
def test_key_input():
- assert INPUT.key_input == INPUT.key_input_image == "in"
+ assert INPUT.input_key == INPUT.input_image_key == "in"
def test_key_output():
@@ -64,6 +57,8 @@ def test_elapsed_time():
def test_get_infos():
infos = INPUT.get_info()
assert (infos["sizex"], infos["sizey"]) == (251, 304)
+ bm_infos = pyotb.BandMathX([INPUT], exp="im1")["out"].get_info()
+ assert infos == bm_infos
def test_get_statistics():
@@ -76,21 +71,21 @@ def test_xy_to_rowcol():
def test_write():
INPUT.write("/tmp/test_write.tif")
- assert INPUT.out.exists()
- INPUT.out.filepath.unlink()
+ assert INPUT["out"].exists()
+ INPUT["out"].filepath.unlink()
def test_output_write():
- INPUT.out.write("/tmp/test_output_write.tif")
- assert INPUT.out.exists()
- INPUT.out.filepath.unlink()
+ INPUT["out"].write("/tmp/test_output_write.tif")
+ assert INPUT["out"].exists()
+ INPUT["out"].filepath.unlink()
# Slicer
def test_slicer_shape():
extract = INPUT[:50, :60, :3]
assert extract.shape == (50, 60, 3)
- assert extract.parameters["cl"] == ("Channel1", "Channel2", "Channel3")
+ assert extract.parameters["cl"] == ["Channel1", "Channel2", "Channel3"]
def test_slicer_preserve_dtype():
@@ -102,6 +97,11 @@ def test_slicer_negative_band_index():
assert INPUT[:50, :60, :-2].shape == (50, 60, 2)
+def test_slicer_in_output():
+ slc = pyotb.BandMath([INPUT], exp="im1b1")["out"][:50, :60, :-2]
+ assert isinstance(slc, pyotb.core.Slicer)
+
+
# Arithmetic
def test_operation():
op = INPUT / 255 * 128
@@ -128,8 +128,8 @@ def test_binary_mask_where():
# Essential apps
def test_app_readimageinfo():
info = pyotb.ReadImageInfo(INPUT, quiet=True)
- assert (info.sizex, info.sizey) == (251, 304)
- assert info["numberbands"] == info.numberbands == 4
+ assert (info["sizex"], info["sizey"]) == (251, 304)
+ assert info["numberbands"] == 4
def test_app_computeimagestats():
@@ -150,16 +150,16 @@ def test_read_values_at_coords():
# BandMath NDVI == RadiometricIndices NDVI ?
def test_ndvi_comparison():
ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (INPUT[:, :, -1] + INPUT[:, :, 0])
- ndvi_indices = pyotb.RadiometricIndices(INPUT, {"list": "Vegetation:NDVI", "channels.red": 1, "channels.nir": 4})
+ ndvi_indices = pyotb.RadiometricIndices(INPUT, {"list": ["Vegetation:NDVI"], "channels.red": 1, "channels.nir": 4})
assert ndvi_bandmath.exp == "((im1b4 - im1b1) / (im1b4 + im1b1))"
ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", pixel_type="float")
- assert ndvi_bandmath.out.filepath.exists()
+ assert ndvi_bandmath["out"].filepath.exists()
ndvi_indices.write("/tmp/ndvi_indices.tif", pixel_type="float")
- assert ndvi_indices.out.filepath.exists()
+ assert ndvi_indices["out"].filepath.exists()
compared = pyotb.CompareImages({"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"})
- assert (compared.count, compared.mse) == (0, 0)
+ assert (compared["count"], compared["mse"]) == (0, 0)
thresholded_indices = pyotb.where(ndvi_indices >= 0.3, 1, 0)
assert thresholded_indices.exp == "((im1b1 >= 0.3) ? 1 : 0)"
diff --git a/tests/test_numpy.py b/tests/test_numpy.py
index e62ffbb..d9c3d7b 100644
--- a/tests/test_numpy.py
+++ b/tests/test_numpy.py
@@ -12,8 +12,8 @@ def test_export():
def test_output_export():
- INPUT.out.export()
- assert INPUT.out.output_image_key in INPUT.out.exports_dic
+ INPUT["out"].export()
+ assert INPUT["out"].output_image_key in INPUT["out"].exports_dic
def test_to_numpy():
diff --git a/tests/test_serialization.py b/tests/test_serialization.py
index b56e447..cb079e1 100644
--- a/tests/test_serialization.py
+++ b/tests/test_serialization.py
@@ -1,5 +1,5 @@
import pyotb
-from tests_data import FILEPATH
+from tests_data import *
def test_pipeline_simple():
@@ -8,13 +8,7 @@ def test_pipeline_simple():
app2 = pyotb.OrthoRectification({'io.in': app1})
app3 = pyotb.ManageNoData({'in': app2})
summary = app3.summarize()
- reference = {'name': 'ManageNoData', 'parameters': {'in': {
- 'name': 'OrthoRectification', 'parameters': {'io.in': {
- 'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}},
- 'map': 'utm',
- 'outputs.isotropic': True}},
- 'mode': 'buildmask'}}
- assert summary == reference
+ assert summary == SIMPLE_SERIALIZATION
def test_pipeline_diamond():
@@ -24,17 +18,4 @@ def test_pipeline_diamond():
app3 = pyotb.ManageNoData({'in': app2})
app4 = pyotb.BandMathX({'il': [app2, app3], 'exp': 'im1+im2'})
summary = app4.summarize()
- reference = {'name': 'BandMathX', 'parameters': {'il': [
- {'name': 'OrthoRectification', 'parameters': {'io.in': {
- 'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}},
- 'map': 'utm',
- 'outputs.isotropic': True}},
- {'name': 'ManageNoData', 'parameters': {'in': {
- 'name': 'OrthoRectification', 'parameters': {
- 'io.in': {'name': 'BandMath', 'parameters': {'il': (FILEPATH,), 'exp': 'im1b1'}},
- 'map': 'utm',
- 'outputs.isotropic': True}},
- 'mode': 'buildmask'}}
- ],
- 'exp': 'im1+im2'}}
- assert summary == reference
+ assert summary == COMPLEX_SERIALIZATION
diff --git a/tests/tests_data.py b/tests/tests_data.py
index b190f40..a002c55 100644
--- a/tests/tests_data.py
+++ b/tests/tests_data.py
@@ -1,3 +1,45 @@
import pyotb
-FILEPATH = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false"
+
+FILEPATH = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
INPUT = pyotb.Input(FILEPATH)
+
+TEST_IMAGE_STATS = {
+ 'out.mean': [79.5505, 109.225, 115.456, 249.349],
+ 'out.min': [33, 64, 91, 47],
+ 'out.max': [255, 255, 230, 255],
+ 'out.std': [51.0754, 35.3152, 23.4514, 20.3827]
+}
+
+SIMPLE_SERIALIZATION = {'name': 'ManageNoData',
+ 'parameters': {'in': {'name': 'OrthoRectification',
+ 'parameters': {'io.in': {'name': 'BandMath',
+ 'parameters': {'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'],
+ 'exp': 'im1b1', 'ram': 256}},
+ 'map.utm.zone': 31, 'map.utm.northhem': False, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0,
+ 'elev.default': 0.0, 'interpolator.bco.radius': 2, 'opt.rpc': 10, 'opt.ram': 256,
+ 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5, 'outputs.sizex': 251, 'outputs.sizey': 304,
+ 'outputs.spacingx': 5.997312068939209, 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}},
+ 'usenan': False, 'mode.buildmask.inv': 1.0, 'mode.buildmask.outv': 0.0, 'mode.changevalue.newv': 0.0, 'mode.apply.ndval': 0.0, 'ram': 256}}
+
+COMPLEX_SERIALIZATION = {
+ 'name': 'BandMathX',
+ 'parameters': {'il': [{'name': 'OrthoRectification',
+ 'parameters': {'io.in': {'name': 'BandMath',
+ 'parameters': {
+ 'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'],
+ 'exp': 'im1b1', 'ram': 256}},
+ 'map.utm.zone': 31, 'map.utm.northhem': False, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, 'elev.default': 0.0,
+ 'interpolator.bco.radius': 2, 'opt.rpc': 10, 'opt.ram': 256, 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5,
+ 'outputs.sizex': 251, 'outputs.sizey': 304, 'outputs.spacingx': 5.997312068939209, 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}},
+ {'name': 'ManageNoData',
+ 'parameters': {'in': {'name': 'OrthoRectification',
+ 'parameters': {'io.in': {'name': 'BandMath',
+ 'parameters': {'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'],
+ 'exp': 'im1b1', 'ram': 256}},
+ 'map.utm.zone': 31, 'map.utm.northhem': False, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, 'elev.default': 0.0, 'interpolator.bco.radius': 2,
+ 'opt.rpc': 10, 'opt.ram': 256, 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5,
+ 'outputs.sizex': 251, 'outputs.sizey': 304, 'outputs.spacingx': 5.997312068939209,
+ 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}},
+ 'usenan': False, 'mode.buildmask.inv': 1.0, 'mode.buildmask.outv': 0.0, 'mode.changevalue.newv': 0.0, 'mode.apply.ndval': 0.0, 'ram': 256}}],
+ 'exp': 'im1+im2', 'ram': 256}
+}
--
GitLab
From 6fec9bb996fd200dfdddc6ce36fd4670c39ca296 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 4 Feb 2023 20:30:45 +0100
Subject: [PATCH 120/399] ENH: use __repr__ with obj id in OTBObject, __str__
is left for uses
---
pyotb/core.py | 32 +++++++++++++++-----------------
1 file changed, 15 insertions(+), 17 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 358ad1e..25f677b 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -240,18 +240,6 @@ class OTBObject(ABC):
"""
return id(self)
- def __getattr__(self, key: str) -> Any:
- """Return object attribute, or if key is found in self.parameters or self.data."""
- if key in dir(self):
- return self.__dict__[key]
- if key in self._out_param_types and key in self.parameters:
- return Output(self, key, self.parameters[key])
- if key in self.parameters:
- return self.parameters[key]
- if key in self.data:
- return self.data[key]
- raise AttributeError(f"{self.name}: unknown attribute '{key}'")
-
def __getitem__(self, key) -> Any | list[float] | float | Slicer:
"""Override the default __getitem__ behaviour.
@@ -286,6 +274,10 @@ class OTBObject(ABC):
key = key + (slice(None, None, None),)
return Slicer(self, *key)
+ def __repr__(self) -> str:
+ """Return a nice string representation with object id."""
+ return f"<pyotb.{self.__class__.__name__} object id {id(self)}>"
+
def __add__(self, other: OTBObject | str | int | float) -> Operation:
"""Addition."""
return self.__create_operator(Operation, "+", self, other)
@@ -670,7 +662,7 @@ class App(OTBObject):
# Set parameters and flush to disk
for key, output_filename in kwargs.items():
- if Path(output_filename).exists():
+ if Path(output_filename.split("?")[0]).exists():
logger.warning("%s: overwriting file %s", self.name, output_filename)
if key in dtypes:
self.propagate_dtype(key, dtypes[key])
@@ -745,8 +737,14 @@ class App(OTBObject):
"""This function is called when we use App()[...].
We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer."""
- if isinstance(key, str) and key in self.parameters_keys:
- return getattr(self, key)
+ if isinstance(key, str):
+ if key in self._out_param_types:
+ return Output(self, key, self.parameters.get(key))
+ if key in self.parameters:
+ return self.parameters[key]
+ if key in self.data:
+ return self.data[key]
+ raise KeyError(f"{self.name} object has no attribute '{key}'")
return super().__getitem__(key)
@@ -1153,8 +1151,8 @@ class Output(OTBObject):
return self.parent_pyotb_app.write({self.output_image_key: filepath}, **kwargs)
def __str__(self) -> str:
- """Return a nice string representation with source app name and object id."""
- return f"<pyotb.Output {self.name} object, id {id(self)}>"
+ """Return string representation of Output filepath."""
+ return str(self.filepath)
def get_nbchannels(inp: str | OTBObject) -> int:
--
GitLab
From 3ac03d3087b2412096e728d8a7759664781dfe6d Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 4 Feb 2023 20:41:18 +0100
Subject: [PATCH 121/399] STYE: linter
---
pyotb/core.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 25f677b..58b054b 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -736,7 +736,8 @@ class App(OTBObject):
def __getitem__(self, key: str) -> Any | list[int | float] | int | float | Slicer:
"""This function is called when we use App()[...].
- We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer."""
+ We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer
+ """
if isinstance(key, str):
if key in self._out_param_types:
return Output(self, key, self.parameters.get(key))
@@ -791,7 +792,7 @@ class Slicer(App):
# Spatial slicing
spatial_slicing = False
- # TODO TBD: handle the step value in the slice so that NN undersampling is possible ? e.g. raster[::2, ::2]
+ # TODO: handle the step value in the slice so that NN undersampling is possible ? e.g. raster[::2, ::2]
if rows.start is not None:
parameters.update({"mode.extent.uly": rows.start})
spatial_slicing = True
--
GitLab
From 8a60776b92e1d7baea7d9dd289f0ad8b9d65ea29 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 4 Feb 2023 21:16:12 +0100
Subject: [PATCH 122/399] ENH: style and raise exceptions when get_first_key
failed
---
pyotb/core.py | 41 +++++++++++++++++++----------------------
1 file changed, 19 insertions(+), 22 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 58b054b..d5124ec 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -5,6 +5,7 @@ from __future__ import annotations
from ast import literal_eval
from pathlib import Path
from time import perf_counter
+from itertools import chain
from typing import Any
from abc import ABC, abstractmethod
@@ -173,13 +174,12 @@ class OTBObject(ABC):
profile: a metadata dict required to write image using rasterio
"""
+ profile = {}
array = self.to_numpy(preserve_dtype=True, copy=False)
height, width, count = array.shape
proj = self.app.GetImageProjection(self.output_image_key)
- profile = {
- 'crs': proj, 'dtype': array.dtype, 'transform': self.transform,
- 'count': count, 'height': height, 'width': width,
- }
+ profile.update({"crs": proj, "dtype": array.dtype, "transform": self.transform})
+ profile.update({"count": count, "height": height, "width": width})
return np.moveaxis(array, 2, 0), profile
def get_rowcol_from_xy(self, x: float, y: float) -> tuple[int, int]:
@@ -449,36 +449,35 @@ class App(OTBObject):
def get_first_key(self, *args: tuple[list[int]]) -> str:
"""Get the first param key for specific file types, try each list in args."""
for param_types in args:
- for key, param_type in sorted(self._all_param_types.items()):
- if param_type in param_types:
+ types = [getattr(otb, "ParameterType_" + key) for key in param_types]
+ for key, value in sorted(self._all_param_types.items()):
+ if value in types:
return key
- return None
+ raise TypeError(f"{self.name}: could not find any parameter of type {tuple(chain(*args))}")
@property
def input_key(self) -> str:
"""Get the name of first input parameter, raster > vector > file."""
return self.get_first_key(
- [otb.ParameterType_InputImage, otb.ParameterType_InputImageList],
- [otb.ParameterType_InputVectorData, otb.ParameterType_InputVectorDataList],
- [otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList]
+ ["InputImage", "InputImageList"],
+ ["InputVectorData", "InputVectorDataList"],
+ ["InputFilename", "InputFilenameList"],
)
@property
def input_image_key(self) -> str:
"""Name of the first input image parameter."""
- return self.get_first_key([otb.ParameterType_InputImage, otb.ParameterType_InputImageList])
+ return self.get_first_key(["InputImage", "InputImageList"])
@property
def output_key(self) -> str:
"""Name of the first output parameter, raster > vector > file."""
- return self.get_first_key(
- [otb.ParameterType_OutputImage], [otb.ParameterType_OutputVectorData], [otb.ParameterType_OutputFilename]
- )
+ return self.get_first_key(["OutputImage"], ["OutputVectorData"], ["OutputFilename"])
@property
def output_image_key(self) -> str:
"""Get the name of first output image parameter."""
- return self.get_first_key([otb.ParameterType_OutputImage])
+ return self.get_first_key(["OutputImage"])
@property
def elapsed_time(self) -> float:
@@ -659,7 +658,8 @@ class App(OTBObject):
dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()}
elif preserve_dtype:
self.propagate_dtype() # all outputs will have the same type as the main input raster
-
+ if not kwargs:
+ raise KeyError(f"{self.name}: at least one filepath is required, if not passed to App during init")
# Set parameters and flush to disk
for key, output_filename in kwargs.items():
if Path(output_filename.split("?")[0]).exists():
@@ -877,8 +877,7 @@ class Operation(App):
super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True)
self.name = name or f'Operation exp="{self.exp}"'
- def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float],
- nb_bands: int = None):
+ def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None):
"""Create a list of 'fake' expressions, one for each band.
E.g for the operation input1 + input2, we create a fake expression that is like "str(input1) + str(input2)"
@@ -960,8 +959,7 @@ class Operation(App):
return exp_bands, ";".join(exp_bands)
@staticmethod
- def make_fake_exp(x: OTBObject | str, band: int, keep_logical: bool = False) \
- -> tuple[str, list[OTBObject], int]:
+ def make_fake_exp(x: OTBObject | str, band: int, keep_logical: bool = False) -> tuple[str, list[OTBObject], int]:
"""This an internal function, only to be used by `build_fake_expressions`.
Enable to create a fake expression just for one input and one band.
@@ -1038,8 +1036,7 @@ class LogicalOperation(Operation):
super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation")
self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands)
- def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float],
- nb_bands: int = None):
+ def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None):
"""Create a list of 'fake' expressions, one for each band.
e.g for the operation input1 > input2, we create a fake expression that is like
--
GitLab
From d678ef5f562cffeafd03baf0b6d256624eff067a Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 4 Feb 2023 21:18:50 +0100
Subject: [PATCH 123/399] ENH: rename filename_extension to ext_fname
---
pyotb/core.py | 14 +++++++-------
tests/test_core.py | 2 +-
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index d5124ec..463338c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -607,7 +607,7 @@ class App(OTBObject):
self.app.ExecuteAndWriteOutput()
self._time_end = perf_counter()
- def write(self, *args, filename_extension: str = "", pixel_type: dict[str, str] | str = None,
+ def write(self, *args, ext_fname: str = "", pixel_type: dict[str, str] | str = None,
preserve_dtype: bool = False, **kwargs):
"""Set output pixel type and write the output raster files.
@@ -616,7 +616,7 @@ class App(OTBObject):
non-standard characters such as a point, e.g. {'io.out':'output.tif'}
- string, useful when there is only one output, e.g. 'output.tif'
- None if output file was passed during App init
- filename_extension: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES")
+ ext_fname: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES")
Will be used for all outputs (Default value = "")
pixel_type: Can be : - dictionary {output_parameter_key: pixeltype} when specifying for several outputs
- str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several
@@ -637,13 +637,13 @@ class App(OTBObject):
kwargs.update({self.output_image_key: str(arg)})
# Append filename extension to filenames
- if filename_extension:
- logger.debug("%s: using extended filename for outputs: %s", self.name, filename_extension)
- if not filename_extension.startswith("?"):
- filename_extension = "?" + filename_extension
+ if ext_fname:
+ logger.debug("%s: using extended filename for outputs: %s", self.name, ext_fname)
+ if not ext_fname.startswith("?"):
+ ext_fname = "?" + ext_fname
for key, value in kwargs.items():
if self._out_param_types[key] == otb.ParameterType_OutputImage and "?" not in value:
- kwargs[key] = value + filename_extension
+ kwargs[key] = value + ext_fname
# Manage output pixel types
dtypes = {}
diff --git a/tests/test_core.py b/tests/test_core.py
index 1b5db4c..eba448e 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -70,7 +70,7 @@ def test_xy_to_rowcol():
def test_write():
- INPUT.write("/tmp/test_write.tif")
+ INPUT.write("/tmp/test_write.tif", ext_fname="nodata=0")
assert INPUT["out"].exists()
INPUT["out"].filepath.unlink()
--
GitLab
From b198e9915522b1d018a22d9b01a75cac75472c52 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 4 Feb 2023 21:29:10 +0100
Subject: [PATCH 124/399] FIX: try to add "?&" to ext_fname test
---
tests/test_core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/test_core.py b/tests/test_core.py
index eba448e..5e59d5b 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -70,7 +70,7 @@ def test_xy_to_rowcol():
def test_write():
- INPUT.write("/tmp/test_write.tif", ext_fname="nodata=0")
+ INPUT.write("/tmp/test_write.tif", ext_fname="?&nodata=0")
assert INPUT["out"].exists()
INPUT["out"].filepath.unlink()
--
GitLab
From d29d17c6d973c216384cc82bce0bdc5e8868cf2d Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 4 Feb 2023 21:35:48 +0100
Subject: [PATCH 125/399] FIX: force use of "?&" in ext_fname when "?" is
missing
---
pyotb/core.py | 2 +-
tests/test_core.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 463338c..4eac666 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -640,7 +640,7 @@ class App(OTBObject):
if ext_fname:
logger.debug("%s: using extended filename for outputs: %s", self.name, ext_fname)
if not ext_fname.startswith("?"):
- ext_fname = "?" + ext_fname
+ ext_fname = "?&" + ext_fname
for key, value in kwargs.items():
if self._out_param_types[key] == otb.ParameterType_OutputImage and "?" not in value:
kwargs[key] = value + ext_fname
diff --git a/tests/test_core.py b/tests/test_core.py
index 5e59d5b..eba448e 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -70,7 +70,7 @@ def test_xy_to_rowcol():
def test_write():
- INPUT.write("/tmp/test_write.tif", ext_fname="?&nodata=0")
+ INPUT.write("/tmp/test_write.tif", ext_fname="nodata=0")
assert INPUT["out"].exists()
INPUT["out"].filepath.unlink()
--
GitLab
From bdbefb2b278c1e85349c4aaec8876fc901d514df Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 4 Feb 2023 21:51:36 +0100
Subject: [PATCH 126/399] STYLE: identation
---
tests/tests_data.py | 24 ++++++++++++------------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/tests/tests_data.py b/tests/tests_data.py
index a002c55..f69edac 100644
--- a/tests/tests_data.py
+++ b/tests/tests_data.py
@@ -9,18 +9,18 @@ TEST_IMAGE_STATS = {
'out.max': [255, 255, 230, 255],
'out.std': [51.0754, 35.3152, 23.4514, 20.3827]
}
-
-SIMPLE_SERIALIZATION = {'name': 'ManageNoData',
- 'parameters': {'in': {'name': 'OrthoRectification',
- 'parameters': {'io.in': {'name': 'BandMath',
- 'parameters': {'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'],
- 'exp': 'im1b1', 'ram': 256}},
- 'map.utm.zone': 31, 'map.utm.northhem': False, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0,
- 'elev.default': 0.0, 'interpolator.bco.radius': 2, 'opt.rpc': 10, 'opt.ram': 256,
- 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5, 'outputs.sizex': 251, 'outputs.sizey': 304,
- 'outputs.spacingx': 5.997312068939209, 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}},
- 'usenan': False, 'mode.buildmask.inv': 1.0, 'mode.buildmask.outv': 0.0, 'mode.changevalue.newv': 0.0, 'mode.apply.ndval': 0.0, 'ram': 256}}
-
+SIMPLE_SERIALIZATION = {
+ 'name': 'ManageNoData',
+ 'parameters': {'in': {'name': 'OrthoRectification',
+ 'parameters': {'io.in': {'name': 'BandMath',
+ 'parameters': {'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'],
+ 'exp': 'im1b1', 'ram': 256}},
+ 'map.utm.zone': 31, 'map.utm.northhem': False, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0,
+ 'elev.default': 0.0, 'interpolator.bco.radius': 2, 'opt.rpc': 10, 'opt.ram': 256,
+ 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5, 'outputs.sizex': 251, 'outputs.sizey': 304,
+ 'outputs.spacingx': 5.997312068939209, 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}},
+ 'usenan': False, 'mode.buildmask.inv': 1.0, 'mode.buildmask.outv': 0.0, 'mode.changevalue.newv': 0.0, 'mode.apply.ndval': 0.0, 'ram': 256}
+}
COMPLEX_SERIALIZATION = {
'name': 'BandMathX',
'parameters': {'il': [{'name': 'OrthoRectification',
--
GitLab
From 249103b2e54c1d3aadae27b8e99c5055a46a3932 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 4 Feb 2023 21:56:02 +0100
Subject: [PATCH 127/399] ENH: better exception when accessing App[param] with
wrong name
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 4eac666..3512131 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -745,7 +745,7 @@ class App(OTBObject):
return self.parameters[key]
if key in self.data:
return self.data[key]
- raise KeyError(f"{self.name} object has no attribute '{key}'")
+ raise KeyError(f"{self.name}: unknown parameter '{key}'")
return super().__getitem__(key)
--
GitLab
From 5e15559891adf2550529e514c2da1fe22cbec091 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 4 Feb 2023 22:11:17 +0100
Subject: [PATCH 128/399] STYLE: tests autoformat, use Output.exists()
---
tests/test_core.py | 5 ++---
tests/test_pipeline.py | 8 +++-----
tests/test_serialization.py | 14 +++++++-------
3 files changed, 12 insertions(+), 15 deletions(-)
diff --git a/tests/test_core.py b/tests/test_core.py
index eba448e..1b6441f 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -52,7 +52,6 @@ def test_elapsed_time():
assert 0 < pyotb.ReadImageInfo(INPUT).elapsed_time < 1
-
# Other functions
def test_get_infos():
infos = INPUT.get_info()
@@ -154,9 +153,9 @@ def test_ndvi_comparison():
assert ndvi_bandmath.exp == "((im1b4 - im1b1) / (im1b4 + im1b1))"
ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", pixel_type="float")
- assert ndvi_bandmath["out"].filepath.exists()
+ assert ndvi_bandmath["out"].exists()
ndvi_indices.write("/tmp/ndvi_indices.tif", pixel_type="float")
- assert ndvi_indices["out"].filepath.exists()
+ assert ndvi_indices["out"].exists()
compared = pyotb.CompareImages({"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"})
assert (compared["count"], compared["mse"]) == (0, 0)
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index 76f9872..19160c5 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -12,7 +12,7 @@ OTBAPPS_BLOCKS = [
lambda inp: pyotb.DynamicConvert({"in": inp}),
lambda inp: pyotb.Mosaic({"il": [inp]}),
lambda inp: pyotb.BandMath({"il": [inp], "exp": "im1b1 + 1"}),
- lambda inp: pyotb.BandMathX({"il": [inp], "exp": "im1"})
+ lambda inp: pyotb.BandMathX({"il": [inp], "exp": "im1"}),
]
PYOTB_BLOCKS = [
@@ -74,14 +74,12 @@ def pipeline2str(pipeline):
a string
"""
- return " > ".join([INPUT.__class__.__name__] + [f"{i}.{app.name.split()[0]}"
- for i, app in enumerate(pipeline)])
+ return " > ".join([INPUT.__class__.__name__] + [f"{i}.{app.name.split()[0]}" for i, app in enumerate(pipeline)])
def make_pipelines_list():
"""Create a list of pipelines using different lengths and blocks"""
- blocks = {FILEPATH: OTBAPPS_BLOCKS, # for filepath, we can't use Slicer or Operation
- INPUT: ALL_BLOCKS}
+ blocks = {FILEPATH: OTBAPPS_BLOCKS, INPUT: ALL_BLOCKS} # for filepath, we can't use Slicer or Operation
pipelines = []
names = []
for inp, blocks in blocks.items():
diff --git a/tests/test_serialization.py b/tests/test_serialization.py
index cb079e1..28aa3a7 100644
--- a/tests/test_serialization.py
+++ b/tests/test_serialization.py
@@ -4,18 +4,18 @@ from tests_data import *
def test_pipeline_simple():
# BandMath -> OrthoRectification -> ManageNoData
- app1 = pyotb.BandMath({'il': [FILEPATH], 'exp': 'im1b1'})
- app2 = pyotb.OrthoRectification({'io.in': app1})
- app3 = pyotb.ManageNoData({'in': app2})
+ app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"})
+ app2 = pyotb.OrthoRectification({"io.in": app1})
+ app3 = pyotb.ManageNoData({"in": app2})
summary = app3.summarize()
assert summary == SIMPLE_SERIALIZATION
def test_pipeline_diamond():
# Diamond graph
- app1 = pyotb.BandMath({'il': [FILEPATH], 'exp': 'im1b1'})
- app2 = pyotb.OrthoRectification({'io.in': app1})
- app3 = pyotb.ManageNoData({'in': app2})
- app4 = pyotb.BandMathX({'il': [app2, app3], 'exp': 'im1+im2'})
+ app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"})
+ app2 = pyotb.OrthoRectification({"io.in": app1})
+ app3 = pyotb.ManageNoData({"in": app2})
+ app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"})
summary = app4.summarize()
assert summary == COMPLEX_SERIALIZATION
--
GitLab
From 0addfdf8f5f759b79f17277597ca188d8454a6db Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 5 Feb 2023 14:27:02 +0100
Subject: [PATCH 129/399] STYLE: var names, types and exceptions
---
pyotb/core.py | 123 ++++++++++++++++++++++++++------------------------
1 file changed, 63 insertions(+), 60 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 3512131..364c7c7 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -259,16 +259,15 @@ class OTBObject(ABC):
# Accessing pixel value(s) using Y/X coordinates
if isinstance(key, tuple) and len(key) >= 2:
row, col = key[0], key[1]
- if isinstance(row, int) and isinstance(col, int):
- if row < 0 or col < 0:
- raise ValueError(f"{self.name}: can't read pixel value at negative coordinates ({row}, {col})")
- channels = None
- if len(key) == 3:
- channels = key[2]
- return self.get_values_at_coords(row, col, channels)
+ if not isinstance(row, int) or not isinstance(col, int):
+ raise TypeError(f"{self.name}: cannnot read pixel values using {type(row)} and {type(col)}")
+ if row < 0 or col < 0:
+ raise ValueError(f"{self.name} cannot read pixel value at negative coordinates ({row}, {col})")
+ channels = key[2] if len(key) == 3 else None
+ return self.get_values_at_coords(row, col, channels)
# Slicing
if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)):
- raise ValueError(f'"{key}"cannot be interpreted as valid slicing. Slicing should be 2D or 3D.')
+ raise ValueError(f'"{key}" cannot be interpreted as valid slicing. Slicing should be 2D or 3D.')
if isinstance(key, tuple) and len(key) == 2:
# Adding a 3rd dimension
key = key + (slice(None, None, None),)
@@ -430,7 +429,7 @@ class App(OTBObject):
self.frozen = frozen
self.quiet = quiet
self.image_dic = image_dic
- self._time_start, self._time_end = 0, 0
+ self._time_start, self._time_end = 0., 0.
self.data, self.parameters, self.exports_dic = {}, {}, {}
# Initialize app, set parameters and execute if not frozen
create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
@@ -446,14 +445,14 @@ class App(OTBObject):
if any(key in self.parameters for key in self._out_param_types):
self.flush() # auto flush if any output param was provided during app init
- def get_first_key(self, *args: tuple[list[int]]) -> str:
+ def get_first_key(self, *type_lists: tuple[list[int]]) -> str:
"""Get the first param key for specific file types, try each list in args."""
- for param_types in args:
+ for param_types in type_lists:
types = [getattr(otb, "ParameterType_" + key) for key in param_types]
for key, value in sorted(self._all_param_types.items()):
if value in types:
return key
- raise TypeError(f"{self.name}: could not find any parameter of type {tuple(chain(*args))}")
+ raise TypeError(f"{self.name}: could not find any parameter of type {tuple(chain(*type_lists))}")
@property
def input_key(self) -> str:
@@ -608,17 +607,17 @@ class App(OTBObject):
self._time_end = perf_counter()
def write(self, *args, ext_fname: str = "", pixel_type: dict[str, str] | str = None,
- preserve_dtype: bool = False, **kwargs):
+ preserve_dtype: bool = False, **kwargs, ) -> bool:
"""Set output pixel type and write the output raster files.
Args:
*args: Can be : - dictionary containing key-arguments enumeration. Useful when a key contains
non-standard characters such as a point, e.g. {'io.out':'output.tif'}
- - string, useful when there is only one output, e.g. 'output.tif'
+ - filepath, useful when there is only one output, e.g. 'output.tif'
- None if output file was passed during App init
ext_fname: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES")
Will be used for all outputs (Default value = "")
- pixel_type: Can be : - dictionary {output_parameter_key: pixeltype} when specifying for several outputs
+ pixel_type: Can be : - dictionary {out_param_key: pixeltype} when specifying for several outputs
- str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several
outputs, all outputs are written with this unique type.
Valid pixel types are uint8, uint16, uint32, int16, int32, float, double,
@@ -626,6 +625,9 @@ class App(OTBObject):
preserve_dtype: propagate main input pixel type to outputs, in case pixel_type is None
**kwargs: keyword arguments e.g. out='output.tif'
+ Returns:
+ True if all files are found on disk
+
"""
# Gather all input arguments in kwargs dict
for arg in args:
@@ -633,39 +635,43 @@ class App(OTBObject):
kwargs.update(arg)
elif isinstance(arg, str) and kwargs:
logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg)
- elif isinstance(arg, (str, Path)) and self.output_image_key:
- kwargs.update({self.output_image_key: str(arg)})
+ elif isinstance(arg, (str, Path)) and self.output_key:
+ kwargs.update({self.output_key: str(arg)})
+ if not kwargs:
+ raise KeyError(f"{self.name}: at least one filepath is required, if not passed to App during init")
+ parameters = kwargs.copy()
# Append filename extension to filenames
if ext_fname:
logger.debug("%s: using extended filename for outputs: %s", self.name, ext_fname)
if not ext_fname.startswith("?"):
ext_fname = "?&" + ext_fname
+ elif not ext_fname.startswith("?&"):
+ ext_fname = "?&" + ext_fname[1:]
for key, value in kwargs.items():
if self._out_param_types[key] == otb.ParameterType_OutputImage and "?" not in value:
- kwargs[key] = value + ext_fname
-
+ parameters[key] = value + ext_fname
# Manage output pixel types
- dtypes = {}
- if pixel_type:
+ data_types = {}
+ if pixel_type is not None:
if isinstance(pixel_type, str):
- type_name = self.app.ConvertPixelTypeToNumpy(parse_pixel_type(pixel_type))
+ dtype = parse_pixel_type(pixel_type)
+ type_name = self.app.ConvertPixelTypeToNumpy(dtype)
logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name)
- for key in kwargs:
+ for key in parameters:
if self._out_param_types[key] == otb.ParameterType_OutputImage:
- dtypes[key] = parse_pixel_type(pixel_type)
+ data_types[key] = dtype
elif isinstance(pixel_type, dict):
- dtypes = {k: parse_pixel_type(v) for k, v in pixel_type.items()}
+ data_types = {key: parse_pixel_type(dtype) for key, dtype in pixel_type.items()}
elif preserve_dtype:
self.propagate_dtype() # all outputs will have the same type as the main input raster
- if not kwargs:
- raise KeyError(f"{self.name}: at least one filepath is required, if not passed to App during init")
+
# Set parameters and flush to disk
- for key, output_filename in kwargs.items():
+ for key, output_filename in parameters.items():
if Path(output_filename.split("?")[0]).exists():
logger.warning("%s: overwriting file %s", self.name, output_filename)
- if key in dtypes:
- self.propagate_dtype(key, dtypes[key])
+ if key in data_types:
+ self.propagate_dtype(key, data_types[key])
self.set_parameters({key: output_filename})
self.flush()
@@ -1153,7 +1159,7 @@ class Output(OTBObject):
return str(self.filepath)
-def get_nbchannels(inp: str | OTBObject) -> int:
+def get_nbchannels(inp: str | Path | OTBObject) -> int:
"""Get the nb of bands of input image.
Args:
@@ -1164,18 +1170,18 @@ def get_nbchannels(inp: str | OTBObject) -> int:
"""
if isinstance(inp, OTBObject):
- nb_channels = inp.shape[-1]
- else:
+ return inp.shape[-1]
+ if isinstance(inp, (str, Path)):
# Executing the app, without printing its log
try:
info = App("ReadImageInfo", inp, quiet=True)
- nb_channels = info.app.GetParameterInt("numberbands")
+ return info["numberbands"]
except RuntimeError as info_err: # this happens when we pass a str that is not a filepath
- raise TypeError(f"Could not get the number of channels of '{inp}' ({info_err})") from info_err
- return nb_channels
+ raise TypeError(f"Could not get the number of channels file '{inp}' ({info_err})") from info_err
+ raise TypeError(f"Can't read number of channels of type '{type(inp)}' object {inp}")
-def get_pixel_type(inp: str | OTBObject) -> str:
+def get_pixel_type(inp: str | Path | OTBObject) -> str:
"""Get the encoding of input image pixels.
Args:
@@ -1186,33 +1192,30 @@ def get_pixel_type(inp: str | OTBObject) -> str:
For an OTBObject with several outputs, only the pixel type of the first output is returned
"""
- if isinstance(inp, str):
+ if isinstance(inp, OTBObject):
+ return inp.app.GetParameterOutputImagePixelType(inp.output_image_key)
+ if isinstance(inp, (str, Path)):
try:
info = App("ReadImageInfo", inp, quiet=True)
+ datatype = info["datatype"] # which is such as short, float...
except RuntimeError as info_err: # this happens when we pass a str that is not a filepath
raise TypeError(f"Could not get the pixel type of `{inp}` ({info_err})") from info_err
- datatype = info.app.GetParameterString("datatype") # which is such as short, float...
- if not datatype:
- raise TypeError(f"Unable to read pixel type of image {inp}")
- datatype_to_pixeltype = {
- "unsigned_char": "uint8",
- "short": "int16",
- "unsigned_short": "uint16",
- "int": "int32",
- "unsigned_int": "uint32",
- "long": "int32",
- "ulong": "uint32",
- "float": "float",
- "double": "double",
- }
- if datatype not in datatype_to_pixeltype:
- raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}")
- pixel_type = getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[datatype]}")
- elif isinstance(inp, OTBObject):
- pixel_type = inp.app.GetParameterOutputImagePixelType(inp.output_image_key)
- else:
- raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}")
- return pixel_type
+ if datatype:
+ datatype_to_pixeltype = {
+ "unsigned_char": "uint8",
+ "short": "int16",
+ "unsigned_short": "uint16",
+ "int": "int32",
+ "unsigned_int": "uint32",
+ "long": "int32",
+ "ulong": "uint32",
+ "float": "float",
+ "double": "double",
+ }
+ if datatype not in datatype_to_pixeltype:
+ raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}")
+ return getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[datatype]}")
+ raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}")
def parse_pixel_type(pixel_type: str | int) -> int:
--
GitLab
From b2cf422165144f872fbdd3cd8d515a1ce6bf4cc2 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 5 Feb 2023 14:38:40 +0100
Subject: [PATCH 130/399] FIX: Outputs, save_objects and getittem, add
App.outputs dict
---
pyotb/core.py | 63 ++++++++++++++++++++-------------------------
tests/tests_data.py | 6 ++---
2 files changed, 31 insertions(+), 38 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 364c7c7..f7447ad 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -259,12 +259,11 @@ class OTBObject(ABC):
# Accessing pixel value(s) using Y/X coordinates
if isinstance(key, tuple) and len(key) >= 2:
row, col = key[0], key[1]
- if not isinstance(row, int) or not isinstance(col, int):
- raise TypeError(f"{self.name}: cannnot read pixel values using {type(row)} and {type(col)}")
- if row < 0 or col < 0:
- raise ValueError(f"{self.name} cannot read pixel value at negative coordinates ({row}, {col})")
- channels = key[2] if len(key) == 3 else None
- return self.get_values_at_coords(row, col, channels)
+ if isinstance(row, int) and isinstance(col, int):
+ if row < 0 or col < 0:
+ raise ValueError(f"{self.name} cannot read pixel value at negative coordinates ({row}, {col})")
+ channels = key[2] if len(key) == 3 else None
+ return self.get_values_at_coords(row, col, channels)
# Slicing
if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)):
raise ValueError(f'"{key}" cannot be interpreted as valid slicing. Slicing should be 2D or 3D.')
@@ -425,12 +424,10 @@ class App(OTBObject):
e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif'
"""
- self.name = name
- self.frozen = frozen
- self.quiet = quiet
- self.image_dic = image_dic
+ self.name, self.image_dic = name, image_dic
+ self.quiet, self.frozen = quiet, frozen
self._time_start, self._time_end = 0., 0.
- self.data, self.parameters, self.exports_dic = {}, {}, {}
+ self.data, self.parameters, self.outputs, self.exports_dic = {}, {}, {}, {}
# Initialize app, set parameters and execute if not frozen
create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
self.app = create(name)
@@ -486,7 +483,7 @@ class App(OTBObject):
@property
def used_outputs(self) -> list[str]:
"""List of used application outputs."""
- return [getattr(self, key) for key in self._out_param_types if key in self.parameters]
+ return [self.outputs[key] for key in self._out_param_types if key in self.parameters]
def set_parameters(self, *args, **kwargs):
"""Set some parameters of the app.
@@ -527,7 +524,7 @@ class App(OTBObject):
) from e
# Update param dict and save values as object attributes
self.parameters.update(parameters)
- self.save_objects()
+ self.save_objects(list(parameters))
def propagate_dtype(self, target_key: str = None, dtype: int = None):
"""Propagate a pixel type from main input to every outputs, or to a target output key only.
@@ -559,18 +556,20 @@ class App(OTBObject):
for key in keys:
self.app.SetParameterOutputImagePixelType(key, dtype)
- def save_objects(self):
- """Saving OTB app parameters and outputs in data and parameters dict."""
- for key in self.parameters_keys:
+ def save_objects(self, keys: list[str] = None):
+ """Save OTB app values in data, parameters and outputs dict, for a list of keys or all parameters."""
+ keys = keys or self.parameters_keys
+ for key in keys:
value = self.parameters.get(key)
if value is None:
try:
value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results)
- if isinstance(value, otb.ApplicationProxy):
- continue
except RuntimeError:
continue # this is when there is no value for key
- # Save app data output or update parameters dict with otb.Application values
+ if value is None or isinstance(value, otb.ApplicationProxy):
+ continue
+ if self._out_param_types.get(key) == otb.ParameterType_OutputImage:
+ self.outputs[key] = Output(self, key, value)
if isinstance(value, OTBObject) or bool(value) or value == 0:
if self.app.GetParameterRole(key) == 0:
self.parameters[key] = value
@@ -674,21 +673,14 @@ class App(OTBObject):
self.propagate_dtype(key, data_types[key])
self.set_parameters({key: output_filename})
self.flush()
-
- def find_outputs(self) -> tuple[str]:
- """Find output files on disk using path found in parameters.
-
- Returns:
- list of files found on disk
-
- """
files, missing = [], []
for out in self.used_outputs:
dest = files if out.exists() else missing
dest.append(str(out.filepath.absolute()))
for filename in missing:
logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
- return tuple(files)
+ raise Exception
+ return bool(files) and not missing
# Private functions
def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]:
@@ -744,16 +736,17 @@ class App(OTBObject):
We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer
"""
+ if isinstance(key, tuple):
+ return super().__getitem__(key)
if isinstance(key, str):
- if key in self._out_param_types:
- return Output(self, key, self.parameters.get(key))
- if key in self.parameters:
- return self.parameters[key]
if key in self.data:
return self.data[key]
- raise KeyError(f"{self.name}: unknown parameter '{key}'")
- return super().__getitem__(key)
-
+ if key in self.outputs:
+ return self.outputs[key]
+ if key in self.parameters:
+ return self.parameters[key]
+ raise KeyError(f"{self.name}: unknown or undefined parameter '{key}'")
+ raise TypeError(f"{self.name}: cannot access object item or slice using {type(key)} object")
class Slicer(App):
"""Slicer objects i.e. when we call something like raster[:, :, 2] from Python."""
diff --git a/tests/tests_data.py b/tests/tests_data.py
index f69edac..de5d727 100644
--- a/tests/tests_data.py
+++ b/tests/tests_data.py
@@ -15,7 +15,7 @@ SIMPLE_SERIALIZATION = {
'parameters': {'io.in': {'name': 'BandMath',
'parameters': {'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'],
'exp': 'im1b1', 'ram': 256}},
- 'map.utm.zone': 31, 'map.utm.northhem': False, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0,
+ 'map.utm.zone': 31, 'map.utm.northhem': True, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0,
'elev.default': 0.0, 'interpolator.bco.radius': 2, 'opt.rpc': 10, 'opt.ram': 256,
'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5, 'outputs.sizex': 251, 'outputs.sizey': 304,
'outputs.spacingx': 5.997312068939209, 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}},
@@ -28,7 +28,7 @@ COMPLEX_SERIALIZATION = {
'parameters': {
'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'],
'exp': 'im1b1', 'ram': 256}},
- 'map.utm.zone': 31, 'map.utm.northhem': False, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, 'elev.default': 0.0,
+ 'map.utm.zone': 31, 'map.utm.northhem': True, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, 'elev.default': 0.0,
'interpolator.bco.radius': 2, 'opt.rpc': 10, 'opt.ram': 256, 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5,
'outputs.sizex': 251, 'outputs.sizey': 304, 'outputs.spacingx': 5.997312068939209, 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}},
{'name': 'ManageNoData',
@@ -36,7 +36,7 @@ COMPLEX_SERIALIZATION = {
'parameters': {'io.in': {'name': 'BandMath',
'parameters': {'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'],
'exp': 'im1b1', 'ram': 256}},
- 'map.utm.zone': 31, 'map.utm.northhem': False, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, 'elev.default': 0.0, 'interpolator.bco.radius': 2,
+ 'map.utm.zone': 31, 'map.utm.northhem': True, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, 'elev.default': 0.0, 'interpolator.bco.radius': 2,
'opt.rpc': 10, 'opt.ram': 256, 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5,
'outputs.sizex': 251, 'outputs.sizey': 304, 'outputs.spacingx': 5.997312068939209,
'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}},
--
GitLab
From cbb49030e994c9f64e7aa128504a958e86c56b90 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 5 Feb 2023 14:43:39 +0100
Subject: [PATCH 131/399] STYLE: remove test Exception
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index f7447ad..c8ff8eb 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -679,7 +679,6 @@ class App(OTBObject):
dest.append(str(out.filepath.absolute()))
for filename in missing:
logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
- raise Exception
return bool(files) and not missing
# Private functions
@@ -748,6 +747,7 @@ class App(OTBObject):
raise KeyError(f"{self.name}: unknown or undefined parameter '{key}'")
raise TypeError(f"{self.name}: cannot access object item or slice using {type(key)} object")
+
class Slicer(App):
"""Slicer objects i.e. when we call something like raster[:, :, 2] from Python."""
--
GitLab
From f008eb0550450b38914303320383343b8935a863 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 5 Feb 2023 15:01:41 +0100
Subject: [PATCH 132/399] ENH: remove find_outputs, just check file exists
during write()
---
pyotb/core.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index c8ff8eb..695039e 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -638,8 +638,8 @@ class App(OTBObject):
kwargs.update({self.output_key: str(arg)})
if not kwargs:
raise KeyError(f"{self.name}: at least one filepath is required, if not passed to App during init")
-
parameters = kwargs.copy()
+
# Append filename extension to filenames
if ext_fname:
logger.debug("%s: using extended filename for outputs: %s", self.name, ext_fname)
@@ -673,10 +673,11 @@ class App(OTBObject):
self.propagate_dtype(key, data_types[key])
self.set_parameters({key: output_filename})
self.flush()
+ # Search and log missing files
files, missing = [], []
- for out in self.used_outputs:
- dest = files if out.exists() else missing
- dest.append(str(out.filepath.absolute()))
+ for key, output_filename in parameters.items():
+ dest = files if Path(output_filename).exists() else missing
+ dest.append(str(Path(output_filename).absolute()))
for filename in missing:
logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
return bool(files) and not missing
--
GitLab
From 427c44aed88c231903d77fefcbb68769bac04123 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 5 Feb 2023 15:08:01 +0100
Subject: [PATCH 133/399] ENH: update tests, now write returns True if file
exist
---
pyotb/core.py | 5 +++--
tests/test_core.py | 23 +++++++----------------
tests/test_pipeline.py | 11 ++++-------
3 files changed, 14 insertions(+), 25 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 695039e..d6b66a8 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -676,8 +676,9 @@ class App(OTBObject):
# Search and log missing files
files, missing = [], []
for key, output_filename in parameters.items():
- dest = files if Path(output_filename).exists() else missing
- dest.append(str(Path(output_filename).absolute()))
+ filepath = Path(output_filename.split("?")[0])
+ dest = files if filepath.exists() else missing
+ dest.append(str(filepath.absolute()))
for filename in missing:
logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
return bool(files) and not missing
diff --git a/tests/test_core.py b/tests/test_core.py
index 1b6441f..824d6ad 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -69,14 +69,12 @@ def test_xy_to_rowcol():
def test_write():
- INPUT.write("/tmp/test_write.tif", ext_fname="nodata=0")
- assert INPUT["out"].exists()
+ assert INPUT.write("/tmp/test_write.tif", ext_fname="nodata=0")
INPUT["out"].filepath.unlink()
def test_output_write():
- INPUT["out"].write("/tmp/test_output_write.tif")
- assert INPUT["out"].exists()
+ assert INPUT["out"].write("/tmp/test_output_write.tif")
INPUT["out"].filepath.unlink()
@@ -151,28 +149,21 @@ def test_ndvi_comparison():
ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (INPUT[:, :, -1] + INPUT[:, :, 0])
ndvi_indices = pyotb.RadiometricIndices(INPUT, {"list": ["Vegetation:NDVI"], "channels.red": 1, "channels.nir": 4})
assert ndvi_bandmath.exp == "((im1b4 - im1b1) / (im1b4 + im1b1))"
-
- ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", pixel_type="float")
- assert ndvi_bandmath["out"].exists()
- ndvi_indices.write("/tmp/ndvi_indices.tif", pixel_type="float")
- assert ndvi_indices["out"].exists()
+ assert ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", pixel_type="float")
+ assert ndvi_indices.write("/tmp/ndvi_indices.tif", pixel_type="float")
compared = pyotb.CompareImages({"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"})
assert (compared["count"], compared["mse"]) == (0, 0)
-
thresholded_indices = pyotb.where(ndvi_indices >= 0.3, 1, 0)
assert thresholded_indices.exp == "((im1b1 >= 0.3) ? 1 : 0)"
-
thresholded_bandmath = pyotb.where(ndvi_bandmath >= 0.3, 1, 0)
assert thresholded_bandmath.exp == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)"
def test_output_in_arg():
- o = pyotb.Output(INPUT, "out")
- t = pyotb.ReadImageInfo(o)
- assert t
+ t = pyotb.ReadImageInfo(INPUT["out"])
+ assert t.data
def test_output_summary():
- o = pyotb.Output(INPUT, "out")
- assert o.summarize()
+ assert INPUT["out"].summarize()
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index 19160c5..eb3b51f 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -123,18 +123,16 @@ def test_pipeline_write(pipe):
out = f"/tmp/out_{i}.tif"
if os.path.isfile(out):
os.remove(out)
- app.write(out)
- assert os.path.isfile(out)
+ assert app.write(out)
@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES)
def test_pipeline_write_nointermediate(pipe):
app = [pipe[-1]][0]
- out = f"/tmp/out_0.tif"
+ out = "/tmp/out_0.tif"
if os.path.isfile(out):
os.remove(out)
- app.write(out)
- assert os.path.isfile(out)
+ assert app.write(out)
@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES)
@@ -143,5 +141,4 @@ def test_pipeline_write_backward(pipe):
out = f"/tmp/out_{i}.tif"
if os.path.isfile(out):
os.remove(out)
- app.write(out)
- assert os.path.isfile(out)
+ assert app.write(out)
--
GitLab
From c109323c9974cf35fd1dd543b6d3eec445f257ed Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 5 Feb 2023 15:12:06 +0100
Subject: [PATCH 134/399] ENH: move summarize back to App, fix Output summarize
parent app
---
pyotb/core.py | 36 ++++++++++++++++++++----------------
1 file changed, 20 insertions(+), 16 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index d6b66a8..2092d4a 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -196,22 +196,6 @@ class OTBObject(ABC):
row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x
return abs(int(row)), int(col)
- def summarize(self) -> dict[str, str | dict[str, Any]]:
- """Serialize an object and its pipeline into a dictionary.
-
- Returns:
- nested dictionary summarizing the pipeline
-
- """
- parameters = self.parameters.copy()
- for key, param in parameters.items():
- # In the following, we replace each parameter which is an OTBObject, with its summary.
- if isinstance(param, OTBObject): # single parameter
- parameters[key] = param.summarize()
- elif isinstance(param, list): # parameter list
- parameters[key] = [p.summarize() if isinstance(p, OTBObject) else p for p in param]
- return {"name": self.app.GetName(), "parameters": parameters}
-
@staticmethod
def __create_operator(op_cls, name, x, y) -> Operation:
"""Create an operator.
@@ -683,6 +667,22 @@ class App(OTBObject):
logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
return bool(files) and not missing
+ def summarize(self) -> dict[str, str | dict[str, Any]]:
+ """Serialize an object and its pipeline into a dictionary.
+
+ Returns:
+ nested dictionary summarizing the pipeline
+
+ """
+ parameters = self.parameters.copy()
+ for key, param in parameters.items():
+ # In the following, we replace each parameter which is an OTBObject, with its summary.
+ if isinstance(param, OTBObject): # single parameter
+ parameters[key] = param.summarize()
+ elif isinstance(param, list): # parameter list
+ parameters[key] = [p.summarize() if isinstance(p, OTBObject) else p for p in param]
+ return {"name": self.app.GetName(), "parameters": parameters}
+
# Private functions
def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]:
"""Gather all input arguments in kwargs dict.
@@ -1149,6 +1149,10 @@ class Output(OTBObject):
return self.parent_pyotb_app.write({self.output_image_key: self.filepath}, **kwargs)
return self.parent_pyotb_app.write({self.output_image_key: filepath}, **kwargs)
+ def summarize(self):
+ """Summarize parent pyotb App."""
+ return self.parent_pyotb_app.summarize()
+
def __str__(self) -> str:
"""Return string representation of Output filepath."""
return str(self.filepath)
--
GitLab
From 394e8d148bfa7a9111c73696cc02f6e5ff6cdbb8 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 5 Feb 2023 15:29:16 +0100
Subject: [PATCH 135/399] ENH: save Output obj before skipping empty key
---
pyotb/core.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 2092d4a..7a77c30 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -549,11 +549,11 @@ class App(OTBObject):
try:
value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results)
except RuntimeError:
- continue # this is when there is no value for key
- if value is None or isinstance(value, otb.ApplicationProxy):
- continue
+ pass # this is when there is no value for key
if self._out_param_types.get(key) == otb.ParameterType_OutputImage:
self.outputs[key] = Output(self, key, value)
+ if value is None or isinstance(value, otb.ApplicationProxy):
+ continue
if isinstance(value, OTBObject) or bool(value) or value == 0:
if self.app.GetParameterRole(key) == 0:
self.parameters[key] = value
--
GitLab
From 0ff7abd62c031ebdc4910dec285e00f8fd4a3a9a Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 5 Feb 2023 15:51:54 +0100
Subject: [PATCH 136/399] CI: linter
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 7a77c30..17db15f 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -549,7 +549,7 @@ class App(OTBObject):
try:
value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results)
except RuntimeError:
- pass # this is when there is no value for key
+ pass # undefined parameter
if self._out_param_types.get(key) == otb.ParameterType_OutputImage:
self.outputs[key] = Output(self, key, value)
if value is None or isinstance(value, otb.ApplicationProxy):
--
GitLab
From 9082d0b7da3acd5411e0a75650387fbf1ad37b30 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 5 Feb 2023 16:51:37 +0100
Subject: [PATCH 137/399] ENH: remove used_outputs
---
pyotb/core.py | 5 -----
1 file changed, 5 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 17db15f..170ee9a 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -464,11 +464,6 @@ class App(OTBObject):
"""Get elapsed time between app init and end of exec or file writing."""
return self._time_end - self._time_start
- @property
- def used_outputs(self) -> list[str]:
- """List of used application outputs."""
- return [self.outputs[key] for key in self._out_param_types if key in self.parameters]
-
def set_parameters(self, *args, **kwargs):
"""Set some parameters of the app.
--
GitLab
From bd4b6e1ec936186b5fc0e78e864ebeb1e8d1bb0d Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 5 Feb 2023 17:35:16 +0100
Subject: [PATCH 138/399] ENH: remove image_dic
---
pyotb/core.py | 27 +++++++++++----------------
1 file changed, 11 insertions(+), 16 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 170ee9a..bd3013a 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -163,8 +163,7 @@ class OTBObject(ABC):
"""
data = self.export(key, preserve_dtype)
- array = data["array"]
- return array.copy() if copy else array
+ return data["array"].copy() if copy else data["array"]
def to_rasterio(self) -> tuple[np.ndarray, dict[str, Any]]:
"""Export image as a numpy array and its metadata compatible with rasterio.
@@ -374,7 +373,7 @@ class OTBObject(ABC):
result_dic = image_dic
result_dic["array"] = result_array
# Importing back to OTB, pass the result_dic just to keep reference
- pyotb_app = App("ExtractROI", image_dic=result_dic, frozen=True, quiet=True)
+ pyotb_app = App("ExtractROI", frozen=True, quiet=True)
if result_array.shape[2] == 1:
pyotb_app.app.ImportImage("in", result_dic)
else:
@@ -387,7 +386,7 @@ class OTBObject(ABC):
class App(OTBObject):
"""Base class that gathers common operations for any OTB application."""
- def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, image_dic: dict = None, **kwargs):
+ def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, **kwargs):
"""Common constructor for OTB applications. Handles in-memory connection between apps.
Args:
@@ -399,32 +398,28 @@ class App(OTBObject):
- list, useful when the user wants to specify the input list 'il'
frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___
quiet: whether to print logs of the OTB app
- image_dic: enables to keep a reference to image_dic. image_dic is a dictionary, such as
- the result of app.ExportImage(). Use it when the app takes a numpy array as input.
- See this related issue for why it is necessary to keep reference of object:
- https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/1824
**kwargs: used for passing application parameters.
e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif'
"""
- self.name, self.image_dic = name, image_dic
+ self.name = name
self.quiet, self.frozen = quiet, frozen
- self._time_start, self._time_end = 0., 0.
- self.data, self.parameters, self.outputs, self.exports_dic = {}, {}, {}, {}
- # Initialize app, set parameters and execute if not frozen
- create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
- self.app = create(name)
+ self.data, self.parameters = {}, {} # params from self.app.GetParameterValue()
+ self.outputs, self.exports_dic = {}, {} # Outputs objects and numpy arrays exports
+ self.app = otb.Registry.CreateApplicationWithoutLogger(name) if quiet else otb.Registry.CreateApplication(name)
self.parameters_keys = tuple(self.app.GetParametersKeys())
self._all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys}
types = (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename)
self._out_param_types = {k: v for k, v in self._all_param_types.items() if v in types}
+ # Init, execute and write (auto flush only when output param was provided)
+ self._time_start, self._time_end = 0., 0.
if args or kwargs:
self.set_parameters(*args, **kwargs)
if not self.frozen:
self.execute()
if any(key in self.parameters for key in self._out_param_types):
- self.flush() # auto flush if any output param was provided during app init
+ self.flush()
def get_first_key(self, *type_lists: tuple[list[int]]) -> str:
"""Get the first param key for specific file types, try each list in args."""
@@ -733,7 +728,7 @@ class App(OTBObject):
We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer
"""
if isinstance(key, tuple):
- return super().__getitem__(key)
+ return super().__getitem__(key) # to read pixel values, or slice
if isinstance(key, str):
if key in self.data:
return self.data[key]
--
GitLab
From 791da78558f2678489b3e6acef0df826bba5b31e Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 5 Feb 2023 17:38:19 +0100
Subject: [PATCH 139/399] STYLE: move functions
---
pyotb/core.py | 91 +++++++++++++++++++++++++--------------------------
1 file changed, 45 insertions(+), 46 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index bd3013a..298367c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -213,52 +213,6 @@ class OTBObject(ABC):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return op_cls(name, x, y)
- # Special functions
- def __hash__(self) -> int:
- """Override the default behaviour of the hash function.
-
- Returns:
- self hash
-
- """
- return id(self)
-
- def __getitem__(self, key) -> Any | list[float] | float | Slicer:
- """Override the default __getitem__ behaviour.
-
- This function enables 2 things :
- - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3]
- selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]]
- selecting 1000x1000 subset : object[:1000, :1000]
- - access pixel value(s) at a specified row, col index
-
- Args:
- key: attribute key
-
- Returns:
- list of pixel values if vector image, or pixel value, or Slicer
-
- """
- # Accessing pixel value(s) using Y/X coordinates
- if isinstance(key, tuple) and len(key) >= 2:
- row, col = key[0], key[1]
- if isinstance(row, int) and isinstance(col, int):
- if row < 0 or col < 0:
- raise ValueError(f"{self.name} cannot read pixel value at negative coordinates ({row}, {col})")
- channels = key[2] if len(key) == 3 else None
- return self.get_values_at_coords(row, col, channels)
- # Slicing
- if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)):
- raise ValueError(f'"{key}" cannot be interpreted as valid slicing. Slicing should be 2D or 3D.')
- if isinstance(key, tuple) and len(key) == 2:
- # Adding a 3rd dimension
- key = key + (slice(None, None, None),)
- return Slicer(self, *key)
-
- def __repr__(self) -> str:
- """Return a nice string representation with object id."""
- return f"<pyotb.{self.__class__.__name__} object id {id(self)}>"
-
def __add__(self, other: OTBObject | str | int | float) -> Operation:
"""Addition."""
return self.__create_operator(Operation, "+", self, other)
@@ -382,6 +336,51 @@ class OTBObject(ABC):
return pyotb_app
return NotImplemented
+ def __hash__(self) -> int:
+ """Override the default behaviour of the hash function.
+
+ Returns:
+ self hash
+
+ """
+ return id(self)
+
+ def __getitem__(self, key) -> Any | list[float] | float | Slicer:
+ """Override the default __getitem__ behaviour.
+
+ This function enables 2 things :
+ - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3]
+ selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]]
+ selecting 1000x1000 subset : object[:1000, :1000]
+ - access pixel value(s) at a specified row, col index
+
+ Args:
+ key: attribute key
+
+ Returns:
+ list of pixel values if vector image, or pixel value, or Slicer
+
+ """
+ # Accessing pixel value(s) using Y/X coordinates
+ if isinstance(key, tuple) and len(key) >= 2:
+ row, col = key[0], key[1]
+ if isinstance(row, int) and isinstance(col, int):
+ if row < 0 or col < 0:
+ raise ValueError(f"{self.name} cannot read pixel value at negative coordinates ({row}, {col})")
+ channels = key[2] if len(key) == 3 else None
+ return self.get_values_at_coords(row, col, channels)
+ # Slicing
+ if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)):
+ raise ValueError(f'"{key}" cannot be interpreted as valid slicing. Slicing should be 2D or 3D.')
+ if isinstance(key, tuple) and len(key) == 2:
+ # Adding a 3rd dimension
+ key = key + (slice(None, None, None),)
+ return Slicer(self, *key)
+
+ def __repr__(self) -> str:
+ """Return a nice string representation with object id."""
+ return f"<pyotb.{self.__class__.__name__} object id {id(self)}>"
+
class App(OTBObject):
"""Base class that gathers common operations for any OTB application."""
--
GitLab
From c8c74094254196bc5929b816028ab0a90e4edc7b Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 5 Feb 2023 18:57:39 +0100
Subject: [PATCH 140/399] FIX: Operation expressions are stored using OTBObject
repr with id, instead of str
---
pyotb/core.py | 28 ++++++++++++++--------------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 298367c..30b67e0 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -379,7 +379,7 @@ class OTBObject(ABC):
def __repr__(self) -> str:
"""Return a nice string representation with object id."""
- return f"<pyotb.{self.__class__.__name__} object id {id(self)}>"
+ return f"<pyotb.{self.__class__.__name__} object, id {id(self)}>"
class App(OTBObject):
@@ -852,15 +852,15 @@ class Operation(App):
# NB: the keys of the dictionary are strings-only, instead of 'complex' objects, to enable easy serialization
self.im_dic = {}
self.im_count = 1
- mapping_str_to_input = {} # to be able to retrieve the real python object from its string representation
+ map_repr_to_input = {} # to be able to retrieve the real python object from its string representation
for inp in self.inputs:
if not isinstance(inp, (int, float)):
if str(inp) not in self.im_dic:
- self.im_dic[str(inp)] = f"im{self.im_count}"
- mapping_str_to_input[str(inp)] = inp
+ self.im_dic[repr(inp)] = f"im{self.im_count}"
+ map_repr_to_input[repr(inp)] = inp
self.im_count += 1
# Getting unique image inputs, in the order im1, im2, im3 ...
- self.unique_inputs = [mapping_str_to_input[str_input] for str_input in sorted(self.im_dic, key=self.im_dic.get)]
+ self.unique_inputs = [map_repr_to_input[id_str] for id_str in sorted(self.im_dic, key=self.im_dic.get)]
self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands)
appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX"
# Execute app
@@ -943,7 +943,7 @@ class Operation(App):
one_band_exp = one_band_fake_exp
for inp in self.inputs:
# Replace the name of in-memory object (e.g. '<pyotb.App object>b1' by 'im1b1')
- one_band_exp = one_band_exp.replace(str(inp), self.im_dic[str(inp)])
+ one_band_exp = one_band_exp.replace(repr(inp), self.im_dic[repr(inp)])
exp_bands.append(one_band_exp)
# Form the final expression (e.g. 'im1b1 + 1; im1b2 + 1')
return exp_bands, ";".join(exp_bands)
@@ -979,8 +979,8 @@ class Operation(App):
inputs, nb_channels = x.input.inputs, x.input.nb_channels
else:
# Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1')
- fake_exp = f"{x.input}b{x.one_band_sliced}"
- inputs, nb_channels = [x.input], {x.input: 1}
+ fake_exp = f"{repr(x.input)}b{x.one_band_sliced}"
+ inputs, nb_channels = [x.input], {repr(x.input): 1}
# For LogicalOperation, we save almost the same attributes as an Operation
elif keep_logical and isinstance(x, LogicalOperation):
fake_exp = x.logical_fake_exp_bands[band - 1]
@@ -995,12 +995,12 @@ class Operation(App):
# We go on with other inputs, i.e. pyotb objects, filepaths...
else:
# Add the band number (e.g. replace '<pyotb.App object>' by '<pyotb.App object>b1')
- fake_exp = f"{x}b{band}"
- inputs, nb_channels = [x], {x: get_nbchannels(x)}
+ fake_exp = f"{repr(x)}b{band}"
+ inputs, nb_channels = [x], {repr(x): get_nbchannels(x)}
return fake_exp, inputs, nb_channels
- def __str__(self) -> str:
+ def __repr__(self) -> str:
"""Return a nice string representation with operator and object id."""
return f"<pyotb.Operation `{self.operator}` object, id {id(self)}>"
@@ -1083,9 +1083,9 @@ class Input(App):
self.propagate_dtype()
self.execute()
- def __str__(self) -> str:
- """Return a nice string representation with file path."""
- return f"<pyotb.Input object from {self.filepath}>"
+ def __repr__(self) -> str:
+ """Return a string representation with file path, used in Operation to store file ref."""
+ return f"<pyotb.Input object, from {self.filepath}>"
class Output(OTBObject):
--
GitLab
From 0270376d522c0701958db9f7f2ccbe250a3430aa Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 5 Feb 2023 19:35:58 +0100
Subject: [PATCH 141/399] CI: merge serialize tests in test_core, add json for
summarize() data
---
.gitlab-ci.yml | 5 --
tests/serialized_apps.json | 132 ++++++++++++++++++++++++++++++++++++
tests/test_core.py | 33 +++++++--
tests/test_serialization.py | 21 ------
tests/tests_data.py | 43 +++---------
5 files changed, 166 insertions(+), 68 deletions(-)
create mode 100644 tests/serialized_apps.json
delete mode 100644 tests/test_serialization.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a38149e..a60518f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -104,11 +104,6 @@ test_pipeline:
script:
- python3 -m pytest --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py
-test_serialization:
- extends: .tests
- script:
- - python3 -m pytest --color=yes --junitxml=test-serialization.xml tests/test_serialization.py
-
# -------------------------------------- Ship ---------------------------------------
pages:
diff --git a/tests/serialized_apps.json b/tests/serialized_apps.json
new file mode 100644
index 0000000..088e853
--- /dev/null
+++ b/tests/serialized_apps.json
@@ -0,0 +1,132 @@
+{
+ "SIMPLE": {
+ "name": "ManageNoData",
+ "parameters": {
+ "in": {
+ "name": "OrthoRectification",
+ "parameters": {
+ "io.in": {
+ "name": "BandMath",
+ "parameters": {
+ "il": [
+ "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
+ ],
+ "exp": "im1b1",
+ "ram": 256
+ }
+ },
+ "map.utm.zone": 31,
+ "map.utm.northhem": true,
+ "map.epsg.code": 4326,
+ "outputs.isotropic": true,
+ "outputs.default": 0.0,
+ "elev.default": 0.0,
+ "interpolator.bco.radius": 2,
+ "opt.rpc": 10,
+ "opt.ram": 256,
+ "opt.gridspacing": 4.0,
+ "outputs.ulx": 560000.8125,
+ "outputs.uly": 5495732.5,
+ "outputs.sizex": 251,
+ "outputs.sizey": 304,
+ "outputs.spacingx": 5.997312068939209,
+ "outputs.spacingy": -5.997312068939209,
+ "outputs.lrx": 561506.125,
+ "outputs.lry": 5493909.5
+ }
+ },
+ "usenan": false,
+ "mode.buildmask.inv": 1.0,
+ "mode.buildmask.outv": 0.0,
+ "mode.changevalue.newv": 0.0,
+ "mode.apply.ndval": 0.0,
+ "ram": 256
+ }
+ },
+ "COMPLEX": {
+ "name": "BandMathX",
+ "parameters": {
+ "il": [
+ {
+ "name": "OrthoRectification",
+ "parameters": {
+ "io.in": {
+ "name": "BandMath",
+ "parameters": {
+ "il": [
+ "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
+ ],
+ "exp": "im1b1",
+ "ram": 256
+ }
+ },
+ "map.utm.zone": 31,
+ "map.utm.northhem": true,
+ "map.epsg.code": 4326,
+ "outputs.isotropic": true,
+ "outputs.default": 0.0,
+ "elev.default": 0.0,
+ "interpolator.bco.radius": 2,
+ "opt.rpc": 10,
+ "opt.ram": 256,
+ "opt.gridspacing": 4.0,
+ "outputs.ulx": 560000.8125,
+ "outputs.uly": 5495732.5,
+ "outputs.sizex": 251,
+ "outputs.sizey": 304,
+ "outputs.spacingx": 5.997312068939209,
+ "outputs.spacingy": -5.997312068939209,
+ "outputs.lrx": 561506.125,
+ "outputs.lry": 5493909.5
+ }
+ },
+ {
+ "name": "ManageNoData",
+ "parameters": {
+ "in": {
+ "name": "OrthoRectification",
+ "parameters": {
+ "io.in": {
+ "name": "BandMath",
+ "parameters": {
+ "il": [
+ "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
+ ],
+ "exp": "im1b1",
+ "ram": 256
+ }
+ },
+ "map.utm.zone": 31,
+ "map.utm.northhem": true,
+ "map.epsg.code": 4326,
+ "outputs.isotropic": true,
+ "outputs.default": 0.0,
+ "elev.default": 0.0,
+ "interpolator.bco.radius": 2,
+ "opt.rpc": 10,
+ "opt.ram": 256,
+ "opt.gridspacing": 4.0,
+ "outputs.ulx": 560000.8125,
+ "outputs.uly": 5495732.5,
+ "outputs.sizex": 251,
+ "outputs.sizey": 304,
+ "outputs.spacingx": 5.997312068939209,
+ "outputs.spacingy": -5.997312068939209,
+ "outputs.lrx": 561506.125,
+ "outputs.lry": 5493909.5
+ }
+ },
+ "usenan": false,
+ "mode.buildmask.inv": 1.0,
+ "mode.buildmask.outv": 0.0,
+ "mode.changevalue.newv": 0.0,
+ "mode.apply.ndval": 0.0,
+ "ram": 256
+ }
+ }
+ ],
+ "exp": "im1+im2",
+ "ram": 256
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/test_core.py b/tests/test_core.py
index 824d6ad..d3c7d59 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -1,7 +1,7 @@
import pytest
import pyotb
-from tests_data import INPUT, TEST_IMAGE_STATS
+from tests_data import *
# Input settings
@@ -53,7 +53,7 @@ def test_elapsed_time():
# Other functions
-def test_get_infos():
+def test_get_info():
infos = INPUT.get_info()
assert (infos["sizex"], infos["sizey"]) == (251, 304)
bm_infos = pyotb.BandMathX([INPUT], exp="im1")["out"].get_info()
@@ -78,6 +78,15 @@ def test_output_write():
INPUT["out"].filepath.unlink()
+def test_output_in_arg():
+ t = pyotb.ReadImageInfo(INPUT["out"])
+ assert t.data
+
+
+def test_output_summary():
+ assert INPUT["out"].summarize()
+
+
# Slicer
def test_slicer_shape():
extract = INPUT[:50, :60, :3]
@@ -160,10 +169,20 @@ def test_ndvi_comparison():
assert thresholded_bandmath.exp == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)"
-def test_output_in_arg():
- t = pyotb.ReadImageInfo(INPUT["out"])
- assert t.data
+def test_pipeline_simple():
+ # BandMath -> OrthoRectification -> ManageNoData
+ app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"})
+ app2 = pyotb.OrthoRectification({"io.in": app1})
+ app3 = pyotb.ManageNoData({"in": app2})
+ summary = app3.summarize()
+ assert summary == SIMPLE_SERIALIZATION
-def test_output_summary():
- assert INPUT["out"].summarize()
+def test_pipeline_diamond():
+ # Diamond graph
+ app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"})
+ app2 = pyotb.OrthoRectification({"io.in": app1})
+ app3 = pyotb.ManageNoData({"in": app2})
+ app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"})
+ summary = app4.summarize()
+ assert summary == COMPLEX_SERIALIZATION
diff --git a/tests/test_serialization.py b/tests/test_serialization.py
deleted file mode 100644
index 28aa3a7..0000000
--- a/tests/test_serialization.py
+++ /dev/null
@@ -1,21 +0,0 @@
-import pyotb
-from tests_data import *
-
-
-def test_pipeline_simple():
- # BandMath -> OrthoRectification -> ManageNoData
- app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"})
- app2 = pyotb.OrthoRectification({"io.in": app1})
- app3 = pyotb.ManageNoData({"in": app2})
- summary = app3.summarize()
- assert summary == SIMPLE_SERIALIZATION
-
-
-def test_pipeline_diamond():
- # Diamond graph
- app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"})
- app2 = pyotb.OrthoRectification({"io.in": app1})
- app3 = pyotb.ManageNoData({"in": app2})
- app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"})
- summary = app4.summarize()
- assert summary == COMPLEX_SERIALIZATION
diff --git a/tests/tests_data.py b/tests/tests_data.py
index de5d727..bfb774b 100644
--- a/tests/tests_data.py
+++ b/tests/tests_data.py
@@ -1,45 +1,18 @@
+import json
+from pathlib import Path
import pyotb
FILEPATH = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
INPUT = pyotb.Input(FILEPATH)
-
TEST_IMAGE_STATS = {
'out.mean': [79.5505, 109.225, 115.456, 249.349],
'out.min': [33, 64, 91, 47],
'out.max': [255, 255, 230, 255],
'out.std': [51.0754, 35.3152, 23.4514, 20.3827]
}
-SIMPLE_SERIALIZATION = {
- 'name': 'ManageNoData',
- 'parameters': {'in': {'name': 'OrthoRectification',
- 'parameters': {'io.in': {'name': 'BandMath',
- 'parameters': {'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'],
- 'exp': 'im1b1', 'ram': 256}},
- 'map.utm.zone': 31, 'map.utm.northhem': True, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0,
- 'elev.default': 0.0, 'interpolator.bco.radius': 2, 'opt.rpc': 10, 'opt.ram': 256,
- 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5, 'outputs.sizex': 251, 'outputs.sizey': 304,
- 'outputs.spacingx': 5.997312068939209, 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}},
- 'usenan': False, 'mode.buildmask.inv': 1.0, 'mode.buildmask.outv': 0.0, 'mode.changevalue.newv': 0.0, 'mode.apply.ndval': 0.0, 'ram': 256}
-}
-COMPLEX_SERIALIZATION = {
- 'name': 'BandMathX',
- 'parameters': {'il': [{'name': 'OrthoRectification',
- 'parameters': {'io.in': {'name': 'BandMath',
- 'parameters': {
- 'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'],
- 'exp': 'im1b1', 'ram': 256}},
- 'map.utm.zone': 31, 'map.utm.northhem': True, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, 'elev.default': 0.0,
- 'interpolator.bco.radius': 2, 'opt.rpc': 10, 'opt.ram': 256, 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5,
- 'outputs.sizex': 251, 'outputs.sizey': 304, 'outputs.spacingx': 5.997312068939209, 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}},
- {'name': 'ManageNoData',
- 'parameters': {'in': {'name': 'OrthoRectification',
- 'parameters': {'io.in': {'name': 'BandMath',
- 'parameters': {'il': ['/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif'],
- 'exp': 'im1b1', 'ram': 256}},
- 'map.utm.zone': 31, 'map.utm.northhem': True, 'map.epsg.code': 4326, 'outputs.isotropic': True, 'outputs.default': 0.0, 'elev.default': 0.0, 'interpolator.bco.radius': 2,
- 'opt.rpc': 10, 'opt.ram': 256, 'opt.gridspacing': 4.0, 'outputs.ulx': 560000.8125, 'outputs.uly': 5495732.5,
- 'outputs.sizex': 251, 'outputs.sizey': 304, 'outputs.spacingx': 5.997312068939209,
- 'outputs.spacingy': -5.997312068939209, 'outputs.lrx': 561506.125, 'outputs.lry': 5493909.5}},
- 'usenan': False, 'mode.buildmask.inv': 1.0, 'mode.buildmask.outv': 0.0, 'mode.changevalue.newv': 0.0, 'mode.apply.ndval': 0.0, 'ram': 256}}],
- 'exp': 'im1+im2', 'ram': 256}
-}
+
+json_file = Path(__file__).parent / "serialized_apps.json"
+with json_file.open("r") as js:
+ data = json.load(js)
+SIMPLE_SERIALIZATION = data["SIMPLE"]
+COMPLEX_SERIALIZATION = data["COMPLEX"]
--
GitLab
From 938de5798bc8f464f5f61f880697b53ba923aff4 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 5 Feb 2023 20:56:38 +0100
Subject: [PATCH 142/399] STYLE: docstrings and type hints
---
pyotb/core.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 30b67e0..eebae77 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -175,9 +175,9 @@ class OTBObject(ABC):
"""
profile = {}
array = self.to_numpy(preserve_dtype=True, copy=False)
- height, width, count = array.shape
proj = self.app.GetImageProjection(self.output_image_key)
profile.update({"crs": proj, "dtype": array.dtype, "transform": self.transform})
+ height, width, count = array.shape
profile.update({"count": count, "height": height, "width": width})
return np.moveaxis(array, 2, 0), profile
@@ -378,7 +378,7 @@ class OTBObject(ABC):
return Slicer(self, *key)
def __repr__(self) -> str:
- """Return a nice string representation with object id."""
+ """Return a string representation with object id, this is a key used to store image ref in Operation dicts."""
return f"<pyotb.{self.__class__.__name__} object, id {id(self)}>"
@@ -742,7 +742,7 @@ class App(OTBObject):
class Slicer(App):
"""Slicer objects i.e. when we call something like raster[:, :, 2] from Python."""
- def __init__(self, obj: App | str, rows: int, cols: int, channels: int):
+ def __init__(self, obj: App | str, rows: slice, cols: slice, channels: slice | list[int] | int):
"""Create a slicer object, that can be used directly for writing or inside a BandMath.
It contains :
--
GitLab
From 10346ab018152f418b07b04b85a5c811b9ac75fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr>
Date: Mon, 6 Feb 2023 09:51:38 +0000
Subject: [PATCH 143/399] Apply 1 suggestion(s) to 1 file(s)
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index eebae77..309fda6 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -625,7 +625,7 @@ class App(OTBObject):
parameters[key] = value + ext_fname
# Manage output pixel types
data_types = {}
- if pixel_type is not None:
+ if pixel_type:
if isinstance(pixel_type, str):
dtype = parse_pixel_type(pixel_type)
type_name = self.app.ConvertPixelTypeToNumpy(dtype)
--
GitLab
From b1b52e2c528518dd1bf8171446168fc689b99d6d Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Feb 2023 10:52:18 +0100
Subject: [PATCH 144/399] ENH: reuse parse_pixel_type in get_pixel_type, or
raise exception
---
pyotb/core.py | 38 ++++++++++++++++++++------------------
1 file changed, 20 insertions(+), 18 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 309fda6..c78b3dc 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1189,20 +1189,7 @@ def get_pixel_type(inp: str | Path | OTBObject) -> str:
except RuntimeError as info_err: # this happens when we pass a str that is not a filepath
raise TypeError(f"Could not get the pixel type of `{inp}` ({info_err})") from info_err
if datatype:
- datatype_to_pixeltype = {
- "unsigned_char": "uint8",
- "short": "int16",
- "unsigned_short": "uint16",
- "int": "int32",
- "unsigned_int": "uint32",
- "long": "int32",
- "ulong": "uint32",
- "float": "float",
- "double": "double",
- }
- if datatype not in datatype_to_pixeltype:
- raise TypeError(f"Unknown data type `{datatype}`. Available ones: {datatype_to_pixeltype}")
- return getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[datatype]}")
+ return parse_pixel_type(datatype)
raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}")
@@ -1216,11 +1203,26 @@ def parse_pixel_type(pixel_type: str | int) -> int:
pixel_type integer value
"""
- if isinstance(pixel_type, str): # this correspond to 'uint8' etc...
- return getattr(otb, f"ImagePixelType_{pixel_type}")
- if isinstance(pixel_type, int):
+ if isinstance(pixel_type, int): # normal OTB int enum
return pixel_type
- raise ValueError(f"Bad pixel type specification ({pixel_type})")
+ if isinstance(pixel_type, str): # correspond to 'uint8' etc...
+ datatype_to_pixeltype = {
+ "unsigned_char": "uint8",
+ "short": "int16",
+ "unsigned_short": "uint16",
+ "int": "int32",
+ "unsigned_int": "uint32",
+ "long": "int32",
+ "ulong": "uint32",
+ "float": "float",
+ "double": "double",
+ }
+ if pixel_type in datatype_to_pixeltype.keys():
+ return getattr(otb, f"ImagePixelType_{pixel_type}")
+ if pixel_type in datatype_to_pixeltype:
+ return getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[pixel_type]}")
+ raise KeyError(f"Unknown data type `{pixel_type}`. Available ones: {datatype_to_pixeltype}")
+ raise TypeError(f"Bad pixel type specification ({pixel_type} of type {type(pixel_type)})")
def is_key_list(pyotb_app: OTBObject, key: str) -> bool:
--
GitLab
From 212011170ab1f894fcd2df301f78ca63acd406ef Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Feb 2023 10:57:25 +0100
Subject: [PATCH 145/399] ENH: fix parse_pixel_type
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index c78b3dc..859c2e6 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1217,7 +1217,7 @@ def parse_pixel_type(pixel_type: str | int) -> int:
"float": "float",
"double": "double",
}
- if pixel_type in datatype_to_pixeltype.keys():
+ if pixel_type in datatype_to_pixeltype.values():
return getattr(otb, f"ImagePixelType_{pixel_type}")
if pixel_type in datatype_to_pixeltype:
return getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[pixel_type]}")
--
GitLab
From 8072693f42847568e11a2e496baab68b966057d3 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Feb 2023 11:07:36 +0100
Subject: [PATCH 146/399] ENH: fix potential problems with Path objects and
/vsi URLs
---
pyotb/core.py | 37 ++++++++++++++++++-------------------
1 file changed, 18 insertions(+), 19 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 859c2e6..66a6906 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -639,19 +639,20 @@ class App(OTBObject):
self.propagate_dtype() # all outputs will have the same type as the main input raster
# Set parameters and flush to disk
- for key, output_filename in parameters.items():
- if Path(output_filename.split("?")[0]).exists():
- logger.warning("%s: overwriting file %s", self.name, output_filename)
+ for key, filepath in parameters.items():
+ if Path(filepath.split("?")[0]).exists():
+ logger.warning("%s: overwriting file %s", self.name, filepath)
if key in data_types:
self.propagate_dtype(key, data_types[key])
- self.set_parameters({key: output_filename})
+ self.set_parameters({key: filepath})
self.flush()
# Search and log missing files
files, missing = [], []
- for key, output_filename in parameters.items():
- filepath = Path(output_filename.split("?")[0])
- dest = files if filepath.exists() else missing
- dest.append(str(filepath.absolute()))
+ for key, filepath in parameters.items():
+ if not filepath.startswith("/vsi"):
+ filepath = Path(filepath.split("?")[0])
+ dest = files if filepath.exists() else missing
+ dest.append(str(filepath.absolute()))
for filename in missing:
logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
return bool(files) and not missing
@@ -1079,7 +1080,7 @@ class Input(App):
"""
super().__init__("ExtractROI", {"in": filepath}, frozen=True)
self.name = f"Input from {filepath}"
- self.filepath = Path(filepath)
+ self.filepath = Path(filepath) if not filepath.startswith("/vsi") else filepath
self.propagate_dtype()
self.execute()
@@ -1091,7 +1092,7 @@ class Input(App):
class Output(OTBObject):
"""Object that behave like a pointer to a specific application output file."""
- def __init__(self, pyotb_app: App, param_key: str = None, filepath: str = None, mkdir: bool = True):
+ def __init__(self, pyotb_app: App, param_key: str = None, filepath: str = "", mkdir: bool = True):
"""Constructor for an Output object.
Args:
@@ -1107,11 +1108,9 @@ class Output(OTBObject):
self.exports_dic = pyotb_app.exports_dic
self.param_key = param_key
self.parameters = self.parent_pyotb_app.parameters
- self.filepath = None
- if filepath:
- if "?" in filepath:
- filepath = filepath.split("?")[0]
- self.filepath = Path(filepath)
+ self.filepath = filepath
+ if not filepath.startswith("/vsi"):
+ self.filepath = Path(filepath.split("?")[0])
if mkdir:
self.make_parent_dirs()
@@ -1122,14 +1121,14 @@ class Output(OTBObject):
def exists(self) -> bool:
"""Check file exist."""
- if self.filepath is None:
- raise ValueError("Filepath is not set")
+ if not isinstance(self.filepath, Path):
+ raise ValueError("Filepath is not set or points to a remote URL")
return self.filepath.exists()
def make_parent_dirs(self):
"""Create missing parent directories."""
- if self.filepath is None:
- raise ValueError("Filepath is not set")
+ if not isinstance(self.filepath, Path):
+ raise ValueError("Filepath is not set or points to a remote URL")
self.filepath.parent.mkdir(parents=True, exist_ok=True)
def write(self, filepath: None | str | Path = None, **kwargs):
--
GitLab
From 70bb5e74bf047cb0d51b52d8e748976c7c0bbf64 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Feb 2023 12:00:43 +0100
Subject: [PATCH 147/399] ENH: add case filepath is not set in Output
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 66a6906..7fd543c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1109,7 +1109,7 @@ class Output(OTBObject):
self.param_key = param_key
self.parameters = self.parent_pyotb_app.parameters
self.filepath = filepath
- if not filepath.startswith("/vsi"):
+ if filepath and not filepath.startswith("/vsi"):
self.filepath = Path(filepath.split("?")[0])
if mkdir:
self.make_parent_dirs()
--
GitLab
From f98710283180d2b09d461bf996dfc4a2f8caa429 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Feb 2023 12:15:34 +0100
Subject: [PATCH 148/399] ENH: back to OTB enum in get_first_key
---
pyotb/core.py | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 7fd543c..465dbbe 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -423,35 +423,36 @@ class App(OTBObject):
def get_first_key(self, *type_lists: tuple[list[int]]) -> str:
"""Get the first param key for specific file types, try each list in args."""
for param_types in type_lists:
- types = [getattr(otb, "ParameterType_" + key) for key in param_types]
for key, value in sorted(self._all_param_types.items()):
- if value in types:
+ if value in param_types:
return key
- raise TypeError(f"{self.name}: could not find any parameter of type {tuple(chain(*type_lists))}")
+ raise KeyError(f"{self.name}: could not find any parameter key matching the provided types")
@property
def input_key(self) -> str:
"""Get the name of first input parameter, raster > vector > file."""
return self.get_first_key(
- ["InputImage", "InputImageList"],
- ["InputVectorData", "InputVectorDataList"],
- ["InputFilename", "InputFilenameList"],
+ [otb.ParameterType_InputImage, otb.ParameterType_InputImageList],
+ [otb.ParameterType_InputVectorData, otb.ParameterType_InputVectorDataList],
+ [otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList],
)
@property
def input_image_key(self) -> str:
"""Name of the first input image parameter."""
- return self.get_first_key(["InputImage", "InputImageList"])
+ return self.get_first_key([otb.ParameterType_InputImage, otb.ParameterType_InputImageList])
@property
def output_key(self) -> str:
"""Name of the first output parameter, raster > vector > file."""
- return self.get_first_key(["OutputImage"], ["OutputVectorData"], ["OutputFilename"])
+ return self.get_first_key(
+ [otb.ParameterType_OutputImage], [otb.ParameterType_OutputVectorData], [otb.ParameterType_OutputFilename]
+ )
@property
def output_image_key(self) -> str:
"""Get the name of first output image parameter."""
- return self.get_first_key(["OutputImage"])
+ return self.get_first_key([otb.ParameterType_OutputImage])
@property
def elapsed_time(self) -> float:
--
GitLab
From d18442bc406b9f4ab10e5140d6737d0fcbf6f080 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Feb 2023 12:16:20 +0100
Subject: [PATCH 149/399] ENH: gest_first_key raises TypeError
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 465dbbe..4f320f2 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -426,7 +426,7 @@ class App(OTBObject):
for key, value in sorted(self._all_param_types.items()):
if value in param_types:
return key
- raise KeyError(f"{self.name}: could not find any parameter key matching the provided types")
+ raise TypeError(f"{self.name}: could not find any parameter key matching the provided types")
@property
def input_key(self) -> str:
--
GitLab
From e216c2f8dcae089045528cd0e31bcde0c11d001b Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Feb 2023 12:25:26 +0100
Subject: [PATCH 150/399] STYLE: remove unused import from itertools
---
pyotb/core.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 4f320f2..94b5f1e 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -5,7 +5,6 @@ from __future__ import annotations
from ast import literal_eval
from pathlib import Path
from time import perf_counter
-from itertools import chain
from typing import Any
from abc import ABC, abstractmethod
--
GitLab
From 55e34924dbd5ec718422b8b3b56e09c8f7274567 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Feb 2023 14:38:41 +0100
Subject: [PATCH 151/399] ENH: move summarize back to OTBObject
---
pyotb/core.py | 36 ++++++++++++++++--------------------
1 file changed, 16 insertions(+), 20 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 94b5f1e..b64c877 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -72,6 +72,22 @@ class OTBObject(ABC):
origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2
return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y
+ def summarize(self) -> dict[str, str | dict[str, Any]]:
+ """Serialize an object and its pipeline into a dictionary.
+
+ Returns:
+ nested dictionary summarizing the pipeline
+
+ """
+ parameters = self.parameters.copy()
+ for key, param in parameters.items():
+ # In the following, we replace each parameter which is an OTBObject, with its summary.
+ if isinstance(param, OTBObject): # single parameter
+ parameters[key] = param.summarize()
+ elif isinstance(param, list): # parameter list
+ parameters[key] = [p.summarize() if isinstance(p, OTBObject) else p for p in param]
+ return {"name": self.app.GetName(), "parameters": parameters}
+
def get_info(self) -> dict[str, (str, float, list[float])]:
"""Return a dict output of ReadImageInfo for the first image output."""
return App("ReadImageInfo", self, quiet=True).data
@@ -657,22 +673,6 @@ class App(OTBObject):
logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
return bool(files) and not missing
- def summarize(self) -> dict[str, str | dict[str, Any]]:
- """Serialize an object and its pipeline into a dictionary.
-
- Returns:
- nested dictionary summarizing the pipeline
-
- """
- parameters = self.parameters.copy()
- for key, param in parameters.items():
- # In the following, we replace each parameter which is an OTBObject, with its summary.
- if isinstance(param, OTBObject): # single parameter
- parameters[key] = param.summarize()
- elif isinstance(param, list): # parameter list
- parameters[key] = [p.summarize() if isinstance(p, OTBObject) else p for p in param]
- return {"name": self.app.GetName(), "parameters": parameters}
-
# Private functions
def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]:
"""Gather all input arguments in kwargs dict.
@@ -1137,10 +1137,6 @@ class Output(OTBObject):
return self.parent_pyotb_app.write({self.output_image_key: self.filepath}, **kwargs)
return self.parent_pyotb_app.write({self.output_image_key: filepath}, **kwargs)
- def summarize(self):
- """Summarize parent pyotb App."""
- return self.parent_pyotb_app.summarize()
-
def __str__(self) -> str:
"""Return string representation of Output filepath."""
return str(self.filepath)
--
GitLab
From 22b5e9746d338411aa545c4d869b62794709bbd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr>
Date: Wed, 8 Feb 2023 13:27:04 +0000
Subject: [PATCH 152/399] More operators tests
---
tests/test_core.py | 56 ++++++++++++++++++++++++++++++++++++++--------
1 file changed, 47 insertions(+), 9 deletions(-)
diff --git a/tests/test_core.py b/tests/test_core.py
index d3c7d59..4e98d50 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -45,7 +45,7 @@ def test_metadata():
def test_nonraster_property():
with pytest.raises(TypeError):
- pyotb.ReadImageInfo(INPUT).dtype
+ assert pyotb.ReadImageInfo(INPUT).dtype == "uint8"
def test_elapsed_time():
@@ -79,12 +79,8 @@ def test_output_write():
def test_output_in_arg():
- t = pyotb.ReadImageInfo(INPUT["out"])
- assert t.data
-
-
-def test_output_summary():
- assert INPUT["out"].summarize()
+ info = pyotb.ReadImageInfo(INPUT["out"])
+ assert info.data
# Slicer
@@ -109,6 +105,44 @@ def test_slicer_in_output():
# Arithmetic
+def test_rational_operators():
+ def _test(func, exp):
+ meas = func(INPUT)
+ ref = pyotb.BandMathX({"il": [FILEPATH], "exp": exp})
+ for i in range(1, 5):
+ compared = pyotb.CompareImages({"ref.in": ref, "meas.in": meas, "ref.channel": i, "meas.channel": i})
+ assert (compared["count"], compared["mse"]) == (0, 0)
+
+ _test(lambda x: x + x, "im1 + im1")
+ _test(lambda x: x - x, "im1 - im1")
+ _test(lambda x: x / x, "im1 div im1")
+ _test(lambda x: x * x, "im1 mult im1")
+ _test(lambda x: x + FILEPATH, "im1 + im1")
+ _test(lambda x: x - FILEPATH, "im1 - im1")
+ _test(lambda x: x / FILEPATH, "im1 div im1")
+ _test(lambda x: x * FILEPATH, "im1 mult im1")
+ _test(lambda x: FILEPATH + x, "im1 + im1")
+ _test(lambda x: FILEPATH - x, "im1 - im1")
+ _test(lambda x: FILEPATH / x, "im1 div im1")
+ _test(lambda x: FILEPATH * x, "im1 mult im1")
+ _test(lambda x: x + 2, "im1 + {2;2;2;2}")
+ _test(lambda x: x - 2, "im1 - {2;2;2;2}")
+ _test(lambda x: x / 2, "0.5 * im1")
+ _test(lambda x: x * 2, "im1 * 2")
+ _test(lambda x: x + 2.0, "im1 + {2.0;2.0;2.0;2.0}")
+ _test(lambda x: x - 2.0, "im1 - {2.0;2.0;2.0;2.0}")
+ _test(lambda x: x / 2.0, "0.5 * im1")
+ _test(lambda x: x * 2.0, "im1 * 2.0")
+ _test(lambda x: 2 + x, "{2;2;2;2} + im1")
+ _test(lambda x: 2 - x, "{2;2;2;2} - im1")
+ _test(lambda x: 2 / x, "{2;2;2;2} div im1")
+ _test(lambda x: 2 * x, "2 * im1")
+ _test(lambda x: 2.0 + x, "{2.0;2.0;2.0;2.0} + im1")
+ _test(lambda x: 2.0 - x, "{2.0;2.0;2.0;2.0} - im1")
+ _test(lambda x: 2.0 / x, "{2.0;2.0;2.0;2.0} div im1")
+ _test(lambda x: 2.0 * x, "2.0 * im1")
+
+
def test_operation():
op = INPUT / 255 * 128
assert op.exp == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)"
@@ -164,9 +198,13 @@ def test_ndvi_comparison():
compared = pyotb.CompareImages({"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"})
assert (compared["count"], compared["mse"]) == (0, 0)
thresholded_indices = pyotb.where(ndvi_indices >= 0.3, 1, 0)
- assert thresholded_indices.exp == "((im1b1 >= 0.3) ? 1 : 0)"
+ assert thresholded_indices["exp"] == "((im1b1 >= 0.3) ? 1 : 0)"
thresholded_bandmath = pyotb.where(ndvi_bandmath >= 0.3, 1, 0)
- assert thresholded_bandmath.exp == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)"
+ assert thresholded_bandmath["exp"] == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)"
+
+
+def test_output_summary():
+ assert INPUT["out"].summarize()
def test_pipeline_simple():
--
GitLab
From 47fb1f471b4f948bc6d16008575da7f7826cbfdd Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 8 Feb 2023 18:48:47 +0000
Subject: [PATCH 153/399] Add more OTBObject abstract properties
---
pyotb/core.py | 134 +++++++++++++++++++++++++++++++--------------
tests/test_core.py | 5 ++
2 files changed, 98 insertions(+), 41 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index b64c877..4676c59 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -16,16 +16,30 @@ from .helpers import logger
class OTBObject(ABC):
"""Abstraction of an image object."""
+ @property
+ @abstractmethod
+ def name(self) -> str:
+ """By default, should return the application name, but a custom name may be passed during init."""
- name: str
- app: otb.Application
- parameters: dict
- exports_dic: dict
+ @property
+ @abstractmethod
+ def app(self) -> otb.Application:
+ """Reference to the main (or last in pipeline) otb.Application instance linked to this object."""
@property
@abstractmethod
- def output_image_key(self):
- """Returns the name of a parameter associated to an image. Property defined in App and Output."""
+ def parameters(self) -> dict[str, Any]:
+ """Should return every input parameters of the main otb.Application instance linked to this object."""
+
+ @property
+ @abstractmethod
+ def output_image_key(self) -> str:
+ """Return the name of a parameter key associated to the main output image of the object."""
+
+ @property
+ @abstractmethod
+ def exports_dic(self) -> dict[str, dict]:
+ """Return an internal dict object containing np.array exports, to avoid duplicated ExportImage() calls."""
@abstractmethod
def write(self):
@@ -118,7 +132,7 @@ class OTBObject(ABC):
elif isinstance(bands, slice):
channels = self.channels_list_from_slice(bands)
elif not isinstance(bands, list):
- raise TypeError(f"{self.app.GetName()}: type '{type(bands)}' cannot be interpreted as a valid slicing")
+ raise TypeError(f"{self.name}: type '{type(bands)}' cannot be interpreted as a valid slicing")
if channels:
app.app.Execute()
app.set_parameters({"cl": [f"Channel{n + 1}" for n in channels]})
@@ -141,7 +155,7 @@ class OTBObject(ABC):
return list(range(0, stop, step))
if start is None and stop is None:
return list(range(0, nb_channels, step))
- raise ValueError(f"{self.app.GetName()}: '{bands}' cannot be interpreted as valid slicing.")
+ raise ValueError(f"{self.name}: '{bands}' cannot be interpreted as valid slicing.")
def export(self, key: str = None, preserve_dtype: bool = True) -> dict[str, dict[str, np.ndarray]]:
"""Export a specific output image as numpy array and store it in object exports_dic.
@@ -400,11 +414,11 @@ class OTBObject(ABC):
class App(OTBObject):
"""Base class that gathers common operations for any OTB application."""
- def __init__(self, name: str, *args, frozen: bool = False, quiet: bool = False, **kwargs):
+ def __init__(self, appname: str, *args, frozen: bool = False, quiet: bool = False, name: str = "", **kwargs):
"""Common constructor for OTB applications. Handles in-memory connection between apps.
Args:
- name: name of the app, e.g. 'BandMath'
+ appname: name of the OTB application to initialize, e.g. 'BandMath'
*args: used for passing application parameters. Can be :
- dictionary containing key-arguments enumeration. Useful when a key is python-reserved
(e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
@@ -412,22 +426,26 @@ class App(OTBObject):
- list, useful when the user wants to specify the input list 'il'
frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___
quiet: whether to print logs of the OTB app
+ name: custom name that will show up in logs, appname will be used if not provided
**kwargs: used for passing application parameters.
e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif'
"""
- self.name = name
+ # Attributes and data structures used by properties
+ create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
+ self._app = create(appname)
+ self._name = name or appname
+ self._time_start, self._time_end = 0.0, 0.0
+ self._parameters, self._exports_dic = {}, {}
+ self.data, self.outputs = {}, {}
self.quiet, self.frozen = quiet, frozen
- self.data, self.parameters = {}, {} # params from self.app.GetParameterValue()
- self.outputs, self.exports_dic = {}, {} # Outputs objects and numpy arrays exports
- self.app = otb.Registry.CreateApplicationWithoutLogger(name) if quiet else otb.Registry.CreateApplication(name)
+ # Param keys and types
self.parameters_keys = tuple(self.app.GetParametersKeys())
self._all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys}
types = (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename)
self._out_param_types = {k: v for k, v in self._all_param_types.items() if v in types}
# Init, execute and write (auto flush only when output param was provided)
- self._time_start, self._time_end = 0., 0.
if args or kwargs:
self.set_parameters(*args, **kwargs)
if not self.frozen:
@@ -435,13 +453,30 @@ class App(OTBObject):
if any(key in self.parameters for key in self._out_param_types):
self.flush()
- def get_first_key(self, *type_lists: tuple[list[int]]) -> str:
- """Get the first param key for specific file types, try each list in args."""
- for param_types in type_lists:
- for key, value in sorted(self._all_param_types.items()):
- if value in param_types:
- return key
- raise TypeError(f"{self.name}: could not find any parameter key matching the provided types")
+ @property
+ def name(self) -> str:
+ """Returns appname by default, or a custom name if passed during App init."""
+ return self._name
+
+ @property
+ def app(self) -> otb.Application:
+ """Property to return an internal _app instance."""
+ return self._app
+
+ @property
+ def parameters(self) -> dict[str, Any]:
+ """Property to return an internal _parameters dict instance."""
+ return self._parameters
+
+ @property
+ def output_image_key(self) -> str:
+ """Get the name of first output image parameter."""
+ return self.get_first_key([otb.ParameterType_OutputImage])
+
+ @property
+ def exports_dic(self) -> dict[str, dict]:
+ """Returns internal _exports_dic object that contains numpy array exports."""
+ return self._exports_dic
@property
def input_key(self) -> str:
@@ -464,16 +499,19 @@ class App(OTBObject):
[otb.ParameterType_OutputImage], [otb.ParameterType_OutputVectorData], [otb.ParameterType_OutputFilename]
)
- @property
- def output_image_key(self) -> str:
- """Get the name of first output image parameter."""
- return self.get_first_key([otb.ParameterType_OutputImage])
-
@property
def elapsed_time(self) -> float:
"""Get elapsed time between app init and end of exec or file writing."""
return self._time_end - self._time_start
+ def get_first_key(self, *type_lists: tuple[list[int]]) -> str:
+ """Get the first param key for specific file types, try each list in args."""
+ for param_types in type_lists:
+ for key, value in sorted(self._all_param_types.items()):
+ if value in param_types:
+ return key
+ raise TypeError(f"{self.name}: could not find any parameter key matching the provided types")
+
def set_parameters(self, *args, **kwargs):
"""Set some parameters of the app.
@@ -512,7 +550,7 @@ class App(OTBObject):
f"(while setting parameter '{key}' to '{obj}': {e})"
) from e
# Update param dict and save values as object attributes
- self.parameters.update(parameters)
+ self._parameters.update(parameters)
self.save_objects(list(parameters))
def propagate_dtype(self, target_key: str = None, dtype: int = None):
@@ -561,7 +599,7 @@ class App(OTBObject):
continue
if isinstance(value, OTBObject) or bool(value) or value == 0:
if self.app.GetParameterRole(key) == 0:
- self.parameters[key] = value
+ self._parameters[key] = value
else:
if isinstance(value, str):
try:
@@ -743,7 +781,7 @@ class App(OTBObject):
class Slicer(App):
"""Slicer objects i.e. when we call something like raster[:, :, 2] from Python."""
- def __init__(self, obj: App | str, rows: slice, cols: slice, channels: slice | list[int] | int):
+ def __init__(self, obj: App, rows: slice, cols: slice, channels: slice | list[int] | int):
"""Create a slicer object, that can be used directly for writing or inside a BandMath.
It contains :
@@ -757,8 +795,7 @@ class Slicer(App):
channels: channels, can be slicing, list or int
"""
- super().__init__("ExtractROI", obj, mode="extent", quiet=True, frozen=True)
- self.name = "Slicer"
+ super().__init__("ExtractROI", obj, mode="extent", quiet=True, frozen=True, name=f"Slicer from {obj.name}")
self.rows, self.cols = rows, cols
parameters = {}
@@ -864,9 +901,8 @@ class Operation(App):
self.unique_inputs = [map_repr_to_input[id_str] for id_str in sorted(self.im_dic, key=self.im_dic.get)]
self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands)
appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX"
- # Execute app
- super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True)
- self.name = name or f'Operation exp="{self.exp}"'
+ name = f'Operation exp="{self.exp}"'
+ super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True, name=name)
def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None):
"""Create a list of 'fake' expressions, one for each band.
@@ -1079,7 +1115,7 @@ class Input(App):
"""
super().__init__("ExtractROI", {"in": filepath}, frozen=True)
- self.name = f"Input from {filepath}"
+ self._name = f"Input from {filepath}"
self.filepath = Path(filepath) if not filepath.startswith("/vsi") else filepath
self.propagate_dtype()
self.execute()
@@ -1102,18 +1138,34 @@ class Output(OTBObject):
mkdir: create missing parent directories
"""
- self.name = f"Output {param_key} from {pyotb_app.name}"
self.parent_pyotb_app = pyotb_app # keep trace of parent app
- self.app = pyotb_app.app
- self.exports_dic = pyotb_app.exports_dic
self.param_key = param_key
- self.parameters = self.parent_pyotb_app.parameters
self.filepath = filepath
if filepath and not filepath.startswith("/vsi"):
self.filepath = Path(filepath.split("?")[0])
if mkdir:
self.make_parent_dirs()
+ @property
+ def name(self):
+ """Return Output name containing filepath."""
+ return f"Output {self.param_key} from {self.parent_pyotb_app.name}"
+
+ @property
+ def app(self) -> otb.Application:
+ """Reference to the parent pyotb otb.Application instance."""
+ return self.parent_pyotb_app.app
+
+ @property
+ def parameters(self) -> dict[str, Any]:
+ """Reference to the parent pyotb App parameters."""
+ return self.parent_pyotb_app.parameters
+
+ @property
+ def exports_dic(self) -> dict[str, dict]:
+ """Returns internal _exports_dic object that contains numpy array exports."""
+ return self.parent_pyotb_app.exports_dic
+
@property
def output_image_key(self) -> str:
"""Force the right key to be used when accessing the OTBObject."""
@@ -1131,7 +1183,7 @@ class Output(OTBObject):
raise ValueError("Filepath is not set or points to a remote URL")
self.filepath.parent.mkdir(parents=True, exist_ok=True)
- def write(self, filepath: None | str | Path = None, **kwargs):
+ def write(self, filepath: None | str | Path = None, **kwargs) -> bool:
"""Write output to disk, filepath is not required if it was provided to parent App during init."""
if filepath is None and self.filepath:
return self.parent_pyotb_app.write({self.output_image_key: self.filepath}, **kwargs)
diff --git a/tests/test_core.py b/tests/test_core.py
index 4e98d50..143bf93 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -15,6 +15,11 @@ def test_wrong_key():
# OTBObject properties
+def test_name():
+ app = pyotb.App("BandMath", [INPUT], exp="im1b1", name="TestName")
+ assert app.name == "TestName"
+
+
def test_key_input():
assert INPUT.input_key == INPUT.input_image_key == "in"
--
GitLab
From 724f7e897723bdb477c8e7936e08cdd2ca335438 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 14 Feb 2023 11:39:40 +0000
Subject: [PATCH 154/399] Remove unused params from App.parameters and
summarize + create App.outputs objects only once
---
pyotb/__init__.py | 3 +-
pyotb/core.py | 164 +++++++++++++++++++------------------
tests/serialized_apps.json | 123 +++++++++++-----------------
tests/test_core.py | 8 +-
4 files changed, 141 insertions(+), 157 deletions(-)
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index 8e22979..d4efd39 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -9,7 +9,8 @@ from .core import (
Input,
Output,
get_nbchannels,
- get_pixel_type
+ get_pixel_type,
+ summarize
)
from .functions import ( # pylint: disable=redefined-builtin
all,
diff --git a/pyotb/core.py b/pyotb/core.py
index 4676c59..cd144f6 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -26,11 +26,6 @@ class OTBObject(ABC):
def app(self) -> otb.Application:
"""Reference to the main (or last in pipeline) otb.Application instance linked to this object."""
- @property
- @abstractmethod
- def parameters(self) -> dict[str, Any]:
- """Should return every input parameters of the main otb.Application instance linked to this object."""
-
@property
@abstractmethod
def output_image_key(self) -> str:
@@ -86,22 +81,6 @@ class OTBObject(ABC):
origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2
return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y
- def summarize(self) -> dict[str, str | dict[str, Any]]:
- """Serialize an object and its pipeline into a dictionary.
-
- Returns:
- nested dictionary summarizing the pipeline
-
- """
- parameters = self.parameters.copy()
- for key, param in parameters.items():
- # In the following, we replace each parameter which is an OTBObject, with its summary.
- if isinstance(param, OTBObject): # single parameter
- parameters[key] = param.summarize()
- elif isinstance(param, list): # parameter list
- parameters[key] = [p.summarize() if isinstance(p, OTBObject) else p for p in param]
- return {"name": self.app.GetName(), "parameters": parameters}
-
def get_info(self) -> dict[str, (str, float, list[float])]:
"""Return a dict output of ReadImageInfo for the first image output."""
return App("ReadImageInfo", self, quiet=True).data
@@ -436,8 +415,9 @@ class App(OTBObject):
create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
self._app = create(appname)
self._name = name or appname
+ self._exports_dic = {}
+ self._settings, self._auto_parameters = {}, {}
self._time_start, self._time_end = 0.0, 0.0
- self._parameters, self._exports_dic = {}, {}
self.data, self.outputs = {}, {}
self.quiet, self.frozen = quiet, frozen
# Param keys and types
@@ -445,12 +425,14 @@ class App(OTBObject):
self._all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys}
types = (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename)
self._out_param_types = {k: v for k, v in self._all_param_types.items() if v in types}
+ for key in self._out_param_types:
+ self.outputs[key] = Output(self, key, self._settings.get(key))
# Init, execute and write (auto flush only when output param was provided)
if args or kwargs:
self.set_parameters(*args, **kwargs)
if not self.frozen:
self.execute()
- if any(key in self.parameters for key in self._out_param_types):
+ if any(key in self._settings for key in self._out_param_types):
self.flush()
@property
@@ -464,20 +446,23 @@ class App(OTBObject):
return self._app
@property
- def parameters(self) -> dict[str, Any]:
- """Property to return an internal _parameters dict instance."""
- return self._parameters
-
- @property
- def output_image_key(self) -> str:
- """Get the name of first output image parameter."""
- return self.get_first_key([otb.ParameterType_OutputImage])
+ def parameters(self):
+ """Return used OTB applications parameters."""
+ return {**self.app.GetParameters(), **self._auto_parameters, **self._settings}
@property
def exports_dic(self) -> dict[str, dict]:
"""Returns internal _exports_dic object that contains numpy array exports."""
return self._exports_dic
+ def get_first_key(self, *type_lists: tuple[list[int]]) -> str:
+ """Get the first param key for specific file types, try each list in args."""
+ for param_types in type_lists:
+ for key, value in sorted(self._all_param_types.items()):
+ if value in param_types:
+ return key
+ raise TypeError(f"{self.name}: could not find any parameter key matching the provided types")
+
@property
def input_key(self) -> str:
"""Get the name of first input parameter, raster > vector > file."""
@@ -499,19 +484,16 @@ class App(OTBObject):
[otb.ParameterType_OutputImage], [otb.ParameterType_OutputVectorData], [otb.ParameterType_OutputFilename]
)
+ @property
+ def output_image_key(self) -> str:
+ """Get the name of first output image parameter."""
+ return self.get_first_key([otb.ParameterType_OutputImage])
+
@property
def elapsed_time(self) -> float:
"""Get elapsed time between app init and end of exec or file writing."""
return self._time_end - self._time_start
- def get_first_key(self, *type_lists: tuple[list[int]]) -> str:
- """Get the first param key for specific file types, try each list in args."""
- for param_types in type_lists:
- for key, value in sorted(self._all_param_types.items()):
- if value in param_types:
- return key
- raise TypeError(f"{self.name}: could not find any parameter key matching the provided types")
-
def set_parameters(self, *args, **kwargs):
"""Set some parameters of the app.
@@ -546,12 +528,12 @@ class App(OTBObject):
self.__set_param(key, obj)
except (RuntimeError, TypeError, ValueError, KeyError) as e:
raise RuntimeError(
- f"{self.name}: something went wrong before execution "
- f"(while setting parameter '{key}' to '{obj}': {e})"
+ f"{self.name}: error before execution, while setting parameter '{key}' to '{obj}': {e})"
) from e
- # Update param dict and save values as object attributes
- self._parameters.update(parameters)
- self.save_objects(list(parameters))
+ # Save / update setting value and update the Output object initialized in __init__ without a filepath
+ self._settings[key] = obj
+ if key in self._out_param_types:
+ self.outputs[key].filepath = obj
def propagate_dtype(self, target_key: str = None, dtype: int = None):
"""Propagate a pixel type from main input to every outputs, or to a target output key only.
@@ -565,7 +547,7 @@ class App(OTBObject):
"""
if not dtype:
- param = self.parameters.get(self.input_image_key)
+ param = self._settings.get(self.input_image_key)
if not param:
logger.warning("%s: could not propagate pixel type from inputs to output", self.name)
return
@@ -583,30 +565,28 @@ class App(OTBObject):
for key in keys:
self.app.SetParameterOutputImagePixelType(key, dtype)
- def save_objects(self, keys: list[str] = None):
+ def save_objects(self):
"""Save OTB app values in data, parameters and outputs dict, for a list of keys or all parameters."""
- keys = keys or self.parameters_keys
- for key in keys:
- value = self.parameters.get(key)
- if value is None:
+ for key in self.parameters_keys:
+ if not self.app.HasValue(key):
+ continue
+ value = self.app.GetParameterValue(key)
+ # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken
+ if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(key):
try:
- value = self.app.GetParameterValue(key) # any other app attribute (e.g. ReadImageInfo results)
+ value = str(value) # some default str values like "mode" or "interpolator"
+ self._auto_parameters[key] = value
+ continue
except RuntimeError:
- pass # undefined parameter
- if self._out_param_types.get(key) == otb.ParameterType_OutputImage:
- self.outputs[key] = Output(self, key, value)
- if value is None or isinstance(value, otb.ApplicationProxy):
- continue
- if isinstance(value, OTBObject) or bool(value) or value == 0:
- if self.app.GetParameterRole(key) == 0:
- self._parameters[key] = value
- else:
- if isinstance(value, str):
- try:
- value = literal_eval(value)
- except (ValueError, SyntaxError):
- pass
- self.data[key] = value
+ continue # grouped parameters
+ # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.)
+ elif self.app.GetParameterRole(key) == 1 and bool(value) or value == 0:
+ if isinstance(value, str):
+ try:
+ value = literal_eval(value)
+ except (ValueError, SyntaxError):
+ pass
+ self.data[key] = value
def execute(self):
"""Execute and write to disk if any output parameter has been set during init."""
@@ -781,7 +761,7 @@ class App(OTBObject):
class Slicer(App):
"""Slicer objects i.e. when we call something like raster[:, :, 2] from Python."""
- def __init__(self, obj: App, rows: slice, cols: slice, channels: slice | list[int] | int):
+ def __init__(self, obj: OTBObject, rows: slice, cols: slice, channels: slice | list[int] | int):
"""Create a slicer object, that can be used directly for writing or inside a BandMath.
It contains :
@@ -1127,8 +1107,9 @@ class Input(App):
class Output(OTBObject):
"""Object that behave like a pointer to a specific application output file."""
+ _filepath: str | Path = None
- def __init__(self, pyotb_app: App, param_key: str = None, filepath: str = "", mkdir: bool = True):
+ def __init__(self, pyotb_app: App, param_key: str = None, filepath: str = None, mkdir: bool = True):
"""Constructor for an Output object.
Args:
@@ -1141,13 +1122,11 @@ class Output(OTBObject):
self.parent_pyotb_app = pyotb_app # keep trace of parent app
self.param_key = param_key
self.filepath = filepath
- if filepath and not filepath.startswith("/vsi"):
- self.filepath = Path(filepath.split("?")[0])
- if mkdir:
- self.make_parent_dirs()
+ if mkdir and filepath is not None:
+ self.make_parent_dirs()
@property
- def name(self):
+ def name(self) -> str:
"""Return Output name containing filepath."""
return f"Output {self.param_key} from {self.parent_pyotb_app.name}"
@@ -1156,11 +1135,6 @@ class Output(OTBObject):
"""Reference to the parent pyotb otb.Application instance."""
return self.parent_pyotb_app.app
- @property
- def parameters(self) -> dict[str, Any]:
- """Reference to the parent pyotb App parameters."""
- return self.parent_pyotb_app.parameters
-
@property
def exports_dic(self) -> dict[str, dict]:
"""Returns internal _exports_dic object that contains numpy array exports."""
@@ -1171,6 +1145,20 @@ class Output(OTBObject):
"""Force the right key to be used when accessing the OTBObject."""
return self.param_key
+ @property
+ def filepath(self) -> str | Path:
+ """Property to manage output URL."""
+ if self._filepath is None:
+ raise ValueError("Filepath is not set")
+ return self._filepath
+
+ @filepath.setter
+ def filepath(self, path: str):
+ if isinstance(path, str):
+ if path and not path.startswith("/vsi"):
+ path = Path(path.split("?")[0])
+ self._filepath = path
+
def exists(self) -> bool:
"""Check file exist."""
if not isinstance(self.filepath, Path):
@@ -1185,7 +1173,7 @@ class Output(OTBObject):
def write(self, filepath: None | str | Path = None, **kwargs) -> bool:
"""Write output to disk, filepath is not required if it was provided to parent App during init."""
- if filepath is None and self.filepath:
+ if filepath is None:
return self.parent_pyotb_app.write({self.output_image_key: self.filepath}, **kwargs)
return self.parent_pyotb_app.write({self.output_image_key: filepath}, **kwargs)
@@ -1293,3 +1281,21 @@ def is_key_images_list(pyotb_app: OTBObject, key: str) -> bool:
def get_out_images_param_keys(app: OTBObject) -> list[str]:
"""Return every output parameter keys of an OTB app."""
return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage]
+
+
+def summarize(obj: App | Output | Any) -> dict[str, str | dict[str, Any]]:
+ """Recursively summarize application parameters, and every App or Output found upstream in the pipeline.
+
+ Returns:
+ nested dictionary with serialized App(s) containing name and parameters of an app and its parents
+
+ """
+ if isinstance(obj, Output):
+ return summarize(obj.parent_pyotb_app)
+ if not isinstance(obj, App):
+ return obj
+ parameters = {
+ key: [summarize(p) for p in param] if isinstance(param, list) else summarize(param)
+ for key, param in obj.parameters.items()
+ }
+ return {"name": obj.app.GetName(), "parameters": parameters}
diff --git a/tests/serialized_apps.json b/tests/serialized_apps.json
index 088e853..c413645 100644
--- a/tests/serialized_apps.json
+++ b/tests/serialized_apps.json
@@ -2,29 +2,13 @@
"SIMPLE": {
"name": "ManageNoData",
"parameters": {
+ "mode": "buildmask",
"in": {
"name": "OrthoRectification",
"parameters": {
- "io.in": {
- "name": "BandMath",
- "parameters": {
- "il": [
- "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
- ],
- "exp": "im1b1",
- "ram": 256
- }
- },
+ "map": "utm",
"map.utm.zone": 31,
"map.utm.northhem": true,
- "map.epsg.code": 4326,
- "outputs.isotropic": true,
- "outputs.default": 0.0,
- "elev.default": 0.0,
- "interpolator.bco.radius": 2,
- "opt.rpc": 10,
- "opt.ram": 256,
- "opt.gridspacing": 4.0,
"outputs.ulx": 560000.8125,
"outputs.uly": 5495732.5,
"outputs.sizex": 251,
@@ -32,15 +16,22 @@
"outputs.spacingx": 5.997312068939209,
"outputs.spacingy": -5.997312068939209,
"outputs.lrx": 561506.125,
- "outputs.lry": 5493909.5
+ "outputs.lry": 5493909.5,
+ "outputs.isotropic": true,
+ "opt.gridspacing": 4.0,
+ "outputs.mode": "auto",
+ "interpolator": "bco",
+ "io.in": {
+ "name": "BandMath",
+ "parameters": {
+ "il": [
+ "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
+ ],
+ "exp": "im1b1"
+ }
+ }
}
- },
- "usenan": false,
- "mode.buildmask.inv": 1.0,
- "mode.buildmask.outv": 0.0,
- "mode.changevalue.newv": 0.0,
- "mode.apply.ndval": 0.0,
- "ram": 256
+ }
}
},
"COMPLEX": {
@@ -50,26 +41,9 @@
{
"name": "OrthoRectification",
"parameters": {
- "io.in": {
- "name": "BandMath",
- "parameters": {
- "il": [
- "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
- ],
- "exp": "im1b1",
- "ram": 256
- }
- },
+ "map": "utm",
"map.utm.zone": 31,
"map.utm.northhem": true,
- "map.epsg.code": 4326,
- "outputs.isotropic": true,
- "outputs.default": 0.0,
- "elev.default": 0.0,
- "interpolator.bco.radius": 2,
- "opt.rpc": 10,
- "opt.ram": 256,
- "opt.gridspacing": 4.0,
"outputs.ulx": 560000.8125,
"outputs.uly": 5495732.5,
"outputs.sizex": 251,
@@ -77,35 +51,32 @@
"outputs.spacingx": 5.997312068939209,
"outputs.spacingy": -5.997312068939209,
"outputs.lrx": 561506.125,
- "outputs.lry": 5493909.5
+ "outputs.lry": 5493909.5,
+ "outputs.isotropic": true,
+ "opt.gridspacing": 4.0,
+ "outputs.mode": "auto",
+ "interpolator": "bco",
+ "io.in": {
+ "name": "BandMath",
+ "parameters": {
+ "il": [
+ "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
+ ],
+ "exp": "im1b1"
+ }
+ }
}
},
{
"name": "ManageNoData",
"parameters": {
+ "mode": "buildmask",
"in": {
"name": "OrthoRectification",
"parameters": {
- "io.in": {
- "name": "BandMath",
- "parameters": {
- "il": [
- "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
- ],
- "exp": "im1b1",
- "ram": 256
- }
- },
+ "map": "utm",
"map.utm.zone": 31,
"map.utm.northhem": true,
- "map.epsg.code": 4326,
- "outputs.isotropic": true,
- "outputs.default": 0.0,
- "elev.default": 0.0,
- "interpolator.bco.radius": 2,
- "opt.rpc": 10,
- "opt.ram": 256,
- "opt.gridspacing": 4.0,
"outputs.ulx": 560000.8125,
"outputs.uly": 5495732.5,
"outputs.sizex": 251,
@@ -113,20 +84,26 @@
"outputs.spacingx": 5.997312068939209,
"outputs.spacingy": -5.997312068939209,
"outputs.lrx": 561506.125,
- "outputs.lry": 5493909.5
+ "outputs.lry": 5493909.5,
+ "outputs.isotropic": true,
+ "opt.gridspacing": 4.0,
+ "outputs.mode": "auto",
+ "interpolator": "bco",
+ "io.in": {
+ "name": "BandMath",
+ "parameters": {
+ "il": [
+ "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
+ ],
+ "exp": "im1b1"
+ }
+ }
}
- },
- "usenan": false,
- "mode.buildmask.inv": 1.0,
- "mode.buildmask.outv": 0.0,
- "mode.changevalue.newv": 0.0,
- "mode.apply.ndval": 0.0,
- "ram": 256
+ }
}
}
],
- "exp": "im1+im2",
- "ram": 256
+ "exp": "im1+im2"
}
}
}
\ No newline at end of file
diff --git a/tests/test_core.py b/tests/test_core.py
index 143bf93..8d9acf8 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -208,8 +208,8 @@ def test_ndvi_comparison():
assert thresholded_bandmath["exp"] == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)"
-def test_output_summary():
- assert INPUT["out"].summarize()
+def test_summarize_output():
+ assert pyotb.summarize(INPUT["out"])
def test_pipeline_simple():
@@ -217,7 +217,7 @@ def test_pipeline_simple():
app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"})
app2 = pyotb.OrthoRectification({"io.in": app1})
app3 = pyotb.ManageNoData({"in": app2})
- summary = app3.summarize()
+ summary = pyotb.summarize(app3)
assert summary == SIMPLE_SERIALIZATION
@@ -227,5 +227,5 @@ def test_pipeline_diamond():
app2 = pyotb.OrthoRectification({"io.in": app1})
app3 = pyotb.ManageNoData({"in": app2})
app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"})
- summary = app4.summarize()
+ summary = pyotb.summarize(app4)
assert summary == COMPLEX_SERIALIZATION
--
GitLab
From 03533380701fda8ccd41003303cd80e757662caa Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 14 Feb 2023 21:17:19 +0100
Subject: [PATCH 155/399] FIX: #89
---
pyotb/core.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index cd144f6..6a0a3e4 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -425,11 +425,12 @@ class App(OTBObject):
self._all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys}
types = (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename)
self._out_param_types = {k: v for k, v in self._all_param_types.items() if v in types}
- for key in self._out_param_types:
- self.outputs[key] = Output(self, key, self._settings.get(key))
# Init, execute and write (auto flush only when output param was provided)
if args or kwargs:
self.set_parameters(*args, **kwargs)
+ # Create Output image objects
+ for key in filter(lambda k: self._out_param_types[k] == types[0], self._out_param_types.keys()):
+ self.outputs[key] = Output(self, key, self._settings.get(key))
if not self.frozen:
self.execute()
if any(key in self._settings for key in self._out_param_types):
--
GitLab
From cb40648ba425756a1fb3929e9534f5c6f0b2a5db Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 14 Feb 2023 21:24:29 +0100
Subject: [PATCH 156/399] ENH: remove OTBObject.write
---
pyotb/core.py | 4 ----
1 file changed, 4 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 6a0a3e4..9992f81 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -36,10 +36,6 @@ class OTBObject(ABC):
def exports_dic(self) -> dict[str, dict]:
"""Return an internal dict object containing np.array exports, to avoid duplicated ExportImage() calls."""
- @abstractmethod
- def write(self):
- """Write image, this is defined in App. Output will use App.write for a specific key."""
-
@property
def metadata(self) -> dict[str, (str, float, list[float])]:
"""Return first output image metadata dictionary."""
--
GitLab
From a87f6f0cb955b17e36304066086cfbd858c8a450 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 14 Feb 2023 22:42:46 +0100
Subject: [PATCH 157/399] ENH: remove app.write(*args) argument - write only
accept 1 arg, or **kwargs
---
pyotb/core.py | 28 ++++++++++++++--------------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 9992f81..3cdf469 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -609,15 +609,15 @@ class App(OTBObject):
self.app.ExecuteAndWriteOutput()
self._time_end = perf_counter()
- def write(self, *args, ext_fname: str = "", pixel_type: dict[str, str] | str = None,
- preserve_dtype: bool = False, **kwargs, ) -> bool:
+ def write(self, path: str | Path | dict[str, str] = None, ext_fname: str = "",
+ pixel_type: dict[str, str] | str = None, preserve_dtype: bool = False, **kwargs, ) -> bool:
"""Set output pixel type and write the output raster files.
Args:
- *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key contains
- non-standard characters such as a point, e.g. {'io.out':'output.tif'}
- - filepath, useful when there is only one output, e.g. 'output.tif'
- - None if output file was passed during App init
+ path: Can be : - filepath, useful when there is only one output, e.g. 'output.tif'
+ - dictionary containing key-arguments enumeration. Useful when a key contains
+ non-standard characters such as a point, e.g. {'io.out':'output.tif'}
+ - None if output file was passed during App init
ext_fname: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES")
Will be used for all outputs (Default value = "")
pixel_type: Can be : - dictionary {out_param_key: pixeltype} when specifying for several outputs
@@ -633,14 +633,14 @@ class App(OTBObject):
"""
# Gather all input arguments in kwargs dict
- for arg in args:
- if isinstance(arg, dict):
- kwargs.update(arg)
- elif isinstance(arg, str) and kwargs:
- logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, arg)
- elif isinstance(arg, (str, Path)) and self.output_key:
- kwargs.update({self.output_key: str(arg)})
- if not kwargs:
+ if path is not None:
+ if isinstance(path, dict):
+ kwargs.update(path)
+ elif isinstance(path, str) and kwargs:
+ logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, path)
+ elif isinstance(path, (str, Path)) and self.output_key:
+ kwargs.update({self.output_key: str(path)})
+ if not (kwargs or self.output_key in self._settings):
raise KeyError(f"{self.name}: at least one filepath is required, if not passed to App during init")
parameters = kwargs.copy()
--
GitLab
From 0e865c9049c524079675cd90b8bdf6704f9ce04d Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 14 Feb 2023 22:58:15 +0100
Subject: [PATCH 158/399] FIX: avoid possible bugs with Output and write
---
pyotb/core.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 3cdf469..df97fad 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -529,7 +529,7 @@ class App(OTBObject):
) from e
# Save / update setting value and update the Output object initialized in __init__ without a filepath
self._settings[key] = obj
- if key in self._out_param_types:
+ if key in self.outputs:
self.outputs[key].filepath = obj
def propagate_dtype(self, target_key: str = None, dtype: int = None):
@@ -640,7 +640,7 @@ class App(OTBObject):
logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, path)
elif isinstance(path, (str, Path)) and self.output_key:
kwargs.update({self.output_key: str(path)})
- if not (kwargs or self.output_key in self._settings):
+ if not (kwargs or any(k in self._settings for k in self._out_param_types)):
raise KeyError(f"{self.name}: at least one filepath is required, if not passed to App during init")
parameters = kwargs.copy()
@@ -676,7 +676,11 @@ class App(OTBObject):
if key in data_types:
self.propagate_dtype(key, data_types[key])
self.set_parameters({key: filepath})
+ if self.frozen:
+ self.execute()
self.flush()
+ if not parameters:
+ return True
# Search and log missing files
files, missing = [], []
for key, filepath in parameters.items():
--
GitLab
From b99fbe8a660b93bb592d7ab747e573a122b1fe7e Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 14 Feb 2023 23:04:01 +0100
Subject: [PATCH 159/399] ENH: raise TypeError when bad filepath is passed to
write
---
pyotb/core.py | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index df97fad..1c3d86f 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -633,15 +633,16 @@ class App(OTBObject):
"""
# Gather all input arguments in kwargs dict
- if path is not None:
- if isinstance(path, dict):
- kwargs.update(path)
- elif isinstance(path, str) and kwargs:
- logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, path)
- elif isinstance(path, (str, Path)) and self.output_key:
- kwargs.update({self.output_key: str(path)})
+ if isinstance(path, dict):
+ kwargs.update(path)
+ elif isinstance(path, str) and kwargs:
+ logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, path)
+ elif isinstance(path, (str, Path)) and self.output_key:
+ kwargs.update({self.output_key: str(path)})
+ elif path is not None:
+ raise TypeError(f"{self.name}: unsupported filepath type ({type(path)})")
if not (kwargs or any(k in self._settings for k in self._out_param_types)):
- raise KeyError(f"{self.name}: at least one filepath is required, if not passed to App during init")
+ raise KeyError(f"{self.name}: at least one filepath is required, if not provided during App init")
parameters = kwargs.copy()
# Append filename extension to filenames
--
GitLab
From 111c14143d5ae1d61943707f5fc31d582703a6b0 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 15 Feb 2023 10:17:57 +0100
Subject: [PATCH 160/399] STYLE: rename save_objects to private and more
accurate __sync_parameters
---
pyotb/core.py | 54 +++++++++++++++++++++++++--------------------------
1 file changed, 27 insertions(+), 27 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 1c3d86f..e02a5f7 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -444,8 +444,8 @@ class App(OTBObject):
@property
def parameters(self):
- """Return used OTB applications parameters."""
- return {**self.app.GetParameters(), **self._auto_parameters, **self._settings}
+ """Return used OTB application parameters."""
+ return {**self._auto_parameters, **self.app.GetParameters(), **self._settings}
@property
def exports_dic(self) -> dict[str, dict]:
@@ -562,29 +562,6 @@ class App(OTBObject):
for key in keys:
self.app.SetParameterOutputImagePixelType(key, dtype)
- def save_objects(self):
- """Save OTB app values in data, parameters and outputs dict, for a list of keys or all parameters."""
- for key in self.parameters_keys:
- if not self.app.HasValue(key):
- continue
- value = self.app.GetParameterValue(key)
- # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken
- if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(key):
- try:
- value = str(value) # some default str values like "mode" or "interpolator"
- self._auto_parameters[key] = value
- continue
- except RuntimeError:
- continue # grouped parameters
- # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.)
- elif self.app.GetParameterRole(key) == 1 and bool(value) or value == 0:
- if isinstance(value, str):
- try:
- value = literal_eval(value)
- except (ValueError, SyntaxError):
- pass
- self.data[key] = value
-
def execute(self):
"""Execute and write to disk if any output parameter has been set during init."""
logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters)
@@ -596,7 +573,7 @@ class App(OTBObject):
self.frozen = False
self._time_end = perf_counter()
logger.debug("%s: execution ended", self.name)
- self.save_objects() # this is required for apps like ReadImageInfo or ComputeImagesStatistics
+ self.__sync_parameters() # this is required for apps like ReadImageInfo or ComputeImagesStatistics
def flush(self):
"""Flush data to disk, this is when WriteOutput is actually called."""
@@ -610,7 +587,7 @@ class App(OTBObject):
self._time_end = perf_counter()
def write(self, path: str | Path | dict[str, str] = None, ext_fname: str = "",
- pixel_type: dict[str, str] | str = None, preserve_dtype: bool = False, **kwargs, ) -> bool:
+ pixel_type: dict[str, str] | str = None, preserve_dtype: bool = False, **kwargs) -> bool:
"""Set output pixel type and write the output raster files.
Args:
@@ -741,6 +718,29 @@ class App(OTBObject):
else:
self.app.SetParameterValue(key, obj)
+ def __sync_parameters(self):
+ """Save OTB parameters in _settings, data and outputs dict, for a list of keys or all parameters."""
+ for key in self.parameters_keys:
+ if not self.app.HasValue(key):
+ continue
+ value = self.app.GetParameterValue(key)
+ # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken
+ if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(key):
+ try:
+ value = str(value) # some default str values like "mode" or "interpolator"
+ self._auto_parameters[key] = value
+ continue
+ except RuntimeError:
+ continue # grouped parameters
+ # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.)
+ elif self.app.GetParameterRole(key) == 1 and bool(value) or value == 0:
+ if isinstance(value, str):
+ try:
+ value = literal_eval(value)
+ except (ValueError, SyntaxError):
+ pass
+ self.data[key] = value
+
# Special functions
def __getitem__(self, key: str) -> Any | list[int | float] | int | float | Slicer:
"""This function is called when we use App()[...].
--
GitLab
From 789913a50f302bbfe8776bc279325ec12cf96b29 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 15 Feb 2023 21:07:57 +0100
Subject: [PATCH 161/399] REFAC: move function back to its original position,
for review purpose
---
pyotb/core.py | 46 +++++++++++++++++++++++-----------------------
1 file changed, 23 insertions(+), 23 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index e02a5f7..4d5f7b2 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -562,6 +562,29 @@ class App(OTBObject):
for key in keys:
self.app.SetParameterOutputImagePixelType(key, dtype)
+ def __sync_parameters(self):
+ """Save OTB parameters in _settings, data and outputs dict, for a list of keys or all parameters."""
+ for key in self.parameters_keys:
+ if not self.app.HasValue(key):
+ continue
+ value = self.app.GetParameterValue(key)
+ # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken
+ if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(key):
+ try:
+ value = str(value) # some default str values like "mode" or "interpolator"
+ self._auto_parameters[key] = value
+ continue
+ except RuntimeError:
+ continue # grouped parameters
+ # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.)
+ elif self.app.GetParameterRole(key) == 1 and bool(value) or value == 0:
+ if isinstance(value, str):
+ try:
+ value = literal_eval(value)
+ except (ValueError, SyntaxError):
+ pass
+ self.data[key] = value
+
def execute(self):
"""Execute and write to disk if any output parameter has been set during init."""
logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters)
@@ -718,29 +741,6 @@ class App(OTBObject):
else:
self.app.SetParameterValue(key, obj)
- def __sync_parameters(self):
- """Save OTB parameters in _settings, data and outputs dict, for a list of keys or all parameters."""
- for key in self.parameters_keys:
- if not self.app.HasValue(key):
- continue
- value = self.app.GetParameterValue(key)
- # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken
- if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(key):
- try:
- value = str(value) # some default str values like "mode" or "interpolator"
- self._auto_parameters[key] = value
- continue
- except RuntimeError:
- continue # grouped parameters
- # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.)
- elif self.app.GetParameterRole(key) == 1 and bool(value) or value == 0:
- if isinstance(value, str):
- try:
- value = literal_eval(value)
- except (ValueError, SyntaxError):
- pass
- self.data[key] = value
-
# Special functions
def __getitem__(self, key: str) -> Any | list[int | float] | int | float | Slicer:
"""This function is called when we use App()[...].
--
GitLab
From 183decee7e245ea0bfc3e3725821a7f9144eeee2 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 16 Mar 2023 14:19:57 +0000
Subject: [PATCH 162/399] ENH: better filter expression for parameter keys, use
enum
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 4d5f7b2..8821e55 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -425,7 +425,7 @@ class App(OTBObject):
if args or kwargs:
self.set_parameters(*args, **kwargs)
# Create Output image objects
- for key in filter(lambda k: self._out_param_types[k] == types[0], self._out_param_types.keys()):
+ for key in filter(lambda k: self._out_param_types[k] == otb.ParameterType_OutputImage, self._out_param_types):
self.outputs[key] = Output(self, key, self._settings.get(key))
if not self.frozen:
self.execute()
--
GitLab
From 3156a3a3f9c9197050328dbd18e3f08e5a2be5cb Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 16 Mar 2023 15:21:21 +0100
Subject: [PATCH 163/399] ENH: add 2 write() tests for frozen apps
---
tests/test_core.py | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/tests/test_core.py b/tests/test_core.py
index 8d9acf8..ebb26cf 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -78,11 +78,27 @@ def test_write():
INPUT["out"].filepath.unlink()
+def test_frozen_app_write():
+ app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True)
+ assert app.write("/tmp/test_frozen_app_write.tif")
+ app["out"].filepath.unlink()
+
+ app = pyotb.BandMath(INPUT, exp="im1b1", out="/tmp/test_frozen_app_write.tif", frozen=True)
+ assert app.write()
+ app["out"].filepath.unlink()
+
+
def test_output_write():
assert INPUT["out"].write("/tmp/test_output_write.tif")
INPUT["out"].filepath.unlink()
+def test_frozen_output_write():
+ app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True)
+ assert app["out"].write("/tmp/test_frozen_app_write.tif")
+ app["out"].filepath.unlink()
+
+
def test_output_in_arg():
info = pyotb.ReadImageInfo(INPUT["out"])
assert info.data
--
GitLab
From fd963531465af35d6a66afcbc6808187b2cfdf91 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 24 Mar 2023 11:32:35 +0100
Subject: [PATCH 164/399] Revert "REFAC: move function back to its original
position, for review purpose"
This reverts commit 789913a50f302bbfe8776bc279325ec12cf96b29.
---
pyotb/core.py | 46 +++++++++++++++++++++++-----------------------
1 file changed, 23 insertions(+), 23 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 8821e55..ad18769 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -562,29 +562,6 @@ class App(OTBObject):
for key in keys:
self.app.SetParameterOutputImagePixelType(key, dtype)
- def __sync_parameters(self):
- """Save OTB parameters in _settings, data and outputs dict, for a list of keys or all parameters."""
- for key in self.parameters_keys:
- if not self.app.HasValue(key):
- continue
- value = self.app.GetParameterValue(key)
- # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken
- if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(key):
- try:
- value = str(value) # some default str values like "mode" or "interpolator"
- self._auto_parameters[key] = value
- continue
- except RuntimeError:
- continue # grouped parameters
- # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.)
- elif self.app.GetParameterRole(key) == 1 and bool(value) or value == 0:
- if isinstance(value, str):
- try:
- value = literal_eval(value)
- except (ValueError, SyntaxError):
- pass
- self.data[key] = value
-
def execute(self):
"""Execute and write to disk if any output parameter has been set during init."""
logger.debug("%s: run execute() with parameters=%s", self.name, self.parameters)
@@ -741,6 +718,29 @@ class App(OTBObject):
else:
self.app.SetParameterValue(key, obj)
+ def __sync_parameters(self):
+ """Save OTB parameters in _settings, data and outputs dict, for a list of keys or all parameters."""
+ for key in self.parameters_keys:
+ if not self.app.HasValue(key):
+ continue
+ value = self.app.GetParameterValue(key)
+ # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken
+ if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(key):
+ try:
+ value = str(value) # some default str values like "mode" or "interpolator"
+ self._auto_parameters[key] = value
+ continue
+ except RuntimeError:
+ continue # grouped parameters
+ # Save static output data (ReadImageInfo, ComputeImageStatistics, etc.)
+ elif self.app.GetParameterRole(key) == 1 and bool(value) or value == 0:
+ if isinstance(value, str):
+ try:
+ value = literal_eval(value)
+ except (ValueError, SyntaxError):
+ pass
+ self.data[key] = value
+
# Special functions
def __getitem__(self, key: str) -> Any | list[int | float] | int | float | Slicer:
"""This function is called when we use App()[...].
--
GitLab
From 236b80ef2ff92a7dc3fadd8c9f9a4645f1947241 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 22 May 2023 11:11:13 +0200
Subject: [PATCH 165/399] ADD: strip_input_paths and strip_output_paths in
summarize() + subtle refactoring to achieve this elegantly
---
pyotb/core.py | 136 +++++++++++++++++++++++++++++++++++++++++++-------
1 file changed, 119 insertions(+), 17 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index ad18769..8c30da8 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -389,6 +389,33 @@ class OTBObject(ABC):
class App(OTBObject):
"""Base class that gathers common operations for any OTB application."""
+ INPUT_IMAGE_TYPES = [
+ # Images only
+ otb.ParameterType_InputImage,
+ otb.ParameterType_InputImageList
+ ]
+
+ INPUT_PARAM_TYPES = INPUT_IMAGE_TYPES + [
+ # Vectors
+ otb.ParameterType_InputVectorData,
+ otb.ParameterType_InputVectorDataList,
+ # Filenames
+ otb.ParameterType_InputFilename,
+ otb.ParameterType_InputFilenameList,
+ ]
+
+ OUTPUT_IMAGES_TYPES = [
+ # Images only
+ otb.ParameterType_OutputImage
+ ]
+
+ OUTPUT_PARAM_TYPES = OUTPUT_IMAGES_TYPES + [
+ # Vectors
+ otb.ParameterType_OutputVectorData,
+ # Filenames
+ otb.ParameterType_OutputFilename,
+ ]
+
def __init__(self, appname: str, *args, frozen: bool = False, quiet: bool = False, name: str = "", **kwargs):
"""Common constructor for OTB applications. Handles in-memory connection between apps.
@@ -452,39 +479,78 @@ class App(OTBObject):
"""Returns internal _exports_dic object that contains numpy array exports."""
return self._exports_dic
- def get_first_key(self, *type_lists: tuple[list[int]]) -> str:
+ def _is_one_of_types(self, key: str, param_types: list[int]) -> bool:
+ """
+ Helper to factor is_input and is_output
+ """
+ if key not in self._all_param_types:
+ raise KeyError(
+ f"key {key} not found in the application parameters types"
+ )
+ return self._all_param_types[key] in param_types
+
+ @property
+ def is_input(self, key) -> bool:
+ """
+ Returns True if the key is an input
+
+ Args:
+ key: parameter key
+
+ Returns:
+ True if the parameter is an input, else False
+
+ """
+ return self._is_one_of_types(
+ key=key,
+ param_types=self.INPUT_PARAM_TYPES
+ )
+
+ @property
+ def is_output(self, key) -> bool:
+ """
+ Returns True if the key is an output
+
+ Args:
+ key: parameter key
+
+ Returns:
+ True if the parameter is an output, else False
+
+ """
+ return self._is_one_of_types(
+ key=key,
+ param_types=self.OUTPUT_PARAM_TYPES
+ )
+ def get_first_key(self, param_types: list[int]) -> str:
"""Get the first param key for specific file types, try each list in args."""
- for param_types in type_lists:
+ for param_type in param_types:
+ # Return the first key, from the alphabetically sorted keys of the
+ # application, which has the parameter type matching param_type.
for key, value in sorted(self._all_param_types.items()):
- if value in param_types:
+ if value == param_type:
return key
raise TypeError(f"{self.name}: could not find any parameter key matching the provided types")
@property
def input_key(self) -> str:
"""Get the name of first input parameter, raster > vector > file."""
- return self.get_first_key(
- [otb.ParameterType_InputImage, otb.ParameterType_InputImageList],
- [otb.ParameterType_InputVectorData, otb.ParameterType_InputVectorDataList],
- [otb.ParameterType_InputFilename, otb.ParameterType_InputFilenameList],
- )
+ return self.get_first_key(self.INPUT_PARAM_TYPES)
@property
def input_image_key(self) -> str:
"""Name of the first input image parameter."""
- return self.get_first_key([otb.ParameterType_InputImage, otb.ParameterType_InputImageList])
+ return self.get_first_key(self.INPUT_IMAGE_TYPES)
@property
def output_key(self) -> str:
"""Name of the first output parameter, raster > vector > file."""
- return self.get_first_key(
- [otb.ParameterType_OutputImage], [otb.ParameterType_OutputVectorData], [otb.ParameterType_OutputFilename]
- )
+ return self.get_first_key(self.OUTPUT_PARAM_TYPES)
@property
def output_image_key(self) -> str:
"""Get the name of first output image parameter."""
- return self.get_first_key([otb.ParameterType_OutputImage])
+ return self.get_first_key(self.OUTPUT_IMAGES_TYPES)
@property
def elapsed_time(self) -> float:
@@ -1285,19 +1351,55 @@ def get_out_images_param_keys(app: OTBObject) -> list[str]:
return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage]
-def summarize(obj: App | Output | Any) -> dict[str, str | dict[str, Any]]:
- """Recursively summarize application parameters, and every App or Output found upstream in the pipeline.
+def summarize(
+ obj: App | Output | Any,
+ strip_input_paths: bool = False,
+ strip_output_paths: bool = False,
+) -> dict[str, str | dict[str, Any]]:
+ """Recursively summarize application parameters, and every App or Output
+ found upstream in the pipeline.
+
+ Args:
+ obj: input object to summarize
+ strip_input_paths: strip all input paths: If enabled, paths related to
+ inputs are truncated after the first "?" character. Can be useful
+ to remove URLs tokens (e.g. SAS or S3 credentials).
+ strip_output_paths: strip all output paths: If enabled, paths related
+ to outputs are truncated after the first "?" character. Can be
+ useful to remove extended filenames.
Returns:
- nested dictionary with serialized App(s) containing name and parameters of an app and its parents
+ nested dictionary with serialized App(s) containing name and
+ parameters of an app and its parents
"""
if isinstance(obj, Output):
return summarize(obj.parent_pyotb_app)
if not isinstance(obj, App):
return obj
+
+ # If we are here, "obj" is an App
+
+ def _summarize_single_param(key, param):
+ """
+ This function truncates inputs or outputs paths, before the first
+ occurrence of character "?".
+ """
+ param_summary = summarize(param)
+ if isinstance(param, str):
+ # If we are here, "param" could be any str parameter value
+ if strip_input_paths and obj.is_input(
+ key) or strip_output_paths and obj.is_output(key):
+ # If we are here, "param" is a path to an input or output
+ # image, vector, or filename
+ param_summary = param_summary.split("?")[0]
+
+ return param_summary
+
parameters = {
- key: [summarize(p) for p in param] if isinstance(param, list) else summarize(param)
+ key: [
+ _summarize_single_param(key, p) for p in param
+ ] if isinstance(param, list) else _summarize_single_param(key, param)
for key, param in obj.parameters.items()
}
return {"name": obj.app.GetName(), "parameters": parameters}
--
GitLab
From 9174c412029bb506b837246564c201b19af3c1f6 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 22 May 2023 11:20:49 +0200
Subject: [PATCH 166/399] ADD: fix property-with-parameters
---
pyotb/core.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 8c30da8..6b4ef8d 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -489,7 +489,6 @@ class App(OTBObject):
)
return self._all_param_types[key] in param_types
- @property
def is_input(self, key) -> bool:
"""
Returns True if the key is an input
@@ -506,7 +505,6 @@ class App(OTBObject):
param_types=self.INPUT_PARAM_TYPES
)
- @property
def is_output(self, key) -> bool:
"""
Returns True if the key is an output
--
GitLab
From 9156b59250d88670347161fd5d6c960c828de7d7 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 12:07:33 +0200
Subject: [PATCH 167/399] ENH: smaller summarize function, check '?' is in path
before split
---
pyotb/core.py | 37 +++++++++++++------------------------
1 file changed, 13 insertions(+), 24 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 6b4ef8d..db392d6 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1350,9 +1350,9 @@ def get_out_images_param_keys(app: OTBObject) -> list[str]:
def summarize(
- obj: App | Output | Any,
- strip_input_paths: bool = False,
- strip_output_paths: bool = False,
+ obj: App | Output | Any,
+ strip_input_paths: bool = False,
+ strip_output_paths: bool = False,
) -> dict[str, str | dict[str, Any]]:
"""Recursively summarize application parameters, and every App or Output
found upstream in the pipeline.
@@ -1371,33 +1371,22 @@ def summarize(
parameters of an app and its parents
"""
- if isinstance(obj, Output):
- return summarize(obj.parent_pyotb_app)
- if not isinstance(obj, App):
- return obj
- # If we are here, "obj" is an App
-
- def _summarize_single_param(key, param):
- """
- This function truncates inputs or outputs paths, before the first
- occurrence of character "?".
- """
+ def strip_path(key: str, param: str | Any):
+ """Truncate text after the first "?" character in filepath."""
param_summary = summarize(param)
- if isinstance(param, str):
- # If we are here, "param" could be any str parameter value
- if strip_input_paths and obj.is_input(
- key) or strip_output_paths and obj.is_output(key):
- # If we are here, "param" is a path to an input or output
- # image, vector, or filename
+ if strip_input_paths and obj.is_input(key) or strip_output_paths and obj.is_output(key):
+ if isinstance(param, str) and "?" in param_summary:
param_summary = param_summary.split("?")[0]
-
return param_summary
+ if isinstance(obj, Output):
+ return summarize(obj.parent_pyotb_app)
+ if not isinstance(obj, App):
+ return obj
+ # If we are here, "obj" is an App
parameters = {
- key: [
- _summarize_single_param(key, p) for p in param
- ] if isinstance(param, list) else _summarize_single_param(key, param)
+ key: [strip_path(key, p) for p in param] if isinstance(param, list) else strip_path(key, param)
for key, param in obj.parameters.items()
}
return {"name": obj.app.GetName(), "parameters": parameters}
--
GitLab
From 58d7401ce16d86a8e4fc44060b59750798d0d673 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 12:12:44 +0200
Subject: [PATCH 168/399] STYLE: add missing underscore and type hints
---
pyotb/core.py | 17 ++++++++---------
1 file changed, 8 insertions(+), 9 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index db392d6..d0a3b48 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -479,7 +479,7 @@ class App(OTBObject):
"""Returns internal _exports_dic object that contains numpy array exports."""
return self._exports_dic
- def _is_one_of_types(self, key: str, param_types: list[int]) -> bool:
+ def __is_one_of_types(self, key: str, param_types: list[int]) -> bool:
"""
Helper to factor is_input and is_output
"""
@@ -489,7 +489,7 @@ class App(OTBObject):
)
return self._all_param_types[key] in param_types
- def is_input(self, key) -> bool:
+ def is_input(self, key: str) -> bool:
"""
Returns True if the key is an input
@@ -500,12 +500,11 @@ class App(OTBObject):
True if the parameter is an input, else False
"""
- return self._is_one_of_types(
- key=key,
- param_types=self.INPUT_PARAM_TYPES
+ return self.__is_one_of_types(
+ key=key, param_types=self.INPUT_PARAM_TYPES
)
- def is_output(self, key) -> bool:
+ def is_output(self, key: str) -> bool:
"""
Returns True if the key is an output
@@ -516,10 +515,10 @@ class App(OTBObject):
True if the parameter is an output, else False
"""
- return self._is_one_of_types(
- key=key,
- param_types=self.OUTPUT_PARAM_TYPES
+ return self.__is_one_of_types(
+ key=key, param_types=self.OUTPUT_PARAM_TYPES
)
+
def get_first_key(self, param_types: list[int]) -> str:
"""Get the first param key for specific file types, try each list in args."""
for param_type in param_types:
--
GitLab
From 26593adab359f38ecafdc6f619c99f063095dca2 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 12:23:56 +0200
Subject: [PATCH 169/399] CI: docstrings
---
pyotb/core.py | 13 ++++---------
1 file changed, 4 insertions(+), 9 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index d0a3b48..b3f2278 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -480,9 +480,7 @@ class App(OTBObject):
return self._exports_dic
def __is_one_of_types(self, key: str, param_types: list[int]) -> bool:
- """
- Helper to factor is_input and is_output
- """
+ """Helper to factor is_input and is_output."""
if key not in self._all_param_types:
raise KeyError(
f"key {key} not found in the application parameters types"
@@ -490,8 +488,7 @@ class App(OTBObject):
return self._all_param_types[key] in param_types
def is_input(self, key: str) -> bool:
- """
- Returns True if the key is an input
+ """Returns True if the key is an input.
Args:
key: parameter key
@@ -505,8 +502,7 @@ class App(OTBObject):
)
def is_output(self, key: str) -> bool:
- """
- Returns True if the key is an output
+ """Returns True if the key is an output.
Args:
key: parameter key
@@ -1353,8 +1349,7 @@ def summarize(
strip_input_paths: bool = False,
strip_output_paths: bool = False,
) -> dict[str, str | dict[str, Any]]:
- """Recursively summarize application parameters, and every App or Output
- found upstream in the pipeline.
+ """Recursively summarize parameters of an App or Output object and its parents.
Args:
obj: input object to summarize
--
GitLab
From 0894ff58ce1178f4a80e495b856a684e01f02d9e Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 22 May 2023 13:53:10 +0200
Subject: [PATCH 170/399] TEST: add a test for the path strippping feature of
summarize()
---
tests/test_core.py | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/tests/test_core.py b/tests/test_core.py
index ebb26cf..7b2d53d 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -228,6 +228,26 @@ def test_summarize_output():
assert pyotb.summarize(INPUT["out"])
+def test_summarize_strip_output():
+ in_fn = FILEPATH
+ in_fn_w_ext = FILEPATH + "?&skipcarto=1"
+ out_fn = "/tmp/output.tif"
+ out_fn_w_ext = out_fn + "?&box=10:10:10:10"
+
+ baseline = [
+ (in_fn, out_fn_w_ext, "out", {}, out_fn_w_ext),
+ (in_fn, out_fn_w_ext, "out", {"strip_output_paths": True}, out_fn),
+ (in_fn_w_ext, out_fn, "in", {}, in_fn_w_ext),
+ (in_fn_w_ext, out_fn, "in", {"strip_input_paths": True}, in_fn)
+ ]
+
+ for inp, out, key, extra_args, expected in baseline:
+ app = pyotb.ExtractROI({"in": inp, "out": out})
+ summary = pyotb.summarize(app, **extra_args)
+ assert summary["parameters"][key] == expected, \
+ f"Failed for input {inp}, output {out}, args {extra_args}"
+
+
def test_pipeline_simple():
# BandMath -> OrthoRectification -> ManageNoData
app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"})
@@ -245,3 +265,5 @@ def test_pipeline_diamond():
app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"})
summary = pyotb.summarize(app4)
assert summary == COMPLEX_SERIALIZATION
+
+test_summarize_strip_output()
\ No newline at end of file
--
GitLab
From e24baa1f5d42532dbe4ffde8f982748f9d2263fa Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 14:47:22 +0200
Subject: [PATCH 171/399] ENH: apply missing patch from previous MR
---
tests/test_core.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/tests/test_core.py b/tests/test_core.py
index 7b2d53d..2752865 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -265,5 +265,3 @@ def test_pipeline_diamond():
app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"})
summary = pyotb.summarize(app4)
assert summary == COMPLEX_SERIALIZATION
-
-test_summarize_strip_output()
\ No newline at end of file
--
GitLab
From 3231055755ba7fb00b34ca2259ac97b685ab1ec6 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 15:54:37 +0200
Subject: [PATCH 172/399] ENH: new function add_vsi_prefix
---
pyotb/core.py | 37 +++++++++++++++++++++++++++++++++++--
1 file changed, 35 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index b3f2278..d1cd680 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -580,8 +580,13 @@ class App(OTBObject):
obj = [obj]
logger.info('%s: argument for parameter "%s" was converted to list', self.name, key)
try:
- # This is when we actually call self.app.SetParameter*
- self.__set_param(key, obj)
+ if self.is_input(key):
+ if is_key_images_list(self, key):
+ self.__set_param(key, [add_vsi_prefix(p) for p in obj])
+ else:
+ self.__set_param(key, add_vsi_prefix(obj))
+ else:
+ self.__set_param(key, obj)
except (RuntimeError, TypeError, ValueError, KeyError) as e:
raise RuntimeError(
f"{self.name}: error before execution, while setting parameter '{key}' to '{obj}': {e})"
@@ -1243,6 +1248,34 @@ class Output(OTBObject):
return str(self.filepath)
+def add_vsi_prefix(filepath: str | Path) -> str:
+ """Append vsi prefixes to file URL or path if needed.
+
+ Args:
+ filepath: file path or URL
+
+ Returns:
+ string with new /vsi prefix(es)
+
+ """
+ if isinstance(filepath, Path):
+ filepath = str(filepath)
+ if isinstance(filepath, str) and not filepath.startswith("/vsi"):
+ # Remote file
+ if filepath.startswith("https://") or filepath.startswith("http://"):
+ filepath = "/vsicurl/" + filepath
+ # Compressed file
+ if filepath.endswith(".tar") or filepath.endswith(".tar.gz") or filepath.endswith(".tgz"):
+ filepath = "/vsitar/" + filepath
+ elif filepath.endswith(".gz"):
+ filepath = "/vsigzip/" + filepath
+ elif filepath.endswith(".7z"):
+ filepath = "/vsi7z/" + filepath
+ elif filepath.endswith(".zip"):
+ filepath = "/vsizip/" + filepath
+ return filepath
+
+
def get_nbchannels(inp: str | Path | OTBObject) -> int:
"""Get the nb of bands of input image.
--
GitLab
From 50a6020ce13ae052dff9af40e332aec32a131f91 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 15:54:40 +0200
Subject: [PATCH 173/399] ENH: update test, now vsicurl is not required
---
tests/serialized_apps.json | 8 ++++----
tests/tests_data.py | 2 +-
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/tests/serialized_apps.json b/tests/serialized_apps.json
index c413645..424afba 100644
--- a/tests/serialized_apps.json
+++ b/tests/serialized_apps.json
@@ -25,7 +25,7 @@
"name": "BandMath",
"parameters": {
"il": [
- "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
+ "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
],
"exp": "im1b1"
}
@@ -60,7 +60,7 @@
"name": "BandMath",
"parameters": {
"il": [
- "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
+ "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
],
"exp": "im1b1"
}
@@ -93,7 +93,7 @@
"name": "BandMath",
"parameters": {
"il": [
- "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
+ "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
],
"exp": "im1b1"
}
@@ -106,4 +106,4 @@
"exp": "im1+im2"
}
}
-}
\ No newline at end of file
+}
diff --git a/tests/tests_data.py b/tests/tests_data.py
index bfb774b..36c3592 100644
--- a/tests/tests_data.py
+++ b/tests/tests_data.py
@@ -2,7 +2,7 @@ import json
from pathlib import Path
import pyotb
-FILEPATH = "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
+FILEPATH = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
INPUT = pyotb.Input(FILEPATH)
TEST_IMAGE_STATS = {
'out.mean': [79.5505, 109.225, 115.456, 249.349],
--
GitLab
From cab52f92c31ad806e7218defd347a0c5c9a4d988 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 16:00:11 +0200
Subject: [PATCH 174/399] CI: docstring indent
---
pyotb/core.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index d1cd680..b52dbfe 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1251,11 +1251,11 @@ class Output(OTBObject):
def add_vsi_prefix(filepath: str | Path) -> str:
"""Append vsi prefixes to file URL or path if needed.
- Args:
- filepath: file path or URL
+ Args:
+ filepath: file path or URL
- Returns:
- string with new /vsi prefix(es)
+ Returns:
+ string with new /vsi prefix(es)
"""
if isinstance(filepath, Path):
--
GitLab
From b7077db05ff38b924310fb579ae162c44750837a Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 16:38:25 +0200
Subject: [PATCH 175/399] ENH: TODO comment about S3 and GCP
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index b52dbfe..7be171d 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1261,7 +1261,7 @@ def add_vsi_prefix(filepath: str | Path) -> str:
if isinstance(filepath, Path):
filepath = str(filepath)
if isinstance(filepath, str) and not filepath.startswith("/vsi"):
- # Remote file
+ # Remote file. TODO: add support for S3 and GCP URLs
if filepath.startswith("https://") or filepath.startswith("http://"):
filepath = "/vsicurl/" + filepath
# Compressed file
--
GitLab
From 74447f1f286a08c2d48ce62a6e228618aaca1340 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 16:49:52 +0200
Subject: [PATCH 176/399] ENH: move is_key_list and is_key_images_list back to
App
---
pyotb/core.py | 54 +++++++++++++++++++++++++--------------------------
1 file changed, 27 insertions(+), 27 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 7be171d..8c9cce4 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -394,7 +394,6 @@ class App(OTBObject):
otb.ParameterType_InputImage,
otb.ParameterType_InputImageList
]
-
INPUT_PARAM_TYPES = INPUT_IMAGE_TYPES + [
# Vectors
otb.ParameterType_InputVectorData,
@@ -404,18 +403,29 @@ class App(OTBObject):
otb.ParameterType_InputFilenameList,
]
- OUTPUT_IMAGES_TYPES = [
+ OUTPUT_IMAGE_TYPES = [
# Images only
otb.ParameterType_OutputImage
]
-
- OUTPUT_PARAM_TYPES = OUTPUT_IMAGES_TYPES + [
+ OUTPUT_PARAM_TYPES = OUTPUT_IMAGE_TYPES + [
# Vectors
otb.ParameterType_OutputVectorData,
# Filenames
otb.ParameterType_OutputFilename,
]
+ INPUT_LIST_TYPES = [
+ otb.ParameterType_InputImageList,
+ otb.ParameterType_StringList,
+ otb.ParameterType_InputFilenameList,
+ otb.ParameterType_ListView,
+ otb.ParameterType_InputVectorDataList,
+ ]
+ INPUT_IMAGES_LIST_TYPES = [
+ otb.ParameterType_InputImageList,
+ otb.ParameterType_InputFilenameList,
+ ]
+
def __init__(self, appname: str, *args, frozen: bool = False, quiet: bool = False, name: str = "", **kwargs):
"""Common constructor for OTB applications. Handles in-memory connection between apps.
@@ -515,6 +525,14 @@ class App(OTBObject):
key=key, param_types=self.OUTPUT_PARAM_TYPES
)
+ def is_key_list(self, key: str) -> bool:
+ """Check if a parameter key is an input parameter list."""
+ return self.app.GetParameterType(key) in self.INPUT_LIST_TYPES
+
+ def is_key_images_list(self, key: str) -> bool:
+ """Check if a parameter key is an input parameter image list."""
+ return self.app.GetParameterType(key) in self.INPUT_IMAGES_LIST_TYPES
+
def get_first_key(self, param_types: list[int]) -> str:
"""Get the first param key for specific file types, try each list in args."""
for param_type in param_types:
@@ -543,7 +561,7 @@ class App(OTBObject):
@property
def output_image_key(self) -> str:
"""Get the name of first output image parameter."""
- return self.get_first_key(self.OUTPUT_IMAGES_TYPES)
+ return self.get_first_key(self.OUTPUT_IMAGE_TYPES)
@property
def elapsed_time(self) -> float:
@@ -576,12 +594,12 @@ class App(OTBObject):
f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}"
)
# When the parameter expects a list, if needed, change the value to list
- if is_key_list(self, key) and not isinstance(obj, (list, tuple)):
+ if self.is_key_list(key) and not isinstance(obj, (list, tuple)):
obj = [obj]
logger.info('%s: argument for parameter "%s" was converted to list', self.name, key)
try:
if self.is_input(key):
- if is_key_images_list(self, key):
+ if self.is_key_images_list(key):
self.__set_param(key, [add_vsi_prefix(p) for p in obj])
else:
self.__set_param(key, add_vsi_prefix(obj))
@@ -749,7 +767,7 @@ class App(OTBObject):
for arg in args:
if isinstance(arg, dict):
kwargs.update(arg)
- elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and is_key_list(self, self.input_key):
+ elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and self.is_key_list(self.input_key):
kwargs.update({self.input_key: arg})
return kwargs
@@ -768,7 +786,7 @@ class App(OTBObject):
elif not isinstance(obj, list): # any other parameters (str, int...)
self.app.SetParameterValue(key, obj)
# Images list
- elif is_key_images_list(self, key):
+ elif self.is_key_images_list(key):
# To enable possible in-memory connections, we go through the list and set the parameters one by one
for inp in obj:
if isinstance(inp, OTBObject):
@@ -1354,24 +1372,6 @@ def parse_pixel_type(pixel_type: str | int) -> int:
raise TypeError(f"Bad pixel type specification ({pixel_type} of type {type(pixel_type)})")
-def is_key_list(pyotb_app: OTBObject, key: str) -> bool:
- """Check if a key of the OTBObject is an input parameter list."""
- types = (
- otb.ParameterType_InputImageList,
- otb.ParameterType_StringList,
- otb.ParameterType_InputFilenameList,
- otb.ParameterType_ListView,
- otb.ParameterType_InputVectorDataList,
- )
- return pyotb_app.app.GetParameterType(key) in types
-
-
-def is_key_images_list(pyotb_app: OTBObject, key: str) -> bool:
- """Check if a key of the OTBObject is an input parameter image list."""
- types = (otb.ParameterType_InputImageList, otb.ParameterType_InputFilenameList)
- return pyotb_app.app.GetParameterType(key) in types
-
-
def get_out_images_param_keys(app: OTBObject) -> list[str]:
"""Return every output parameter keys of an OTB app."""
return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage]
--
GitLab
From c1488e9909e43d219f57cc1968ca777e3ffe2114 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 17:07:50 +0200
Subject: [PATCH 177/399] ENH: add test case for input file with existing vsi
prefix
---
tests/test_core.py | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/tests/test_core.py b/tests/test_core.py
index 2752865..3afffd1 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -6,9 +6,16 @@ from tests_data import *
# Input settings
def test_parameters():
+ assert INPUT.parameters
+ assert INPUT.parameters["in"] == FILEPATH
assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304)
+def test_input_with_vsi():
+ # Ensure old way is still working - this should raise RuntimeError when the path is malformed
+ pyotb.Input("/vsicurl/" + FILEPATH)
+
+
def test_wrong_key():
with pytest.raises(KeyError):
pyotb.BandMath(INPUT, expression="im1b1")
--
GitLab
From 90f5845afbcc2e897a4d6ffead84d0645e573713 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 17:18:50 +0200
Subject: [PATCH 178/399] ENH: enhance add_vsi_prefix function, add ftp:// and
multi prefix checks, works with extfn
---
pyotb/core.py | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 8c9cce4..5cd566a 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1239,7 +1239,7 @@ class Output(OTBObject):
@filepath.setter
def filepath(self, path: str):
if isinstance(path, str):
- if path and not path.startswith("/vsi"):
+ if path and not path.startswith(["/vsi", "http", "ftp"]):
path = Path(path.split("?")[0])
self._filepath = path
@@ -1279,17 +1279,18 @@ def add_vsi_prefix(filepath: str | Path) -> str:
if isinstance(filepath, Path):
filepath = str(filepath)
if isinstance(filepath, str) and not filepath.startswith("/vsi"):
- # Remote file. TODO: add support for S3 and GCP URLs
- if filepath.startswith("https://") or filepath.startswith("http://"):
+ # Remote file. TODO: add support for S3 / GS / AZ
+ if filepath.startswith(["https://", "http://", "ftp://"]):
filepath = "/vsicurl/" + filepath
# Compressed file
- if filepath.endswith(".tar") or filepath.endswith(".tar.gz") or filepath.endswith(".tgz"):
+ basename = filepath.split("?")[0]
+ if basename.endswith([".tar", ".tar.gz", ".tgz"]):
filepath = "/vsitar/" + filepath
- elif filepath.endswith(".gz"):
+ elif basename.endswith(".gz"):
filepath = "/vsigzip/" + filepath
- elif filepath.endswith(".7z"):
+ elif basename.endswith(".7z"):
filepath = "/vsi7z/" + filepath
- elif filepath.endswith(".zip"):
+ elif basename.endswith(".zip"):
filepath = "/vsizip/" + filepath
return filepath
--
GitLab
From 148b0cb06e494949543c1eb35e0f24eea88c7c53 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 17:24:54 +0200
Subject: [PATCH 179/399] FIX: startswith and endswith multi check requires
tuple argument
---
pyotb/core.py | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 5cd566a..d7de84a 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1180,7 +1180,9 @@ class Input(App):
"""
super().__init__("ExtractROI", {"in": filepath}, frozen=True)
self._name = f"Input from {filepath}"
- self.filepath = Path(filepath) if not filepath.startswith("/vsi") else filepath
+ if not filepath.startswith(("/vsi", "http", "ftp")):
+ filepath = Path(filepath)
+ self.filepath = filepath
self.propagate_dtype()
self.execute()
@@ -1239,7 +1241,7 @@ class Output(OTBObject):
@filepath.setter
def filepath(self, path: str):
if isinstance(path, str):
- if path and not path.startswith(["/vsi", "http", "ftp"]):
+ if path and not path.startswith(("/vsi", "http", "ftp")):
path = Path(path.split("?")[0])
self._filepath = path
@@ -1280,11 +1282,11 @@ def add_vsi_prefix(filepath: str | Path) -> str:
filepath = str(filepath)
if isinstance(filepath, str) and not filepath.startswith("/vsi"):
# Remote file. TODO: add support for S3 / GS / AZ
- if filepath.startswith(["https://", "http://", "ftp://"]):
+ if filepath.startswith(("https://", "http://", "ftp://")):
filepath = "/vsicurl/" + filepath
# Compressed file
basename = filepath.split("?")[0]
- if basename.endswith([".tar", ".tar.gz", ".tgz"]):
+ if basename.endswith((".tar", ".tar.gz", ".tgz")):
filepath = "/vsitar/" + filepath
elif basename.endswith(".gz"):
filepath = "/vsigzip/" + filepath
--
GitLab
From 3f5fc1454b859b952d13e090958e803cbc2f7d82 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 17:37:26 +0200
Subject: [PATCH 180/399] FIX: add test error message if image could not be
loaded
---
tests/tests_data.py | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/tests/tests_data.py b/tests/tests_data.py
index 36c3592..b54c6f2 100644
--- a/tests/tests_data.py
+++ b/tests/tests_data.py
@@ -2,8 +2,14 @@ import json
from pathlib import Path
import pyotb
-FILEPATH = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
-INPUT = pyotb.Input(FILEPATH)
+FILEPATH = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tiff"
+try:
+ INPUT = pyotb.Input(FILEPATH)
+except RuntimeError as err:
+ if str(err.__cause__).startswith("Cannot open image "):
+ raise Exception("Unable to access the remote image, GitLab could be offline.") from err
+ raise Exception("Unexpected error while fetching test data.") from err
+
TEST_IMAGE_STATS = {
'out.mean': [79.5505, 109.225, 115.456, 249.349],
'out.min': [33, 64, 91, 47],
--
GitLab
From 853a9f49ac3b25d7c5cbea958c32150697e0406f Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 17:37:49 +0200
Subject: [PATCH 181/399] FIX: typo
---
tests/tests_data.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/tests_data.py b/tests/tests_data.py
index b54c6f2..ac789ad 100644
--- a/tests/tests_data.py
+++ b/tests/tests_data.py
@@ -2,7 +2,7 @@ import json
from pathlib import Path
import pyotb
-FILEPATH = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tiff"
+FILEPATH = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
try:
INPUT = pyotb.Input(FILEPATH)
except RuntimeError as err:
--
GitLab
From 781f10a5cdda63268a15b8be7342a01b1fe14ba5 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 17:51:14 +0200
Subject: [PATCH 182/399] ENH: error message in test_data
---
tests/tests_data.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/tests_data.py b/tests/tests_data.py
index ac789ad..3aefc9c 100644
--- a/tests/tests_data.py
+++ b/tests/tests_data.py
@@ -7,7 +7,7 @@ try:
INPUT = pyotb.Input(FILEPATH)
except RuntimeError as err:
if str(err.__cause__).startswith("Cannot open image "):
- raise Exception("Unable to access the remote image, GitLab could be offline.") from err
+ raise Exception("Unable to access the remote image, GitLab might be offline.") from err
raise Exception("Unexpected error while fetching test data.") from err
TEST_IMAGE_STATS = {
--
GitLab
From 3a476b69b371c42db44ed718bd0ea9891e1d4e5b Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 19:10:08 +0200
Subject: [PATCH 183/399] ENH: add test for new vsi behaviour with case remote
compressed file
---
tests/test_core.py | 15 +++++++++++++--
tests/tests_data.py | 2 +-
2 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/tests/test_core.py b/tests/test_core.py
index 3afffd1..5a313e0 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -11,8 +11,19 @@ def test_parameters():
assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304)
-def test_input_with_vsi():
- # Ensure old way is still working - this should raise RuntimeError when the path is malformed
+def test_input_vsi():
+ # Simple remote file
+ info = pyotb.ReadImageInfo("https://fake.com/image.tif", frozen=True)
+ assert info.app.GetParameterValue("in") == "/vsicurl/https://fake.com/image.tif"
+ assert info.parameters["in"] == "https://fake.com/image.tif"
+ # Compressed remote file
+ info = pyotb.ReadImageInfo("https://fake.com/image.tif.zip", frozen=True)
+ assert info.app.GetParameterValue("in") == "/vsizip//vsicurl/https://fake.com/image.tif.zip"
+ assert info.parameters["in"] == "https://fake.com/image.tif.zip"
+
+
+def test_input_vsi_from_user():
+ # Ensure old way is still working: ExtractROI will raise RuntimeError if a path is malformed
pyotb.Input("/vsicurl/" + FILEPATH)
diff --git a/tests/tests_data.py b/tests/tests_data.py
index 3aefc9c..0f10b18 100644
--- a/tests/tests_data.py
+++ b/tests/tests_data.py
@@ -7,7 +7,7 @@ try:
INPUT = pyotb.Input(FILEPATH)
except RuntimeError as err:
if str(err.__cause__).startswith("Cannot open image "):
- raise Exception("Unable to access the remote image, GitLab might be offline.") from err
+ raise Exception("Unable to access remote image, GitLab might be offline.") from err
raise Exception("Unexpected error while fetching test data.") from err
TEST_IMAGE_STATS = {
--
GitLab
From 22a968cbab7f49ebdb48ef530c21b1e89b65b5da Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 22 May 2023 20:47:51 +0200
Subject: [PATCH 184/399] TEST: piped curl -> zip -> tiff
---
tests/test_core.py | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/tests/test_core.py b/tests/test_core.py
index 5a313e0..228ed07 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -20,6 +20,16 @@ def test_input_vsi():
info = pyotb.ReadImageInfo("https://fake.com/image.tif.zip", frozen=True)
assert info.app.GetParameterValue("in") == "/vsizip//vsicurl/https://fake.com/image.tif.zip"
assert info.parameters["in"] == "https://fake.com/image.tif.zip"
+ # Piped curl --> zip --> tiff
+ ziped_tif_urls = (
+ "https://github.com/OSGeo/gdal/raw/master"
+ "/autotest/gcore/data/byte.tif.zip", # without /vsi
+ "/vsizip/vsicurl/https://github.com/OSGeo/gdal/raw/master"
+ "/autotest/gcore/data/byte.tif.zip", # with /vsi
+ )
+ for ziped_tif_url in ziped_tif_urls:
+ info = pyotb.ReadImageInfo(ziped_tif_url)
+ assert info["sizex"] == 20
def test_input_vsi_from_user():
--
GitLab
From 59b7d0339350592ab478e6c54a785644aa585b9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr>
Date: Mon, 22 May 2023 20:42:18 +0000
Subject: [PATCH 185/399] ENH : Check explicit http:// and https:// to avoid
any problems
---
pyotb/core.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index d7de84a..f380964 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1180,7 +1180,7 @@ class Input(App):
"""
super().__init__("ExtractROI", {"in": filepath}, frozen=True)
self._name = f"Input from {filepath}"
- if not filepath.startswith(("/vsi", "http", "ftp")):
+ if not filepath.startswith(("/vsi", "http://", "https://", "ftp://")):
filepath = Path(filepath)
self.filepath = filepath
self.propagate_dtype()
@@ -1241,7 +1241,7 @@ class Output(OTBObject):
@filepath.setter
def filepath(self, path: str):
if isinstance(path, str):
- if path and not path.startswith(("/vsi", "http", "ftp")):
+ if path and not path.startswith(("/vsi", "http://", "https://", "ftp://")):
path = Path(path.split("?")[0])
self._filepath = path
--
GitLab
From bd093487f06dd02c8ae2d975f1600bfe08117c12 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 20:52:14 +0000
Subject: [PATCH 186/399] ENH: use dict to search for vsi suffixes
---
pyotb/core.py | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index f380964..6debae4 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1285,15 +1285,18 @@ def add_vsi_prefix(filepath: str | Path) -> str:
if filepath.startswith(("https://", "http://", "ftp://")):
filepath = "/vsicurl/" + filepath
# Compressed file
+ prefixes = {
+ ".tar": "vsitar",
+ ".tgz": "vsitar",
+ ".gz": "vsigzip",
+ ".7z": "vsi7z",
+ ".zip": "vsizip",
+ ".rar": "vsirar"
+ }
basename = filepath.split("?")[0]
- if basename.endswith((".tar", ".tar.gz", ".tgz")):
- filepath = "/vsitar/" + filepath
- elif basename.endswith(".gz"):
- filepath = "/vsigzip/" + filepath
- elif basename.endswith(".7z"):
- filepath = "/vsi7z/" + filepath
- elif basename.endswith(".zip"):
- filepath = "/vsizip/" + filepath
+ ext = Path(basename).suffix
+ if ext in prefixes:
+ filepath = f"/{prefixes[ext]}/{filepath}"
return filepath
--
GitLab
From 20a9225f85adc6d89c3abe7288c176e36da559cf Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 22:56:04 +0200
Subject: [PATCH 187/399] FIX: bad ident in MR patch
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 6debae4..03b1203 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1296,7 +1296,7 @@ def add_vsi_prefix(filepath: str | Path) -> str:
basename = filepath.split("?")[0]
ext = Path(basename).suffix
if ext in prefixes:
- filepath = f"/{prefixes[ext]}/{filepath}"
+ filepath = f"/{prefixes[ext]}/{filepath}"
return filepath
--
GitLab
From 984691e45ccdc132174853c43debd3dbc8c6d1e1 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 22 May 2023 23:47:46 +0200
Subject: [PATCH 188/399] ENH: make recursion in summarize function more
readable
---
pyotb/core.py | 22 ++++++++++------------
1 file changed, 10 insertions(+), 12 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 03b1203..74e426f 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1404,22 +1404,20 @@ def summarize(
parameters of an app and its parents
"""
-
- def strip_path(key: str, param: str | Any):
- """Truncate text after the first "?" character in filepath."""
- param_summary = summarize(param)
- if strip_input_paths and obj.is_input(key) or strip_output_paths and obj.is_output(key):
- if isinstance(param, str) and "?" in param_summary:
- param_summary = param_summary.split("?")[0]
- return param_summary
+ def strip_path(param: str | Any):
+ if not isinstance(param, str):
+ return summarize(param)
+ return param.split("?")[0]
if isinstance(obj, Output):
return summarize(obj.parent_pyotb_app)
if not isinstance(obj, App):
return obj
# If we are here, "obj" is an App
- parameters = {
- key: [strip_path(key, p) for p in param] if isinstance(param, list) else strip_path(key, param)
- for key, param in obj.parameters.items()
- }
+ parameters = {}
+ for key, param in obj.parameters.items():
+ if strip_input_paths and obj.is_input(key) or strip_output_paths and obj.is_output(key):
+ parameters[key] = [strip_path(p) for p in param] if isinstance(param, list) else strip_path(param)
+ else:
+ parameters[key] = [summarize(p) for p in param] if isinstance(param, list) else summarize(param)
return {"name": obj.app.GetName(), "parameters": parameters}
--
GitLab
From bbecbd2b2e05a39fa1b97cd5e35491bea397aacb Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 23 May 2023 00:02:10 +0200
Subject: [PATCH 189/399] ENH: add list param case in summarize recursion
---
pyotb/core.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 74e426f..28e7e0d 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1409,15 +1409,17 @@ def summarize(
return summarize(param)
return param.split("?")[0]
+ if isinstance(obj, list):
+ return [summarize(o) for o in obj]
if isinstance(obj, Output):
return summarize(obj.parent_pyotb_app)
if not isinstance(obj, App):
return obj
- # If we are here, "obj" is an App
+
parameters = {}
for key, param in obj.parameters.items():
if strip_input_paths and obj.is_input(key) or strip_output_paths and obj.is_output(key):
parameters[key] = [strip_path(p) for p in param] if isinstance(param, list) else strip_path(param)
else:
- parameters[key] = [summarize(p) for p in param] if isinstance(param, list) else summarize(param)
+ parameters[key] = summarize(param)
return {"name": obj.app.GetName(), "parameters": parameters}
--
GitLab
From 78d771abcfab878711721e61752847f7801350bf Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 23 May 2023 11:08:52 +0200
Subject: [PATCH 190/399] TEST: check if remote tif file is accessible before
trying to use it
---
.gitlab-ci.yml | 2 +-
pyproject.toml | 2 +-
tests/tests_data.py | 15 ++++++++-------
3 files changed, 10 insertions(+), 9 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a60518f..f5a291e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -86,7 +86,7 @@ pages_test:
junit: test-*.xml
before_script:
- wget $IMAGE_URL -O $TEST_INPUT_IMAGE
- - pip install pytest
+ - pip install pytest requests
test_core:
extends: .tests
diff --git a/pyproject.toml b/pyproject.toml
index 845e218..ac4d3df 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -26,7 +26,7 @@ classifiers=[
]
[project.optional-dependencies]
-dev = ["pytest", "pylint", "codespell", "pydocstyle", "tomli"]
+dev = ["pytest", "pylint", "codespell", "pydocstyle", "tomli", "requests"]
[project.urls]
documentation = "https://pyotb.readthedocs.io"
diff --git a/tests/tests_data.py b/tests/tests_data.py
index 0f10b18..359081d 100644
--- a/tests/tests_data.py
+++ b/tests/tests_data.py
@@ -1,15 +1,16 @@
import json
from pathlib import Path
+import requests
import pyotb
+
FILEPATH = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
-try:
- INPUT = pyotb.Input(FILEPATH)
-except RuntimeError as err:
- if str(err.__cause__).startswith("Cannot open image "):
- raise Exception("Unable to access remote image, GitLab might be offline.") from err
- raise Exception("Unexpected error while fetching test data.") from err
+response = requests.get(FILEPATH, timeout=5)
+code = response.status_code
+if code != 200:
+ raise requests.HTTPError(f"Unable to fetch remote image, GitLab might be offline (HTTP {code}).")
+INPUT = pyotb.Input(FILEPATH)
TEST_IMAGE_STATS = {
'out.mean': [79.5505, 109.225, 115.456, 249.349],
'out.min': [33, 64, 91, 47],
@@ -18,7 +19,7 @@ TEST_IMAGE_STATS = {
}
json_file = Path(__file__).parent / "serialized_apps.json"
-with json_file.open("r") as js:
+with json_file.open("r", encoding="utf-8") as js:
data = json.load(js)
SIMPLE_SERIALIZATION = data["SIMPLE"]
COMPLEX_SERIALIZATION = data["COMPLEX"]
--
GitLab
From 681222b71adafe60be25b6111a5c87eddfb5c321 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 24 May 2023 10:55:39 +0200
Subject: [PATCH 191/399] FEAT: accept dotted parameters using underscore
---
pyotb/core.py | 2 ++
tests/test_core.py | 5 +++++
2 files changed, 7 insertions(+)
diff --git a/pyotb/core.py b/pyotb/core.py
index 28e7e0d..903f71f 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -589,6 +589,8 @@ class App(OTBObject):
parameters.update(self.__parse_args(args))
# Going through all arguments
for key, obj in parameters.items():
+ if "_" in key:
+ key = key.replace("_", ".")
if key not in self.parameters_keys:
raise KeyError(
f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}"
diff --git a/tests/test_core.py b/tests/test_core.py
index 228ed07..f766c58 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -37,6 +37,11 @@ def test_input_vsi_from_user():
pyotb.Input("/vsicurl/" + FILEPATH)
+def test_param_with_underscore():
+ app = pyotb.OrthoRectification(io_in=INPUT, map_epsg_code=2154)
+ assert app.parameters["map.epsg.code"] == 2154
+
+
def test_wrong_key():
with pytest.raises(KeyError):
pyotb.BandMath(INPUT, expression="im1b1")
--
GitLab
From 08de3dd8b83cc925ae7c3e146a927b7ae3025dd5 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 31 May 2023 10:04:14 +0000
Subject: [PATCH 192/399] Update README.md
---
.gitlab-ci.yml | 20 ++-
AUTHORS.md | 10 ++
README.md | 337 +++++-----------------------------------
doc/MISC.md | 5 +-
doc/comparison_otb.md | 133 +++++++++++-----
doc/features.md | 7 +-
doc/installation.md | 30 +++-
doc/interaction.md | 17 +-
doc/managing_loggers.md | 5 +-
doc/otb_versions.md | 13 +-
doc/troubleshooting.md | 9 +-
11 files changed, 230 insertions(+), 356 deletions(-)
create mode 100644 AUTHORS.md
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f5a291e..be29725 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -20,13 +20,21 @@ stages:
allow_failure: true
codespell:
+ rules:
+ - changes:
+ - pyotb/*.py
+ - tests/*.py
+ - README.md
extends: .static_analysis
before_script:
- pip install codespell
script:
- - codespell {pyotb,tests}
+ - codespell {pyotb,tests,doc,README.md}
flake8:
+ rules:
+ - changes:
+ - pyotb/*.py
extends: .static_analysis
before_script:
- pip install flake8
@@ -34,6 +42,9 @@ flake8:
- flake8 --max-line-length=120 $PWD/pyotb --ignore=F403,E402,F401,W503,W504
pydocstyle:
+ rules:
+ - changes:
+ - pyotb/*.py
extends: .static_analysis
before_script:
- pip install pydocstyle tomli
@@ -41,6 +52,9 @@ pydocstyle:
- pydocstyle $PWD/pyotb
pylint:
+ rules:
+ - changes:
+ - pyotb/*.py
extends: .static_analysis
before_script:
- pip install pylint
@@ -73,6 +87,10 @@ pages_test:
.tests:
stage: Tests
+ rules:
+ - changes:
+ - pyotb/*.py
+ - tests/*.py
allow_failure: false
variables:
OTB_ROOT: /opt/otb
diff --git a/AUTHORS.md b/AUTHORS.md
new file mode 100644
index 0000000..7ac46df
--- /dev/null
+++ b/AUTHORS.md
@@ -0,0 +1,10 @@
+# Project authors
+
+## Initial codebase
+
+* Nicolas NARÇON - IT engineer @ ESA - Rome (Italy)
+
+## Current maintainers
+
+* Rémi CRESSON - RS research engineer @ INRAe - Montpellier (France)
+* Vincent DELBAR - GIS engineer @ La TeleScop - Montpellier (France)
diff --git a/README.md b/README.md
index 60659a8..d72b294 100644
--- a/README.md
+++ b/README.md
@@ -1,318 +1,67 @@
-# pyotb: a pythonic extension of OTB
+# pyotb: a pythonic extension of Orfeo Toolbox
-Full documentation is available at [pyotb.readthedocs.io](https://pyotb.readthedocs.io/)
-
-[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/releases)
+[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/releases)
[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/master)
+[](https://pyotb.readthedocs.io/en/master/)
+**pyotb** wraps the [Orfeo Toolbox](https://www.orfeo-toolbox.org/) (OTB)
+python bindings to make it more developer friendly.
-## Installation
-Requirements:
-- python>=3.5 and numpy
-- OrfeoToolBox python API
-
-```bash
-pip install pyotb --upgrade
-```
-
-For Python>=3.6, latest version available is pyotb 1.5.1 For Python 3.5, latest version available is pyotb 1.2.2
-
-## Quickstart: running an OTB application as a oneliner
-pyotb has been written so that it is more convenient to run an application in Python.
-
-You can pass the parameters of an application as a dictionary :
-```python
-import pyotb
-resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear',
- 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5})
-```
-Note that pyotb has a 'lazy' evaluation: it only performs operation when it is needed, i.e. results are written to disk.
-Thus, the previous line doesn't trigger the application.
-
-To actually trigger the application execution, you need to write the result to disk:
-
-```python
-resampled.write('output.tif') # this is when the application actually runs
-```
-
-## Using Python keyword arguments
-It is also possible to use the Python keyword arguments notation for passing the parameters:
-```python
-output = pyotb.SuperImpose(inr='reference_image.tif', inm='image.tif')
-```
-is equivalent to:
-```python
-output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'})
-```
-
-Limitations : for this notation, python doesn't accept the parameter `in` or any parameter that contains a `.`. E.g., it is not possible to use `pyotb.RigidTransformResample(in=input_path...)` or `pyotb.VectorDataExtractROI(io.vd=vector_path...)`.
-
-
-
-
-## In-memory connections
-The big asset of pyotb is the ease of in-memory connections between apps.
-
-Let's start from our previous example. Consider the case where one wants to apply optical calibration and binary morphological dilatation
-following the undersampling.
-
-Using pyotb, you can pass the output of an app as input of another app :
-```python
-import pyotb
-
-resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear',
- 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5})
-
-calibrated = pyotb.OpticalCalibration({'in': resampled, 'level': 'toa'})
-
-dilated = pyotb.BinaryMorphologicalOperation({'in': calibrated, 'out': 'output.tif', 'filter': 'dilate',
- 'structype': 'ball', 'xradius': 3, 'yradius': 3})
-dilated.write('result.tif')
-```
-
-## Writing the result of an app
-Any pyotb object can be written to disk using the `write` method, e.g. :
-
-```python
-import pyotb
-
-resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear',
- 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5})
-# Here you can set optionally pixel type and extended filename variables
-resampled.write({'out': 'output.tif'}, pixel_type='uint16', filename_extension='?nodata=65535')
-```
-
-Another possibility for writing results is to set the output parameter when initializing the application:
-```python
-import pyotb
-
-resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', 'out': 'output.tif',
- 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5})
-# Here you can set optionally pixel type and extended filename variables
-resampled.write(pixel_type='uint16', filename_extension='?nodata=65535')
-```
-
-## Arithmetic operations
-Every pyotb object supports arithmetic operations, such as addition, subtraction, comparison...
-Consider an example where we want to compute a vegeteation mask from NDVI, i.e. the arithmetic operation `(nir - red) / (nir + red) > 0.3`
-
-With pyotb, one can simply do :
-```python
-import pyotb
-
-# transforming filepaths to pyotb objects
-nir, red = pyotb.Input('nir.tif'), pyotb.Input('red.tif')
-
-res = (nir - red) / (nir + red) > 0.3
-print(res.exp) # prints the BandMath expression: "((im1b1 - im2b1) / (im1b1 + im2b1)) > 0.3 ? 1 : 0"
-res.write('vegetation_mask.tif', pixel_type='uint8')
-```
-
-## Slicing
-pyotb objects support slicing in a Python fashion :
-
-```python
-import pyotb
-
-# transforming filepath to pyotb object
-inp = pyotb.Input('my_image.tif')
-
-inp[:, :, :3] # selecting first 3 bands
-inp[:, :, [0, 1, 4]] # selecting bands 1, 2 & 5
-inp[:1000, :1000] # selecting 1000x1000 subset, same as inp[:1000, :1000, :]
-inp[:100, :100].write('my_image_roi.tif') # write cropped image to disk
-```
-
-## Numpy-inspired functions
-Some functions have been written, entirely based on OTB, to mimic the behavior of some well-known numpy functions.
-### pyotb.where
-Equivalent of `numpy.where`.
-It is the equivalent of the muparser syntax `condition ? x : y` that can be used in OTB's BandMath.
-
-```python
-import pyotb
-
-# transforming filepaths to pyotb objects
-labels, image1, image2 = pyotb.Input('labels.tif'), pyotb.Input('image1.tif') , pyotb.Input('image2.tif')
-
-# If labels = 1, returns image1. Else, returns image2
-res = pyotb.where(labels == 1, image1, image2) # this would also work: pyotb.where(labels == 1, 'image1.tif', 'image2.tif')
-
-# A more complex example
-# If labels = 1, returns image1. If labels = 2, returns image2. If labels = 3, returns 3. Else 0
-res = pyotb.where(labels == 1, image1,
- pyotb.where(labels == 2, image2,
- pyotb.where(labels == 3, 3, 0)))
-
-```
-
-### pyotb.clip
-Equivalent of `numpy.clip`. Clip (limit) the values in a raster to a range.
-
-```python
-import pyotb
-
-res = pyotb.clip('my_image.tif', 0, 255) # clips the values between 0 and 255
-```
-
-### pyotb.all
-Equivalent of `numpy.all`.
-
-For only one image, this function checks that all bands of the image are True (i.e. !=0) and outputs
-a singleband boolean raster.
-For several images, this function checks that all images are True (i.e. !=0) and outputs
-a boolean raster, with as many bands as the inputs.
-
-
-### pyotb.any
-Equivalent of `numpy.any`.
-
-For only one image, this function checks that at least one band of the image is True (i.e. !=0) and outputs
-a singleband boolean raster.
-For several images, this function checks that at least one of the images is True (i.e. !=0) and outputs
-a boolean raster, with as many bands as the inputs.
-
-
-## Interaction with Numpy
-
-pyotb objects can be transparently used in numpy functions.
-
-For example:
-
-```python
-import pyotb
-import numpy as np
-
-inp = pyotb.Input('image.tif') # this is a pyotb object
-
-# Creating a numpy array of noise
-white_noise = np.random.normal(0, 50, size=inp.shape) # this is a numpy object
-
-# Adding the noise to the image
-noisy_image = inp + white_noise # magic: this is a pyotb object that has the same georeference as input.
- # `np.add(inp, white_noise)` would have worked the same
-noisy_image.write('image_plus_noise.tif')
-```
-Limitations :
-- The whole image is loaded into memory
-- The georeference can not be modified. Thus, numpy operations can not change the image or pixel size
-
-
-## Export to rasterio
-pyotb objects can also be exported in a format that is usable by rasterio.
-
-For example:
+## Key features
-```python
-import pyotb
-import rasterio
-from scipy import ndimage
-
-# Pansharpening + NDVI + creating bare soils mask
-pxs = pyotb.BundleToPerfectSensor(inp='panchromatic.tif', inxs='multispectral.tif')
-ndvi = pyotb.RadiometricIndices({'in': pxs, 'channels.red': 3, 'channels.nir': 4, 'list': 'Vegetation:NDVI'})
-bare_soil_mask = (ndvi < 0.3)
-
-# Exporting the result as array & profile usable by rasterio
-mask_array, profile = bare_soil_mask.to_rasterio()
-
-# Doing something in Python that is not possible with OTB, e.g. gathering the contiguous groups of pixels
-# with an integer index
-labeled_mask_array, nb_groups = ndimage.label(mask_array)
-
-# Writing the result to disk
-with rasterio.open('labeled_bare_soil.tif', 'w', **profile) as f:
- f.write(labeled_mask_array)
-
-```
-This way of exporting pyotb objects is more flexible that exporting to numpy, as the user gets the `profile` dictionary.
-If the georeference or pixel size is modified, the user can update the `profile` accordingly.
+- Easy use of OTB applications from python
+- Simplify common sophisticated I/O features of OTB
+- Lazy execution of in-memory pipelines with OTB streaming mechanism
+- Interoperable with popular python libraries (numpy, rasterio)
+- Extensible
+Documentation hosted at [pyotb.readthedocs.io](https://pyotb.readthedocs.io/).
-## Interaction with Tensorflow
+## Example
-We saw that numpy operations had some limitations. To bypass those limitations, it is possible to use some Tensorflow operations on pyotb objects.
+Building a simple pipeline with OTB applications
-
-You need a working installation of OTBTF >=3.0 for this and then the code is like this:
-
-```python
+```py
import pyotb
-def scalar_product(x1, x2):
- """This is a function composed of tensorflow operations."""
- import tensorflow as tf
- return tf.reduce_sum(tf.multiply(x1, x2), axis=-1)
+# RigidTransformResample application, with input parameters as dict
+resampled = pyotb.RigidTransformResample({
+ "in": "https://some.remote.data/input.tif", # Note: no /vsicurl/...
+ "interpolator": "linear",
+ "transform.type.id.scaley": 0.5,
+ "transform.type.id.scalex": 0.5
+})
-# Compute the scalar product
-res = pyotb.run_tf_function(scalar_product)('image1.tif', 'image2.tif') # magic: this is a pyotb object
-res.write('scalar_product.tif')
-```
+# OpticalCalibration, with automatic input parameters resolution
+calib = pyotb.OpticalCalibration(resampled)
-For some easy syntax, one can use `pyotb.run_tf_function` as a function decorator, such as:
-```python
-import pyotb
+# BandMath, with input parameters passed as kwargs
+ndvi = pyotb.BandMath(calib, exp="ndvi(im1b1, im1b4)")
-@pyotb.run_tf_function # The decorator enables the use of pyotb objects as inputs/output of the function
-def scalar_product(x1, x2):
- import tensorflow as tf
- return tf.reduce_sum(tf.multiply(x1, x2), axis=-1)
+# Pythonic slicing using lazy computation (no memory used)
+roi = ndvi[20:586, 9:572]
-res = scalar_product('image1.tif', 'image2.tif') # magic: this is a pyotb object
+# Pipeline execution
+# The actual computation happens here !
+roi.write("output.tif", pixel_type="float")
```
-Advantages :
-- The process supports streaming, hence the whole image is **not** loaded into memory
-- Can be integrated in OTB pipelines
-
-Limitations :
-- It is not possible to use the tensorflow python API inside a script where OTBTF is used because of compilation issues
-between Tensorflow and OTBTF, i.e. `import tensorflow` doesn't work in a script where OTBTF apps have been initialized
-
-
-## Some examples
-### Compute the mean of several rasters, taking into account NoData
-Let's consider we have at disposal 73 NDVI rasters for a year, where clouds have been masked with NoData (nodata value of -10 000 for example).
-
-Goal: compute the mean across time (keeping the spatial dimension) of the NDVIs, excluding cloudy pixels. Piece of code to achieve that:
+pyotb's objects also enable easy interoperability with
+[numpy](https://numpy.org/) and [rasterio](https://rasterio.readthedocs.io/):
```python
-import pyotb
+# Numpy and RasterIO style attributes
+print(roi.shape, roi.dtype, roi.transform)
+print(roi.metadata)
-nodata = -10000
-ndvis = [pyotb.Input(path) for path in ndvi_paths]
+# Other useful information
+print(roi.get_infos())
+print(roi.get_statistics())
-# For each pixel location, summing all valid NDVI values
-summed = sum([pyotb.where(ndvi != nodata, ndvi, 0) for ndvi in ndvis])
-
-# Printing the generated BandMath expression
-print(summed.exp) # this returns a very long exp: "0 + ((im1b1 != -10000) ? im1b1 : 0) + ((im2b1 != -10000) ? im2b1 : 0) + ... + ((im73b1 != -10000) ? im73b1 : 0)"
-
-# For each pixel location, getting the count of valid pixels
-count = sum([pyotb.where(ndvi == nodata, 0, 1) for ndvi in ndvis])
-
-mean = summed / count # BandMath exp of this is very long: "(0 + ((im1b1 != -10000) ? im1b1 : 0) + ... + ((im73b1 != -10000) ? im73b1 : 0)) / (0 + ((im1b1 == -10000) ? 0 : 1) + ... + ((im73b1 == -10000) ? 0 : 1))"
-mean.write('ndvi_annual_mean.tif')
+array = roi.to_numpy()
+array, profile = roi.to_rasterio()
```
-Note that no actual computation is executed before the last line where the result is written to disk.
-
-### Process raw Pleiades data
-This is a common case of Pleiades data preprocessing : optical calibration -> orthorectification -> pansharpening
+## Contributing
-```python
-import pyotb
-srtm = '/media/data/raster/nasa/srtm_30m'
-geoid = '/media/data/geoid/egm96.grd'
-
-pan = pyotb.OpticalCalibration('IMG_PHR1A_P_001/DIM_PHR1A_P_201509011347379_SEN_1791374101-001.XML', level='toa')
-ms = pyotb.OpticalCalibration('IMG_PHR1A_MS_002/DIM_PHR1A_MS_201509011347379_SEN_1791374101-002.XML', level='toa')
-
-pan_ortho = pyotb.OrthoRectification({'io.in': pan, 'elev.dem': srtm, 'elev.geoid': geoid})
-ms_ortho = pyotb.OrthoRectification({'io.in': ms, 'elev.dem': srtm, 'elev.geoid': geoid})
-
-pxs = pyotb.BundleToPerfectSensor(inp=pan_ortho, inxs=ms_ortho, method='bayes', mode="default")
-
-# Here we trigger every app in the pipeline and the process is blocked until result is written to disk
-pxs.write('pxs_image.tif', pixel_type='uint16', filename_extension='?gdal:co:COMPRESS=DEFLATE&gdal:co:PREDICTOR=2')
-```
+Contributions are welcome on [Github](https://github.com/orfeotoolbox/pyotb) or the source repository hosted on the Orfeo ToolBox [GitLab](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb).
diff --git a/doc/MISC.md b/doc/MISC.md
index 8da0eea..2a92b03 100644
--- a/doc/MISC.md
+++ b/doc/MISC.md
@@ -1,4 +1,5 @@
## Miscellaneous: Work with images with different footprints / resolutions
+
OrfeoToolBox provides a handy `Superimpose` application that enables the projection of an image into the geometry of another one.
In pyotb, a function has been created to handle more than 2 images.
@@ -17,6 +18,7 @@ print(s2_image.shape) # (286, 195, 4)
print(vhr_image.shape) # (2048, 2048, 3)
print(labels.shape) # (1528, 1360, 1)
```
+
Our goal is to obtain all images at the same footprint, same resolution and same shape.
Let's consider we want the intersection of all footprints and the same resolution as `labels` image.
@@ -25,7 +27,8 @@ Let's consider we want the intersection of all footprints and the same resolutio
Here is the final result :
-The piece of code to achieve this :
+The piece of code to achieve this :
+
```python
s2_image, vhr_image, labels = pyotb.define_processing_area(s2_image, vhr_image, labels, window_rule='intersection',
pixel_size_rule='same_as_input',
diff --git a/doc/comparison_otb.md b/doc/comparison_otb.md
index 612eab6..a733678 100644
--- a/doc/comparison_otb.md
+++ b/doc/comparison_otb.md
@@ -1,7 +1,9 @@
## Comparison between otbApplication and pyotb
### Single application execution
+
Using OTB, the code would be like :
+
```python
import otbApplication
@@ -12,66 +14,123 @@ resampled.SetParameterString('interpolator', 'linear')
resampled.SetParameterFloat('transform.type.id.scalex', 0.5)
resampled.SetParameterFloat('transform.type.id.scaley', 0.5)
resampled.SetParameterString('out', 'output.tif')
-resampled.SetParameterOutputImagePixelType('out', otbApplication.ImagePixelType_uint16)
+resampled.SetParameterOutputImagePixelType(
+ 'out', otbApplication.ImagePixelType_uint16
+)
+
resampled.ExecuteAndWriteOutput()
```
Using pyotb:
+
```python
import pyotb
-resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear',
- 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5})
+
+resampled = pyotb.RigidTransformResample({
+ 'in': 'my_image.tif',
+ 'interpolator': 'linear',
+ 'transform.type.id.scaley': 0.5,
+ 'transform.type.id.scalex': 0.5
+})
+
resampled.write('output.tif', pixel_type='uint16')
```
### In-memory connections
-Using OTB :
+<table>
+<tr>
+<th> OTB </th>
+<th> pyotb </th>
+</tr>
+<tr>
+<td>
+
```python
import otbApplication
-resampled = otbApplication.Registry.CreateApplication('RigidTransformResample')
-resampled.SetParameterString('in', 'my_image.tif')
-resampled.SetParameterString('interpolator', 'linear')
-resampled.SetParameterFloat('transform.type.id.scalex', 0.5)
-resampled.SetParameterFloat('transform.type.id.scaley', 0.5)
-resampled.Execute()
-
-calibrated = otbApplication.Registry.CreateApplication('OpticalCalibration')
-calibrated.ConnectImage('in', resampled, 'out')
-calibrated.SetParameterString('level', 'toa')
-calibrated.Execute()
-
-dilated = otbApplication.Registry.CreateApplication('BinaryMorphologicalOperation')
-dilated.ConnectImage('in', calibrated, 'out')
-dilated.SetParameterString("filter", 'dilate')
-dilated.SetParameterString("structype", 'ball')
-dilated.SetParameterInt("xradius", 3)
-dilated.SetParameterInt("yradius", 3)
-dilated.SetParameterString('out', 'output.tif')
-dilated.SetParameterOutputImagePixelType('out', otbApplication.ImagePixelType_uint16)
-dilated.ExecuteAndWriteOutput()
+app1 = otbApplication.Registry.CreateApplication(
+ 'RigidTransformResample'
+)
+app1.SetParameterString('in', 'my_image.tif')
+app1.SetParameterString('interpolator', 'linear')
+app1.SetParameterFloat(
+ 'transform.type.id.scalex',
+ 0.5
+)
+app1.SetParameterFloat(
+ 'transform.type.id.scaley',
+ 0.5
+)
+app1.Execute()
+
+app2 = otbApplication.Registry.CreateApplication(
+ 'OpticalCalibration'
+)
+app2.ConnectImage('in', app1, 'out')
+app2.SetParameterString('level', 'toa')
+app2.Execute()
+
+app3 = otbApplication.Registry.CreateApplication(
+ 'BinaryMorphologicalOperation'
+)
+app3.ConnectImage('in', app2, 'out')
+app3.SetParameterString('filter', 'dilate')
+app3.SetParameterString('structype', 'ball')
+app3.SetParameterInt('xradius', 3)
+app3.SetParameterInt('yradius', 3)
+app3.SetParameterString('out', 'output.tif')
+app3.SetParameterOutputImagePixelType(
+ 'out',
+ otbApplication.ImagePixelType_uint16
+)
+app3.ExecuteAndWriteOutput()
```
-Using pyotb:
+</td>
+<td>
+
```python
import pyotb
-resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear',
- 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5})
-
-calibrated = pyotb.OpticalCalibration({'in': resampled, 'level': 'toa'})
-
-dilated = pyotb.BinaryMorphologicalOperation({'in': calibrated, 'out': 'output.tif', 'filter': 'dilate',
- 'structype': 'ball', 'xradius': 3, 'yradius': 3})
-dilated.write('result.tif', pixel_type='uint16')
+app1 = pyotb.RigidTransformResample({
+ 'in': 'my_image.tif',
+ 'interpolator': 'linear',
+ 'transform.type.id.scaley': 0.5,
+ 'transform.type.id.scalex': 0.5
+})
+
+app2 = pyotb.OpticalCalibration({
+ 'in': app1,
+ 'level': 'toa'
+})
+
+app3 = pyotb.BinaryMorphologicalOperation({
+ 'in': app2,
+ 'out': 'output.tif',
+ 'filter': 'dilate',
+ 'structype': 'ball',
+ 'xradius': 3,
+ 'yradius': 3
+})
+
+app3.write(
+ 'result.tif',
+ pixel_type='uint16'
+)
```
+</td>
+</tr>
+</table>
+
### Arithmetic operations
+
Every pyotb object supports arithmetic operations, such as addition, subtraction, comparison...
Consider an example where we want to perform the arithmetic operation `image1 * image2 - 2*image3`
Using OTB, the following code works for 3-bands images :
+
```python
import otbApplication
@@ -85,6 +144,7 @@ bmx.ExecuteAndWriteOutput()
```
With pyotb, the following works with images of any number of bands :
+
```python
import pyotb
@@ -98,6 +158,7 @@ res.write('output.tif', pixel_type='uint8')
### Slicing
Using OTB, for selection bands or ROI, the code looks like:
+
```python
import otbApplication
@@ -119,7 +180,7 @@ extracted.SetParameterFloat('mode.extent.lry', 999)
extracted.Execute()
```
-Instead, using pyotb:
+Instead, using pyotb:
```python
import pyotb
@@ -129,4 +190,4 @@ inp = pyotb.Input('my_image.tif')
extracted = inp[:, :, :3] # selecting first 3 bands
extracted = inp[:1000, :1000] # selecting 1000x1000 subset
-```
\ No newline at end of file
+```
diff --git a/doc/features.md b/doc/features.md
index 7bc78ce..cc2c39e 100644
--- a/doc/features.md
+++ b/doc/features.md
@@ -1,8 +1,10 @@
## Arithmetic operations
+
Every pyotb object supports arithmetic operations, such as addition, subtraction, comparison...
Consider an example where we want to compute a vegeteation mask from NDVI, i.e. the arithmetic operation `(nir - red) / (nir + red) > 0.3`
With pyotb, one can simply do :
+
```python
import pyotb
@@ -15,6 +17,7 @@ res.write('vegetation_mask.tif', pixel_type='uint8')
```
## Slicing
+
pyotb objects support slicing in a Python fashion :
```python
@@ -30,6 +33,7 @@ inp[:100, :100].write('my_image_roi.tif') # write cropped image to disk
```
## Shape attributes
+
You can access the shape of any in-memory pyotb object.
```python
@@ -39,6 +43,3 @@ import pyotb
inp = pyotb.Input('my_image.tif')
print(inp[:1000, :500].shape) # (1000, 500, 4)
```
-
-
-
diff --git a/doc/installation.md b/doc/installation.md
index dd7b729..de5c0ae 100644
--- a/doc/installation.md
+++ b/doc/installation.md
@@ -1,11 +1,31 @@
+## Prerequisites
-## Requirements
-- Python>=3.5 and NumPy
-- Orfeo ToolBox python API (instructions available [on the official website](https://www.orfeo-toolbox.org/CookBook/Installation.html))
+Requirements:
+
+- Python >= 3.7 and NumPy
+- Orfeo ToolBox binaries (follow these
+ [instructions](https://www.orfeo-toolbox.org/CookBook/Installation.html))
+- Orfeo ToolBox python binding (follow these
+ [instructions](https://www.orfeo-toolbox.org/CookBook/Installation.html))
+
+## Install with pip
-## Installation
```bash
pip install pyotb --upgrade
```
-For Python>=3.6, latest version available is pyotb 1.5.0. For Python 3.5, latest version available is pyotb 1.2.2
+For development, use the following:
+
+```bash
+git clone https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb
+cd pyotb
+pip install -e ".[dev]"
+```
+
+## Old versions
+
+If you need compatibility with python3.6, install `pyotb<2.0` and for
+ python3.5 use `pyotb==1.2.2`.
+
+For Python>=3.6, latest version available is pyotb 1.5.0.
+For Python 3.5, latest version available is pyotb 1.2.2
diff --git a/doc/interaction.md b/doc/interaction.md
index d44ca68..217dbaa 100644
--- a/doc/interaction.md
+++ b/doc/interaction.md
@@ -1,16 +1,15 @@
## Export to Numpy
pyotb objects can be exported to numpy array.
+
```python
import pyotb
import numpy as np
calibrated = pyotb.OpticalCalibration('image.tif', level='toa') # this is a pyotb object
arr = np.asarray(calibrated) # same as calibrated.to_numpy()
-
```
-
## Interaction with Numpy
pyotb objects can be transparently used in numpy functions.
@@ -31,13 +30,14 @@ noisy_image = inp + white_noise # magic: this is a pyotb object that has the sa
# `np.add(inp, white_noise)` would have worked the same
noisy_image.write('image_plus_noise.tif')
```
-Limitations :
+
+Limitations :
- The whole image is loaded into memory
- The georeference can not be modified. Thus, numpy operations can not change the image or pixel size
-
## Export to rasterio
+
pyotb objects can also be exported in a format that is usable by rasterio.
For example:
@@ -62,17 +62,15 @@ labeled_mask_array, nb_groups = ndimage.label(mask_array)
# Writing the result to disk
with rasterio.open('labeled_bare_soil.tif', 'w', **profile) as f:
f.write(labeled_mask_array)
-
```
+
This way of exporting pyotb objects is more flexible that exporting to numpy, as the user gets the `profile` dictionary.
If the georeference or pixel size is modified, the user can update the `profile` accordingly.
-
## Interaction with Tensorflow
We saw that numpy operations had some limitations. To bypass those limitations, it is possible to use some Tensorflow operations on pyotb objects.
-
You need a working installation of OTBTF >=3.0 for this and then the code is like this:
```python
@@ -89,6 +87,7 @@ res.write('scalar_product.tif')
```
For some easy syntax, one can use `pyotb.run_tf_function` as a function decorator, such as:
+
```python
import pyotb
@@ -107,5 +106,5 @@ Advantages :
Limitations :
-- It is not possible to use the tensorflow python API inside a script where OTBTF is used because of compilation issues
-between Tensorflow and OTBTF, i.e. `import tensorflow` doesn't work in a script where OTBTF apps have been initialized
+- It is not possible to use the tensorflow python API inside a script where OTBTF is used because of compilation issues
+ between Tensorflow and OTBTF, i.e. `import tensorflow` doesn't work in a script where OTBTF apps have been initialized
diff --git a/doc/managing_loggers.md b/doc/managing_loggers.md
index f329d5e..31c960b 100644
--- a/doc/managing_loggers.md
+++ b/doc/managing_loggers.md
@@ -9,6 +9,7 @@ If none of those two variables is set, the logger level will be set to 'INFO'.
Available levels are : DEBUG, INFO, WARNING, ERROR, CRITICAL
You may also change the logger level after import (for pyotb only) with the function `set_logger_level`.
+
```python
import pyotb
pyotb.set_logger_level('DEBUG')
@@ -20,6 +21,7 @@ One useful trick is to redirect these logs to a file. This can be done using the
## Named applications in logs
It is possible to change an app name in order to track it easily in the logs :
+
```python
import os
os.environ['PYOTB_LOGGER_LEVEL'] = 'DEBUG'
@@ -29,8 +31,9 @@ bm = pyotb.BandMath(['image.tif'], exp='im1b1 * 100')
bm.name = 'CustomBandMathApp'
bm.execute()
```
+
```text
2022-06-14 14:22:38 (DEBUG) [pyOTB] CustomBandMathApp: run execute() with parameters={'exp': 'im1b1 * 100', 'il': ['/home/vidlb/Téléchargements/test_4b.tif']}
2022-06-14 14:22:38 (INFO) BandMath: Image #1 has 4 components
2022-06-14 14:22:38 (DEBUG) [pyOTB] CustomBandMathApp: execution succeeded
-```
\ No newline at end of file
+```
diff --git a/doc/otb_versions.md b/doc/otb_versions.md
index 6b06c28..7eeb5cf 100644
--- a/doc/otb_versions.md
+++ b/doc/otb_versions.md
@@ -1,21 +1,25 @@
## System with multiple OTB versions
-If you want to quickly switch between OTB versions, or override the default system version, you may use the `OTB_ROOT` env variable :
+If you want to quickly switch between OTB versions, or override the default system version, you may use the `OTB_ROOT` env variable :
+
```python
import os
# This is equivalent to "[set/export] OTB_ROOT=/opt/otb" before launching python
os.environ['OTB_ROOT'] = '/opt/otb'
import pyotb
```
+
```text
2022-06-14 13:59:03 (INFO) [pyOTB] Preparing environment for OTB in /opt/otb
2022-06-14 13:59:04 (INFO) [pyOTB] Successfully loaded 126 OTB applications
```
-If you try to import pyotb without having set environment, it will try to find any OTB version installed on your system:
+If you try to import pyotb without having set environment, it will try to find any OTB version installed on your system:
+
```python
import pyotb
```
+
```text
2022-06-14 13:55:41 (INFO) [pyOTB] Failed to import OTB. Searching for it...
2022-06-14 13:55:41 (INFO) [pyOTB] Found /opt/otb/lib/otb/
@@ -24,21 +28,26 @@ import pyotb
2022-06-14 13:55:43 (INFO) [pyOTB] Preparing environment for OTB in /home/otbuser/Applications/OTB-8.0.1-Linux64
2022-06-14 13:55:44 (INFO) [pyOTB] Successfully loaded 117 OTB applications
```
+
Here is the path precedence for this automatic env configuration :
+
```text
OTB_ROOT env variable > python bindings directory
OR search for releases installations : HOME
OR (for linux) : /opt/otbtf > /opt/otb > /usr/local > /usr
OR (for windows) : C:/Program Files
```
+
N.B. : in case `otbApplication` is found in `PYTHONPATH` (and if `OTB_ROOT` was not set), the OTB which the python API is linked to will be used.
## Fresh OTB installation
If you've just installed OTB binaries in a Linux environment, you may encounter an error at first import, pyotb will help you fix it :
+
```python
import pyotb
```
+
```text
2022-06-14 14:00:34 (INFO) [pyOTB] Preparing environment for OTB in /home/otbuser/Applications/OTB-8.0.1-Linux64
2022-07-07 16:56:04 (CRITICAL) [pyOTB] An error occurred while importing OTB Python API
diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md
index e6b0b4c..e96e3e7 100644
--- a/doc/troubleshooting.md
+++ b/doc/troubleshooting.md
@@ -1,4 +1,4 @@
-## Troubleshooting: Known limitations
+## Troubleshooting: known limitations
### Failure of intermediate writing
@@ -7,8 +7,9 @@ the writings are requested. Some examples can be found below:
#### Example of failures involving slicing
-For some applications (non-exhaustive know list: OpticalCalibration, DynamicConvert, BandMath), we can face unexpected
-failures when using channels slicing
+For some applications (non-exhaustive know list: OpticalCalibration, DynamicConvert, BandMath), we can face unexpected
+ failures when using channels slicing
+
```python
import pyotb
@@ -28,6 +29,7 @@ one_band.write('one_band.tif') # Failure here
```
When writing is triggered right after the application declaration, no problem occurs:
+
```python
import pyotb
@@ -51,7 +53,6 @@ inp.write('stretched.tif')
one_band.write('one_band.tif')
```
-
#### Example of failures involving arithmetic operation
One can meet errors when using arithmetic operations at the end of a pipeline when DynamicConvert, BandMath or
--
GitLab
From ea02e5331c7dc0b379a527afefa37c3d773fa6b4 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 1 Jun 2023 17:55:11 +0000
Subject: [PATCH 193/399] Format code with black
---
pyotb/__init__.py | 13 +-
pyotb/apps.py | 49 ++++--
pyotb/core.py | 372 ++++++++++++++++++++++++++++++++++-----------
pyotb/functions.py | 266 ++++++++++++++++++++++----------
pyotb/helpers.py | 77 +++++++---
pyproject.toml | 17 ++-
6 files changed, 576 insertions(+), 218 deletions(-)
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index d4efd39..c9fe258 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -4,19 +4,12 @@ __version__ = "2.0.0"
from .helpers import logger, set_logger_level
from .apps import *
-from .core import (
- App,
- Input,
- Output,
- get_nbchannels,
- get_pixel_type,
- summarize
-)
+from .core import App, Input, Output, get_nbchannels, get_pixel_type, summarize
from .functions import ( # pylint: disable=redefined-builtin
all,
any,
- where,
clip,
+ define_processing_area,
run_tf_function,
- define_processing_area
+ where,
)
diff --git a/pyotb/apps.py b/pyotb/apps.py
index bad5089..6f54b27 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -1,12 +1,14 @@
# -*- coding: utf-8 -*-
"""Search for OTB (set env if necessary), subclass core.App for each available application."""
from __future__ import annotations
+
import os
-import sys
import subprocess
+import sys
from pathlib import Path
import otbApplication as otb # pylint: disable=import-error
+
from .core import App
from .helpers import logger
@@ -31,36 +33,48 @@ def get_available_applications(as_subprocess: bool = False) -> list[str]:
if "PYTHONPATH" not in env:
env["PYTHONPATH"] = ""
env["PYTHONPATH"] += ":" + str(Path(otb.__file__).parent)
- env["OTB_LOGGER_LEVEL"] = "CRITICAL" # in order to suppress warnings while listing applications
+ env[
+ "OTB_LOGGER_LEVEL"
+ ] = "CRITICAL" # in order to suppress warnings while listing applications
pycmd = "import otbApplication; print(otbApplication.Registry.GetAvailableApplications())"
cmd_args = [sys.executable, "-c", pycmd]
try:
params = {"env": env, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE}
with subprocess.Popen(cmd_args, **params) as process:
- logger.debug('Exec "%s \'%s\'"', ' '.join(cmd_args[:-1]), pycmd)
+ logger.debug("Exec \"%s '%s'\"", " ".join(cmd_args[:-1]), pycmd)
stdout, stderr = process.communicate()
stdout, stderr = stdout.decode(), stderr.decode()
# ast.literal_eval is secure and will raise more handy Exceptions than eval
from ast import literal_eval # pylint: disable=import-outside-toplevel
+
app_list = literal_eval(stdout.strip())
assert isinstance(app_list, (tuple, list))
except subprocess.SubprocessError:
logger.debug("Failed to call subprocess")
except (ValueError, SyntaxError, AssertionError):
- logger.debug("Failed to decode output or convert to tuple:\nstdout=%s\nstderr=%s", stdout, stderr)
+ logger.debug(
+ "Failed to decode output or convert to tuple:\nstdout=%s\nstderr=%s",
+ stdout,
+ stderr,
+ )
if not app_list:
- logger.debug("Failed to list applications in an independent process. Falling back to local python import")
+ logger.debug(
+ "Failed to list applications in an independent process. Falling back to local python import"
+ )
# Find applications using the normal way
if not app_list:
app_list = otb.Registry.GetAvailableApplications()
if not app_list:
- raise SystemExit("Unable to load applications. Set env variable OTB_APPLICATION_PATH and try again.")
+ raise SystemExit(
+ "Unable to load applications. Set env variable OTB_APPLICATION_PATH and try again."
+ )
logger.info("Successfully loaded %s OTB applications", len(app_list))
return app_list
class OTBTFApp(App):
"""Helper for OTBTF."""
+
@staticmethod
def set_nb_sources(*args, n_sources: int = None):
"""Set the number of sources of TensorflowModelServe. Can be either user-defined or deduced from the args.
@@ -72,13 +86,17 @@ class OTBTFApp(App):
"""
if n_sources:
- os.environ['OTB_TF_NSOURCES'] = str(int(n_sources))
+ os.environ["OTB_TF_NSOURCES"] = str(int(n_sources))
else:
# Retrieving the number of `source#.il` parameters
- params_dic = {k: v for arg in args if isinstance(arg, dict) for k, v in arg.items()}
- n_sources = len([k for k in params_dic if 'source' in k and k.endswith('.il')])
+ params_dic = {
+ k: v for arg in args if isinstance(arg, dict) for k, v in arg.items()
+ }
+ n_sources = len(
+ [k for k in params_dic if "source" in k and k.endswith(".il")]
+ )
if n_sources >= 1:
- os.environ['OTB_TF_NSOURCES'] = str(n_sources)
+ os.environ["OTB_TF_NSOURCES"] = str(n_sources)
def __init__(self, name: str, *args, n_sources: int = None, **kwargs):
"""Constructor for an OTBTFApp object.
@@ -98,17 +116,22 @@ class OTBTFApp(App):
AVAILABLE_APPLICATIONS = get_available_applications(as_subprocess=True)
# This is to enable aliases of Apps, i.e. using apps like `pyotb.AppName(...)` instead of `pyotb.App("AppName", ...)`
-_CODE_TEMPLATE = """
+_CODE_TEMPLATE = (
+ """
class {name}(App):
- """ """
+ """
+ """
def __init__(self, *args, **kwargs):
super().__init__('{name}', *args, **kwargs)
"""
+)
for _app in AVAILABLE_APPLICATIONS:
# Customize the behavior for some OTBTF applications. `OTB_TF_NSOURCES` is now handled by pyotb
if _app in ("PatchesExtraction", "TensorflowModelTrain", "TensorflowModelServe"):
- exec(_CODE_TEMPLATE.format(name=_app).replace("(App)", "(OTBTFApp)")) # pylint: disable=exec-used
+ exec( # pylint: disable=exec-used
+ _CODE_TEMPLATE.format(name=_app).replace("(App)", "(OTBTFApp)")
+ )
# Default behavior for any OTB application
else:
exec(_CODE_TEMPLATE.format(name=_app)) # pylint: disable=exec-used
diff --git a/pyotb/core.py b/pyotb/core.py
index 903f71f..dfe2598 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -2,11 +2,11 @@
"""This module is the core of pyotb."""
from __future__ import annotations
+from abc import ABC, abstractmethod
from ast import literal_eval
from pathlib import Path
from time import perf_counter
from typing import Any
-from abc import ABC, abstractmethod
import numpy as np
import otbApplication as otb # pylint: disable=import-error
@@ -16,6 +16,7 @@ from .helpers import logger
class OTBObject(ABC):
"""Abstraction of an image object."""
+
@property
@abstractmethod
def name(self) -> str:
@@ -85,7 +86,9 @@ class OTBObject(ABC):
"""Return a dict output of ComputeImagesStatistics for the first image output."""
return App("ComputeImagesStatistics", self, quiet=True).data
- def get_values_at_coords(self, row: int, col: int, bands: int = None) -> list[int | float] | int | float:
+ def get_values_at_coords(
+ self, row: int, col: int, bands: int = None
+ ) -> list[int | float] | int | float:
"""Get pixel value(s) at a given YX coordinates.
Args:
@@ -107,7 +110,9 @@ class OTBObject(ABC):
elif isinstance(bands, slice):
channels = self.channels_list_from_slice(bands)
elif not isinstance(bands, list):
- raise TypeError(f"{self.name}: type '{type(bands)}' cannot be interpreted as a valid slicing")
+ raise TypeError(
+ f"{self.name}: type '{type(bands)}' cannot be interpreted as a valid slicing"
+ )
if channels:
app.app.Execute()
app.set_parameters({"cl": [f"Channel{n + 1}" for n in channels]})
@@ -130,9 +135,13 @@ class OTBObject(ABC):
return list(range(0, stop, step))
if start is None and stop is None:
return list(range(0, nb_channels, step))
- raise ValueError(f"{self.name}: '{bands}' cannot be interpreted as valid slicing.")
+ raise ValueError(
+ f"{self.name}: '{bands}' cannot be interpreted as valid slicing."
+ )
- def export(self, key: str = None, preserve_dtype: bool = True) -> dict[str, dict[str, np.ndarray]]:
+ def export(
+ self, key: str = None, preserve_dtype: bool = True
+ ) -> dict[str, dict[str, np.ndarray]]:
"""Export a specific output image as numpy array and store it in object exports_dic.
Args:
@@ -149,10 +158,14 @@ class OTBObject(ABC):
if key not in self.exports_dic:
self.exports_dic[key] = self.app.ExportImage(key)
if preserve_dtype:
- self.exports_dic[key]["array"] = self.exports_dic[key]["array"].astype(self.dtype)
+ self.exports_dic[key]["array"] = self.exports_dic[key]["array"].astype(
+ self.dtype
+ )
return self.exports_dic[key]
- def to_numpy(self, key: str = None, preserve_dtype: bool = True, copy: bool = False) -> np.ndarray:
+ def to_numpy(
+ self, key: str = None, preserve_dtype: bool = True, copy: bool = False
+ ) -> np.ndarray:
"""Export a pyotb object to numpy array.
Args:
@@ -370,12 +383,18 @@ class OTBObject(ABC):
row, col = key[0], key[1]
if isinstance(row, int) and isinstance(col, int):
if row < 0 or col < 0:
- raise ValueError(f"{self.name} cannot read pixel value at negative coordinates ({row}, {col})")
+ raise ValueError(
+ f"{self.name} cannot read pixel value at negative coordinates ({row}, {col})"
+ )
channels = key[2] if len(key) == 3 else None
return self.get_values_at_coords(row, col, channels)
# Slicing
- if not isinstance(key, tuple) or (isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)):
- raise ValueError(f'"{key}" cannot be interpreted as valid slicing. Slicing should be 2D or 3D.')
+ if not isinstance(key, tuple) or (
+ isinstance(key, tuple) and (len(key) < 2 or len(key) > 3)
+ ):
+ raise ValueError(
+ f'"{key}" cannot be interpreted as valid slicing. Slicing should be 2D or 3D.'
+ )
if isinstance(key, tuple) and len(key) == 2:
# Adding a 3rd dimension
key = key + (slice(None, None, None),)
@@ -392,7 +411,7 @@ class App(OTBObject):
INPUT_IMAGE_TYPES = [
# Images only
otb.ParameterType_InputImage,
- otb.ParameterType_InputImageList
+ otb.ParameterType_InputImageList,
]
INPUT_PARAM_TYPES = INPUT_IMAGE_TYPES + [
# Vectors
@@ -426,7 +445,15 @@ class App(OTBObject):
otb.ParameterType_InputFilenameList,
]
- def __init__(self, appname: str, *args, frozen: bool = False, quiet: bool = False, name: str = "", **kwargs):
+ def __init__(
+ self,
+ appname: str,
+ *args,
+ frozen: bool = False,
+ quiet: bool = False,
+ name: str = "",
+ **kwargs,
+ ):
"""Common constructor for OTB applications. Handles in-memory connection between apps.
Args:
@@ -445,7 +472,11 @@ class App(OTBObject):
"""
# Attributes and data structures used by properties
- create = otb.Registry.CreateApplicationWithoutLogger if quiet else otb.Registry.CreateApplication
+ create = (
+ otb.Registry.CreateApplicationWithoutLogger
+ if quiet
+ else otb.Registry.CreateApplication
+ )
self._app = create(appname)
self._name = name or appname
self._exports_dic = {}
@@ -455,14 +486,25 @@ class App(OTBObject):
self.quiet, self.frozen = quiet, frozen
# Param keys and types
self.parameters_keys = tuple(self.app.GetParametersKeys())
- self._all_param_types = {k: self.app.GetParameterType(k) for k in self.parameters_keys}
- types = (otb.ParameterType_OutputImage, otb.ParameterType_OutputVectorData, otb.ParameterType_OutputFilename)
- self._out_param_types = {k: v for k, v in self._all_param_types.items() if v in types}
+ self._all_param_types = {
+ k: self.app.GetParameterType(k) for k in self.parameters_keys
+ }
+ types = (
+ otb.ParameterType_OutputImage,
+ otb.ParameterType_OutputVectorData,
+ otb.ParameterType_OutputFilename,
+ )
+ self._out_param_types = {
+ k: v for k, v in self._all_param_types.items() if v in types
+ }
# Init, execute and write (auto flush only when output param was provided)
if args or kwargs:
self.set_parameters(*args, **kwargs)
# Create Output image objects
- for key in filter(lambda k: self._out_param_types[k] == otb.ParameterType_OutputImage, self._out_param_types):
+ for key in filter(
+ lambda k: self._out_param_types[k] == otb.ParameterType_OutputImage,
+ self._out_param_types,
+ ):
self.outputs[key] = Output(self, key, self._settings.get(key))
if not self.frozen:
self.execute()
@@ -492,9 +534,7 @@ class App(OTBObject):
def __is_one_of_types(self, key: str, param_types: list[int]) -> bool:
"""Helper to factor is_input and is_output."""
if key not in self._all_param_types:
- raise KeyError(
- f"key {key} not found in the application parameters types"
- )
+ raise KeyError(f"key {key} not found in the application parameters types")
return self._all_param_types[key] in param_types
def is_input(self, key: str) -> bool:
@@ -507,9 +547,7 @@ class App(OTBObject):
True if the parameter is an input, else False
"""
- return self.__is_one_of_types(
- key=key, param_types=self.INPUT_PARAM_TYPES
- )
+ return self.__is_one_of_types(key=key, param_types=self.INPUT_PARAM_TYPES)
def is_output(self, key: str) -> bool:
"""Returns True if the key is an output.
@@ -521,9 +559,7 @@ class App(OTBObject):
True if the parameter is an output, else False
"""
- return self.__is_one_of_types(
- key=key, param_types=self.OUTPUT_PARAM_TYPES
- )
+ return self.__is_one_of_types(key=key, param_types=self.OUTPUT_PARAM_TYPES)
def is_key_list(self, key: str) -> bool:
"""Check if a parameter key is an input parameter list."""
@@ -541,7 +577,9 @@ class App(OTBObject):
for key, value in sorted(self._all_param_types.items()):
if value == param_type:
return key
- raise TypeError(f"{self.name}: could not find any parameter key matching the provided types")
+ raise TypeError(
+ f"{self.name}: could not find any parameter key matching the provided types"
+ )
@property
def input_key(self) -> str:
@@ -598,7 +636,11 @@ class App(OTBObject):
# When the parameter expects a list, if needed, change the value to list
if self.is_key_list(key) and not isinstance(obj, (list, tuple)):
obj = [obj]
- logger.info('%s: argument for parameter "%s" was converted to list', self.name, key)
+ logger.info(
+ '%s: argument for parameter "%s" was converted to list',
+ self.name,
+ key,
+ )
try:
if self.is_input(key):
if self.is_key_images_list(key):
@@ -630,19 +672,28 @@ class App(OTBObject):
if not dtype:
param = self._settings.get(self.input_image_key)
if not param:
- logger.warning("%s: could not propagate pixel type from inputs to output", self.name)
+ logger.warning(
+ "%s: could not propagate pixel type from inputs to output",
+ self.name,
+ )
return
if isinstance(param, (list, tuple)):
param = param[0] # first image in "il"
try:
dtype = get_pixel_type(param)
except (TypeError, RuntimeError):
- logger.warning('%s: unable to identify pixel type of key "%s"', self.name, param)
+ logger.warning(
+ '%s: unable to identify pixel type of key "%s"', self.name, param
+ )
return
if target_key:
keys = [target_key]
else:
- keys = [k for k, v in self._out_param_types.items() if v == otb.ParameterType_OutputImage]
+ keys = [
+ k
+ for k, v in self._out_param_types.items()
+ if v == otb.ParameterType_OutputImage
+ ]
for key in keys:
self.app.SetParameterOutputImagePixelType(key, dtype)
@@ -653,7 +704,9 @@ class App(OTBObject):
try:
self.app.Execute()
except (RuntimeError, FileNotFoundError) as e:
- raise RuntimeError(f"{self.name}: error during during app execution ({e}") from e
+ raise RuntimeError(
+ f"{self.name}: error during during app execution ({e}"
+ ) from e
self.frozen = False
self._time_end = perf_counter()
logger.debug("%s: execution ended", self.name)
@@ -665,13 +718,22 @@ class App(OTBObject):
logger.debug("%s: flushing data to disk", self.name)
self.app.WriteOutput()
except RuntimeError:
- logger.debug("%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput", self.name)
+ logger.debug(
+ "%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput",
+ self.name,
+ )
self._time_start = perf_counter()
self.app.ExecuteAndWriteOutput()
self._time_end = perf_counter()
- def write(self, path: str | Path | dict[str, str] = None, ext_fname: str = "",
- pixel_type: dict[str, str] | str = None, preserve_dtype: bool = False, **kwargs) -> bool:
+ def write(
+ self,
+ path: str | Path | dict[str, str] = None,
+ ext_fname: str = "",
+ pixel_type: dict[str, str] | str = None,
+ preserve_dtype: bool = False,
+ **kwargs,
+ ) -> bool:
"""Set output pixel type and write the output raster files.
Args:
@@ -697,24 +759,35 @@ class App(OTBObject):
if isinstance(path, dict):
kwargs.update(path)
elif isinstance(path, str) and kwargs:
- logger.warning('%s: keyword arguments specified, ignoring argument "%s"', self.name, path)
+ logger.warning(
+ '%s: keyword arguments specified, ignoring argument "%s"',
+ self.name,
+ path,
+ )
elif isinstance(path, (str, Path)) and self.output_key:
kwargs.update({self.output_key: str(path)})
elif path is not None:
raise TypeError(f"{self.name}: unsupported filepath type ({type(path)})")
if not (kwargs or any(k in self._settings for k in self._out_param_types)):
- raise KeyError(f"{self.name}: at least one filepath is required, if not provided during App init")
+ raise KeyError(
+ f"{self.name}: at least one filepath is required, if not provided during App init"
+ )
parameters = kwargs.copy()
# Append filename extension to filenames
if ext_fname:
- logger.debug("%s: using extended filename for outputs: %s", self.name, ext_fname)
+ logger.debug(
+ "%s: using extended filename for outputs: %s", self.name, ext_fname
+ )
if not ext_fname.startswith("?"):
ext_fname = "?&" + ext_fname
elif not ext_fname.startswith("?&"):
ext_fname = "?&" + ext_fname[1:]
for key, value in kwargs.items():
- if self._out_param_types[key] == otb.ParameterType_OutputImage and "?" not in value:
+ if (
+ self._out_param_types[key] == otb.ParameterType_OutputImage
+ and "?" not in value
+ ):
parameters[key] = value + ext_fname
# Manage output pixel types
data_types = {}
@@ -722,12 +795,16 @@ class App(OTBObject):
if isinstance(pixel_type, str):
dtype = parse_pixel_type(pixel_type)
type_name = self.app.ConvertPixelTypeToNumpy(dtype)
- logger.debug('%s: output(s) will be written with type "%s"', self.name, type_name)
+ logger.debug(
+ '%s: output(s) will be written with type "%s"', self.name, type_name
+ )
for key in parameters:
if self._out_param_types[key] == otb.ParameterType_OutputImage:
data_types[key] = dtype
elif isinstance(pixel_type, dict):
- data_types = {key: parse_pixel_type(dtype) for key, dtype in pixel_type.items()}
+ data_types = {
+ key: parse_pixel_type(dtype) for key, dtype in pixel_type.items()
+ }
elif preserve_dtype:
self.propagate_dtype() # all outputs will have the same type as the main input raster
@@ -751,7 +828,11 @@ class App(OTBObject):
dest = files if filepath.exists() else missing
dest.append(str(filepath.absolute()))
for filename in missing:
- logger.error("%s: execution seems to have failed, %s does not exist", self.name, filename)
+ logger.error(
+ "%s: execution seems to have failed, %s does not exist",
+ self.name,
+ filename,
+ )
return bool(files) and not missing
# Private functions
@@ -769,11 +850,17 @@ class App(OTBObject):
for arg in args:
if isinstance(arg, dict):
kwargs.update(arg)
- elif isinstance(arg, (str, OTBObject)) or isinstance(arg, list) and self.is_key_list(self.input_key):
+ elif (
+ isinstance(arg, (str, OTBObject))
+ or isinstance(arg, list)
+ and self.is_key_list(self.input_key)
+ ):
kwargs.update({self.input_key: arg})
return kwargs
- def __set_param(self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]):
+ def __set_param(
+ self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]
+ ):
"""Set one parameter, decide which otb.Application method to use depending on target object."""
if obj is None or (isinstance(obj, (list, tuple)) and not obj):
self.app.ClearValue(key)
@@ -781,9 +868,13 @@ class App(OTBObject):
# Single-parameter cases
if isinstance(obj, OTBObject):
self.app.ConnectImage(key, obj.app, obj.output_image_key)
- elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB
+ elif isinstance(
+ obj, otb.Application
+ ): # this is for backward comp with plain OTB
self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0])
- elif key == "ram": # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200
+ elif (
+ key == "ram"
+ ): # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200
self.app.SetParameterInt("ram", int(obj))
elif not isinstance(obj, list): # any other parameters (str, int...)
self.app.SetParameterValue(key, obj)
@@ -793,7 +884,9 @@ class App(OTBObject):
for inp in obj:
if isinstance(inp, OTBObject):
self.app.ConnectImage(key, inp.app, inp.output_image_key)
- elif isinstance(inp, otb.Application): # this is for backward comp with plain OTB
+ elif isinstance(
+ inp, otb.Application
+ ): # this is for backward comp with plain OTB
self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0])
else: # here `input` should be an image filepath
# Append `input` to the list, do not overwrite any previously set element of the image list
@@ -809,9 +902,13 @@ class App(OTBObject):
continue
value = self.app.GetParameterValue(key)
# TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken
- if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(key):
+ if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(
+ key
+ ):
try:
- value = str(value) # some default str values like "mode" or "interpolator"
+ value = str(
+ value
+ ) # some default str values like "mode" or "interpolator"
self._auto_parameters[key] = value
continue
except RuntimeError:
@@ -841,13 +938,21 @@ class App(OTBObject):
if key in self.parameters:
return self.parameters[key]
raise KeyError(f"{self.name}: unknown or undefined parameter '{key}'")
- raise TypeError(f"{self.name}: cannot access object item or slice using {type(key)} object")
+ raise TypeError(
+ f"{self.name}: cannot access object item or slice using {type(key)} object"
+ )
class Slicer(App):
"""Slicer objects i.e. when we call something like raster[:, :, 2] from Python."""
- def __init__(self, obj: OTBObject, rows: slice, cols: slice, channels: slice | list[int] | int):
+ def __init__(
+ self,
+ obj: OTBObject,
+ rows: slice,
+ cols: slice,
+ channels: slice | list[int] | int,
+ ):
"""Create a slicer object, that can be used directly for writing or inside a BandMath.
It contains :
@@ -861,7 +966,14 @@ class Slicer(App):
channels: channels, can be slicing, list or int
"""
- super().__init__("ExtractROI", obj, mode="extent", quiet=True, frozen=True, name=f"Slicer from {obj.name}")
+ super().__init__(
+ "ExtractROI",
+ obj,
+ mode="extent",
+ quiet=True,
+ frozen=True,
+ name=f"Slicer from {obj.name}",
+ )
self.rows, self.cols = rows, cols
parameters = {}
@@ -879,7 +991,9 @@ class Slicer(App):
elif isinstance(channels, tuple):
channels = list(channels)
elif not isinstance(channels, list):
- raise ValueError(f"Invalid type for channels, should be int, slice or list of bands. : {channels}")
+ raise ValueError(
+ f"Invalid type for channels, should be int, slice or list of bands. : {channels}"
+ )
# Change the potential negative index values to reverse index
channels = [c if c >= 0 else nb_channels + c for c in channels]
parameters.update({"cl": [f"Channel{i + 1}" for i in channels]})
@@ -891,17 +1005,23 @@ class Slicer(App):
parameters.update({"mode.extent.uly": rows.start})
spatial_slicing = True
if rows.stop is not None and rows.stop != -1:
- parameters.update({"mode.extent.lry": rows.stop - 1}) # subtract 1 to respect python convention
+ parameters.update(
+ {"mode.extent.lry": rows.stop - 1}
+ ) # subtract 1 to respect python convention
spatial_slicing = True
if cols.start is not None:
parameters.update({"mode.extent.ulx": cols.start})
spatial_slicing = True
if cols.stop is not None and cols.stop != -1:
- parameters.update({"mode.extent.lrx": cols.stop - 1}) # subtract 1 to respect python convention
+ parameters.update(
+ {"mode.extent.lrx": cols.stop - 1}
+ ) # subtract 1 to respect python convention
spatial_slicing = True
# These are some attributes when the user simply wants to extract *one* band to be used in an Operation
if not spatial_slicing and isinstance(channels, list) and len(channels) == 1:
- self.one_band_sliced = channels[0] + 1 # OTB convention: channels start at 1
+ self.one_band_sliced = (
+ channels[0] + 1
+ ) # OTB convention: channels start at 1
self.input = obj
# Execute app
@@ -956,7 +1076,9 @@ class Operation(App):
# NB: the keys of the dictionary are strings-only, instead of 'complex' objects, to enable easy serialization
self.im_dic = {}
self.im_count = 1
- map_repr_to_input = {} # to be able to retrieve the real python object from its string representation
+ map_repr_to_input = (
+ {}
+ ) # to be able to retrieve the real python object from its string representation
for inp in self.inputs:
if not isinstance(inp, (int, float)):
if str(inp) not in self.im_dic:
@@ -964,13 +1086,23 @@ class Operation(App):
map_repr_to_input[repr(inp)] = inp
self.im_count += 1
# Getting unique image inputs, in the order im1, im2, im3 ...
- self.unique_inputs = [map_repr_to_input[id_str] for id_str in sorted(self.im_dic, key=self.im_dic.get)]
+ self.unique_inputs = [
+ map_repr_to_input[id_str]
+ for id_str in sorted(self.im_dic, key=self.im_dic.get)
+ ]
self.exp_bands, self.exp = self.get_real_exp(self.fake_exp_bands)
appname = "BandMath" if len(self.exp_bands) == 1 else "BandMathX"
name = f'Operation exp="{self.exp}"'
- super().__init__(appname, il=self.unique_inputs, exp=self.exp, quiet=True, name=name)
+ super().__init__(
+ appname, il=self.unique_inputs, exp=self.exp, quiet=True, name=name
+ )
- def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None):
+ def build_fake_expressions(
+ self,
+ operator: str,
+ inputs: list[OTBObject | str | int | float],
+ nb_bands: int = None,
+ ):
"""Create a list of 'fake' expressions, one for each band.
E.g for the operation input1 + input2, we create a fake expression that is like "str(input1) + str(input2)"
@@ -989,12 +1121,21 @@ class Operation(App):
pass
# For any other operations, the output number of bands is the same as inputs
else:
- if any(isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced") for inp in inputs):
+ if any(
+ isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced")
+ for inp in inputs
+ ):
nb_bands = 1
else:
- nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))]
+ nb_bands_list = [
+ get_nbchannels(inp)
+ for inp in inputs
+ if not isinstance(inp, (float, int))
+ ]
# check that all inputs have the same nb of bands
- if len(nb_bands_list) > 1 and not all(x == nb_bands_list[0] for x in nb_bands_list):
+ if len(nb_bands_list) > 1 and not all(
+ x == nb_bands_list[0] for x in nb_bands_list
+ ):
raise ValueError("All images do not have the same number of bands")
nb_bands = nb_bands_list[0]
@@ -1008,10 +1149,14 @@ class Operation(App):
if len(inputs) == 3 and k == 0:
# When cond is monoband whereas the result is multiband, we expand the cond to multiband
cond_band = 1 if nb_bands != inp.shape[2] else band
- fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp(inp, cond_band, keep_logical=True)
+ fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp(
+ inp, cond_band, keep_logical=True
+ )
else:
# Any other input
- fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp(inp, band, keep_logical=False)
+ fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp(
+ inp, band, keep_logical=False
+ )
expressions.append(fake_exp)
# Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates)
if i == 0 and corresponding_inputs and nb_channels:
@@ -1025,7 +1170,9 @@ class Operation(App):
# We create here the "fake" expression. For example, for a BandMathX expression such as '2 * im1 + im2',
# the false expression stores the expression 2 * str(input1) + str(input2)
fake_exp = f"({expressions[0]} {operator} {expressions[1]})"
- elif len(inputs) == 3 and operator == "?": # this is only for ternary expression
+ elif (
+ len(inputs) == 3 and operator == "?"
+ ): # this is only for ternary expression
fake_exp = f"({expressions[0]} ? {expressions[1]} : {expressions[2]})"
self.fake_exp_bands.append(fake_exp)
@@ -1052,7 +1199,9 @@ class Operation(App):
return exp_bands, ";".join(exp_bands)
@staticmethod
- def make_fake_exp(x: OTBObject | str, band: int, keep_logical: bool = False) -> tuple[str, list[OTBObject], int]:
+ def make_fake_exp(
+ x: OTBObject | str, band: int, keep_logical: bool = False
+ ) -> tuple[str, list[OTBObject], int]:
"""This an internal function, only to be used by `build_fake_expressions`.
Enable to create a fake expression just for one input and one band.
@@ -1127,9 +1276,16 @@ class LogicalOperation(Operation):
"""
self.logical_fake_exp_bands = []
super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation")
- self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands)
+ self.logical_exp_bands, self.logical_exp = self.get_real_exp(
+ self.logical_fake_exp_bands
+ )
- def build_fake_expressions(self, operator: str, inputs: list[OTBObject | str | int | float], nb_bands: int = None):
+ def build_fake_expressions(
+ self,
+ operator: str,
+ inputs: list[OTBObject | str | int | float],
+ nb_bands: int = None,
+ ):
"""Create a list of 'fake' expressions, one for each band.
e.g for the operation input1 > input2, we create a fake expression that is like
@@ -1142,19 +1298,30 @@ class LogicalOperation(Operation):
"""
# For any other operations, the output number of bands is the same as inputs
- if any(isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced") for inp in inputs):
+ if any(
+ isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced")
+ for inp in inputs
+ ):
nb_bands = 1
else:
- nb_bands_list = [get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))]
+ nb_bands_list = [
+ get_nbchannels(inp)
+ for inp in inputs
+ if not isinstance(inp, (float, int))
+ ]
# check that all inputs have the same nb of bands
- if len(nb_bands_list) > 1 and not all(x == nb_bands_list[0] for x in nb_bands_list):
+ if len(nb_bands_list) > 1 and not all(
+ x == nb_bands_list[0] for x in nb_bands_list
+ ):
raise ValueError("All images do not have the same number of bands")
nb_bands = nb_bands_list[0]
# Create a list of fake exp, each item of the list corresponding to one band
for i, band in enumerate(range(1, nb_bands + 1)):
expressions = []
for inp in inputs:
- fake_exp, corresp_inputs, nb_channels = super().make_fake_exp(inp, band, keep_logical=True)
+ fake_exp, corresp_inputs, nb_channels = super().make_fake_exp(
+ inp, band, keep_logical=True
+ )
expressions.append(fake_exp)
# Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates)
if i == 0 and corresp_inputs and nb_channels:
@@ -1195,9 +1362,16 @@ class Input(App):
class Output(OTBObject):
"""Object that behave like a pointer to a specific application output file."""
+
_filepath: str | Path = None
- def __init__(self, pyotb_app: App, param_key: str = None, filepath: str = None, mkdir: bool = True):
+ def __init__(
+ self,
+ pyotb_app: App,
+ param_key: str = None,
+ filepath: str = None,
+ mkdir: bool = True,
+ ):
"""Constructor for an Output object.
Args:
@@ -1262,7 +1436,9 @@ class Output(OTBObject):
def write(self, filepath: None | str | Path = None, **kwargs) -> bool:
"""Write output to disk, filepath is not required if it was provided to parent App during init."""
if filepath is None:
- return self.parent_pyotb_app.write({self.output_image_key: self.filepath}, **kwargs)
+ return self.parent_pyotb_app.write(
+ {self.output_image_key: self.filepath}, **kwargs
+ )
return self.parent_pyotb_app.write({self.output_image_key: filepath}, **kwargs)
def __str__(self) -> str:
@@ -1293,7 +1469,7 @@ def add_vsi_prefix(filepath: str | Path) -> str:
".gz": "vsigzip",
".7z": "vsi7z",
".zip": "vsizip",
- ".rar": "vsirar"
+ ".rar": "vsirar",
}
basename = filepath.split("?")[0]
ext = Path(basename).suffix
@@ -1319,8 +1495,12 @@ def get_nbchannels(inp: str | Path | OTBObject) -> int:
try:
info = App("ReadImageInfo", inp, quiet=True)
return info["numberbands"]
- except RuntimeError as info_err: # this happens when we pass a str that is not a filepath
- raise TypeError(f"Could not get the number of channels file '{inp}' ({info_err})") from info_err
+ except (
+ RuntimeError
+ ) as info_err: # this happens when we pass a str that is not a filepath
+ raise TypeError(
+ f"Could not get the number of channels file '{inp}' ({info_err})"
+ ) from info_err
raise TypeError(f"Can't read number of channels of type '{type(inp)}' object {inp}")
@@ -1341,8 +1521,12 @@ def get_pixel_type(inp: str | Path | OTBObject) -> str:
try:
info = App("ReadImageInfo", inp, quiet=True)
datatype = info["datatype"] # which is such as short, float...
- except RuntimeError as info_err: # this happens when we pass a str that is not a filepath
- raise TypeError(f"Could not get the pixel type of `{inp}` ({info_err})") from info_err
+ except (
+ RuntimeError
+ ) as info_err: # this happens when we pass a str that is not a filepath
+ raise TypeError(
+ f"Could not get the pixel type of `{inp}` ({info_err})"
+ ) from info_err
if datatype:
return parse_pixel_type(datatype)
raise TypeError(f"Could not get the pixel type of {type(inp)} object {inp}")
@@ -1376,13 +1560,21 @@ def parse_pixel_type(pixel_type: str | int) -> int:
return getattr(otb, f"ImagePixelType_{pixel_type}")
if pixel_type in datatype_to_pixeltype:
return getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[pixel_type]}")
- raise KeyError(f"Unknown data type `{pixel_type}`. Available ones: {datatype_to_pixeltype}")
- raise TypeError(f"Bad pixel type specification ({pixel_type} of type {type(pixel_type)})")
+ raise KeyError(
+ f"Unknown data type `{pixel_type}`. Available ones: {datatype_to_pixeltype}"
+ )
+ raise TypeError(
+ f"Bad pixel type specification ({pixel_type} of type {type(pixel_type)})"
+ )
def get_out_images_param_keys(app: OTBObject) -> list[str]:
"""Return every output parameter keys of an OTB app."""
- return [key for key in app.GetParametersKeys() if app.GetParameterType(key) == otb.ParameterType_OutputImage]
+ return [
+ key
+ for key in app.GetParametersKeys()
+ if app.GetParameterType(key) == otb.ParameterType_OutputImage
+ ]
def summarize(
@@ -1406,6 +1598,7 @@ def summarize(
parameters of an app and its parents
"""
+
def strip_path(param: str | Any):
if not isinstance(param, str):
return summarize(param)
@@ -1420,8 +1613,17 @@ def summarize(
parameters = {}
for key, param in obj.parameters.items():
- if strip_input_paths and obj.is_input(key) or strip_output_paths and obj.is_output(key):
- parameters[key] = [strip_path(p) for p in param] if isinstance(param, list) else strip_path(param)
+ if (
+ strip_input_paths
+ and obj.is_input(key)
+ or strip_output_paths
+ and obj.is_output(key)
+ ):
+ parameters[key] = (
+ [strip_path(p) for p in param]
+ if isinstance(param, list)
+ else strip_path(param)
+ )
else:
parameters[key] = summarize(param)
return {"name": obj.app.GetName(), "parameters": parameters}
diff --git a/pyotb/functions.py b/pyotb/functions.py
index 6cc7889..96431cd 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -1,19 +1,22 @@
# -*- coding: utf-8 -*-
"""This module provides a set of functions for pyotb."""
from __future__ import annotations
+
import inspect
import os
+import subprocess
import sys
-import uuid
import textwrap
-import subprocess
+import uuid
from collections import Counter
-from .core import App, Operation, LogicalOperation, Input, get_nbchannels
+from .core import App, Input, LogicalOperation, Operation, get_nbchannels
from .helpers import logger
-def where(cond: App | str, x: App | str | int | float, y: App | str | int | float) -> Operation:
+def where(
+ cond: App | str, x: App | str | int | float, y: App | str | int | float
+) -> Operation:
"""Functionally similar to numpy.where. Where cond is True (!=0), returns x. Else returns y.
Args:
@@ -36,18 +39,29 @@ def where(cond: App | str, x: App | str | int | float, y: App | str | int | floa
y_nb_channels = get_nbchannels(y)
if x_nb_channels and y_nb_channels:
if x_nb_channels != y_nb_channels:
- raise ValueError('X and Y images do not have the same number of bands. '
- f'X has {x_nb_channels} bands whereas Y has {y_nb_channels} bands')
+ raise ValueError(
+ "X and Y images do not have the same number of bands. "
+ f"X has {x_nb_channels} bands whereas Y has {y_nb_channels} bands"
+ )
x_or_y_nb_channels = x_nb_channels if x_nb_channels else y_nb_channels
cond_nb_channels = get_nbchannels(cond)
- if cond_nb_channels != 1 and x_or_y_nb_channels and cond_nb_channels != x_or_y_nb_channels:
- raise ValueError('Condition and X&Y do not have the same number of bands. Condition has '
- f'{cond_nb_channels} bands whereas X&Y have {x_or_y_nb_channels} bands')
+ if (
+ cond_nb_channels != 1
+ and x_or_y_nb_channels
+ and cond_nb_channels != x_or_y_nb_channels
+ ):
+ raise ValueError(
+ "Condition and X&Y do not have the same number of bands. Condition has "
+ f"{cond_nb_channels} bands whereas X&Y have {x_or_y_nb_channels} bands"
+ )
# If needed, duplicate the single band binary mask to multiband to match the dimensions of x & y
if cond_nb_channels == 1 and x_or_y_nb_channels and x_or_y_nb_channels != 1:
- logger.info('The condition has one channel whereas X/Y has/have %s channels. Expanding number'
- ' of channels of condition to match the number of channels of X/Y', x_or_y_nb_channels)
+ logger.info(
+ "The condition has one channel whereas X/Y has/have %s channels. Expanding number"
+ " of channels of condition to match the number of channels of X/Y",
+ x_or_y_nb_channels,
+ )
# Get the number of bands of the result
if x_or_y_nb_channels: # if X or Y is a raster
@@ -55,10 +69,12 @@ def where(cond: App | str, x: App | str | int | float, y: App | str | int | floa
else: # if only cond is a raster
out_nb_channels = cond_nb_channels
- return Operation('?', cond, x, y, nb_bands=out_nb_channels)
+ return Operation("?", cond, x, y, nb_bands=out_nb_channels)
-def clip(image: App | str, v_min: App | str | int | float, v_max: App | str | int | float):
+def clip(
+ image: App | str, v_min: App | str | int | float, v_max: App | str | int | float
+):
"""Clip values of image in a range of values.
Args:
@@ -96,7 +112,11 @@ def all(*inputs): # pylint: disable=redefined-builtin
if len(inputs) == 1 and isinstance(inputs[0], (list, tuple)):
inputs = inputs[0]
# Add support for generator inputs (to have the same behavior as built-in `all` function)
- if isinstance(inputs, tuple) and len(inputs) == 1 and inspect.isgenerator(inputs[0]):
+ if (
+ isinstance(inputs, tuple)
+ and len(inputs) == 1
+ and inspect.isgenerator(inputs[0])
+ ):
inputs = list(inputs[0])
# Transforming potential filepaths to pyotb objects
inputs = [Input(inp) if isinstance(inp, str) else inp for inp in inputs]
@@ -107,7 +127,7 @@ def all(*inputs): # pylint: disable=redefined-builtin
if isinstance(inp, LogicalOperation):
res = inp[:, :, 0]
else:
- res = (inp[:, :, 0] != 0)
+ res = inp[:, :, 0] != 0
for band in range(1, inp.shape[-1]):
if isinstance(inp, LogicalOperation):
res = res & inp[:, :, band]
@@ -147,7 +167,11 @@ def any(*inputs): # pylint: disable=redefined-builtin
if len(inputs) == 1 and isinstance(inputs[0], (list, tuple)):
inputs = inputs[0]
# Add support for generator inputs (to have the same behavior as built-in `any` function)
- if isinstance(inputs, tuple) and len(inputs) == 1 and inspect.isgenerator(inputs[0]):
+ if (
+ isinstance(inputs, tuple)
+ and len(inputs) == 1
+ and inspect.isgenerator(inputs[0])
+ ):
inputs = list(inputs[0])
# Transforming potential filepaths to pyotb objects
inputs = [Input(inp) if isinstance(inp, str) else inp for inp in inputs]
@@ -158,7 +182,7 @@ def any(*inputs): # pylint: disable=redefined-builtin
if isinstance(inp, LogicalOperation):
res = inp[:, :, 0]
else:
- res = (inp[:, :, 0] != 0)
+ res = inp[:, :, 0] != 0
for band in range(1, inp.shape[-1]):
if isinstance(inp, LogicalOperation):
@@ -203,10 +227,14 @@ def run_tf_function(func):
"""
try:
- from .apps import TensorflowModelServe # pylint: disable=import-outside-toplevel
+ from .apps import ( # pylint: disable=import-outside-toplevel
+ TensorflowModelServe,
+ )
except ImportError:
- logger.error('Could not run Tensorflow function: failed to import TensorflowModelServe.'
- 'Check that you have OTBTF configured (https://github.com/remicres/otbtf#how-to-install)')
+ logger.error(
+ "Could not run Tensorflow function: failed to import TensorflowModelServe."
+ "Check that you have OTBTF configured (https://github.com/remicres/otbtf#how-to-install)"
+ )
raise
def get_tf_pycmd(output_dir, channels, scalar_inputs):
@@ -228,7 +256,8 @@ def run_tf_function(func):
create_and_save_model_str = func_def_str
# Adding the instructions to create the model and save it to output dir
- create_and_save_model_str += textwrap.dedent(f"""
+ create_and_save_model_str += textwrap.dedent(
+ f"""
import tensorflow as tf
model_inputs = []
@@ -248,11 +277,12 @@ def run_tf_function(func):
# Create and save the .pb model
model = tf.keras.Model(inputs=model_inputs, outputs=output)
model.save("{output_dir}")
- """)
+ """
+ )
return create_and_save_model_str
- def wrapper(*inputs, tmp_dir='/tmp'):
+ def wrapper(*inputs, tmp_dir="/tmp"):
"""For the user point of view, this function simply applies some TensorFlow operations to some rasters.
Implicitly, it saves a .pb model that describe the TF operations, then creates an OTB ModelServe application
@@ -284,22 +314,35 @@ def run_tf_function(func):
# Create and save the model. This is executed **inside an independent process** because (as of 2022-03),
# tensorflow python library and OTBTF are incompatible
- out_savedmodel = os.path.join(tmp_dir, f'tmp_otbtf_model_{uuid.uuid4()}')
+ out_savedmodel = os.path.join(tmp_dir, f"tmp_otbtf_model_{uuid.uuid4()}")
pycmd = get_tf_pycmd(out_savedmodel, channels, scalar_inputs)
cmd_args = [sys.executable, "-c", pycmd]
try:
- subprocess.run(cmd_args, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
+ subprocess.run(
+ cmd_args,
+ env=os.environ,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=True,
+ )
except subprocess.SubprocessError:
logger.debug("Failed to call subprocess")
if not os.path.isdir(out_savedmodel):
logger.info("Failed to save the model")
# Initialize the OTBTF model serving application
- model_serve = TensorflowModelServe({'model.dir': out_savedmodel, 'optim.disabletiling': 'on',
- 'model.fullyconv': 'on'}, n_sources=len(raster_inputs), frozen=True)
+ model_serve = TensorflowModelServe(
+ {
+ "model.dir": out_savedmodel,
+ "optim.disabletiling": "on",
+ "model.fullyconv": "on",
+ },
+ n_sources=len(raster_inputs),
+ frozen=True,
+ )
# Set parameters and execute
for i, inp in enumerate(raster_inputs):
- model_serve.set_parameters({f'source{i + 1}.il': [inp]})
+ model_serve.set_parameters({f"source{i + 1}.il": [inp]})
model_serve.execute()
# TODO: handle the deletion of the temporary model ?
@@ -308,9 +351,14 @@ def run_tf_function(func):
return wrapper
-def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_rule: str = 'minimal',
- interpolator: str = 'nn', reference_window_input: dict = None,
- reference_pixel_size_input: str = None) -> list[App]:
+def define_processing_area(
+ *args,
+ window_rule: str = "intersection",
+ pixel_size_rule: str = "minimal",
+ interpolator: str = "nn",
+ reference_window_input: dict = None,
+ reference_pixel_size_input: str = None,
+) -> list[App]:
"""Given several inputs, this function handles the potential resampling and cropping to same extent.
WARNING: Not fully implemented / tested
@@ -338,7 +386,7 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
metadatas = {}
for inp in inputs:
if isinstance(inp, str): # this is for filepaths
- metadata = Input(inp).app.GetImageMetaData('out')
+ metadata = Input(inp).app.GetImageMetaData("out")
elif isinstance(inp, App):
metadata = inp.app.GetImageMetaData(inp.output_param)
else:
@@ -348,100 +396,147 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
# Get a metadata of an arbitrary image. This is just to compare later with other images
any_metadata = next(iter(metadatas.values()))
# Checking if all images have the same projection
- if not all(metadata['ProjectionRef'] == any_metadata['ProjectionRef']
- for metadata in metadatas.values()):
- logger.warning('All images may not have the same CRS, which might cause unpredictable results')
+ if not all(
+ metadata["ProjectionRef"] == any_metadata["ProjectionRef"]
+ for metadata in metadatas.values()
+ ):
+ logger.warning(
+ "All images may not have the same CRS, which might cause unpredictable results"
+ )
# Handling different spatial footprints
# TODO: there seems to have a bug, ImageMetaData is not updated when running an app,
# cf https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2234. Should we use ImageOrigin instead?
- if not all(metadata['UpperLeftCorner'] == any_metadata['UpperLeftCorner']
- and metadata['LowerRightCorner'] == any_metadata['LowerRightCorner']
- for metadata in metadatas.values()):
+ if not all(
+ metadata["UpperLeftCorner"] == any_metadata["UpperLeftCorner"]
+ and metadata["LowerRightCorner"] == any_metadata["LowerRightCorner"]
+ for metadata in metadatas.values()
+ ):
# Retrieving the bounding box that will be common for all inputs
- if window_rule == 'intersection':
+ if window_rule == "intersection":
# The coordinates depend on the orientation of the axis of projection
- if any_metadata['GeoTransform'][1] >= 0:
- ulx = max(metadata['UpperLeftCorner'][0] for metadata in metadatas.values())
- lrx = min(metadata['LowerRightCorner'][0] for metadata in metadatas.values())
+ if any_metadata["GeoTransform"][1] >= 0:
+ ulx = max(
+ metadata["UpperLeftCorner"][0] for metadata in metadatas.values()
+ )
+ lrx = min(
+ metadata["LowerRightCorner"][0] for metadata in metadatas.values()
+ )
else:
- ulx = min(metadata['UpperLeftCorner'][0] for metadata in metadatas.values())
- lrx = max(metadata['LowerRightCorner'][0] for metadata in metadatas.values())
- if any_metadata['GeoTransform'][-1] >= 0:
- lry = min(metadata['LowerRightCorner'][1] for metadata in metadatas.values())
- uly = max(metadata['UpperLeftCorner'][1] for metadata in metadatas.values())
+ ulx = min(
+ metadata["UpperLeftCorner"][0] for metadata in metadatas.values()
+ )
+ lrx = max(
+ metadata["LowerRightCorner"][0] for metadata in metadatas.values()
+ )
+ if any_metadata["GeoTransform"][-1] >= 0:
+ lry = min(
+ metadata["LowerRightCorner"][1] for metadata in metadatas.values()
+ )
+ uly = max(
+ metadata["UpperLeftCorner"][1] for metadata in metadatas.values()
+ )
else:
- lry = max(metadata['LowerRightCorner'][1] for metadata in metadatas.values())
- uly = min(metadata['UpperLeftCorner'][1] for metadata in metadatas.values())
-
- elif window_rule == 'same_as_input':
- ulx = metadatas[reference_window_input]['UpperLeftCorner'][0]
- lrx = metadatas[reference_window_input]['LowerRightCorner'][0]
- lry = metadatas[reference_window_input]['LowerRightCorner'][1]
- uly = metadatas[reference_window_input]['UpperLeftCorner'][1]
- elif window_rule == 'specify':
+ lry = max(
+ metadata["LowerRightCorner"][1] for metadata in metadatas.values()
+ )
+ uly = min(
+ metadata["UpperLeftCorner"][1] for metadata in metadatas.values()
+ )
+
+ elif window_rule == "same_as_input":
+ ulx = metadatas[reference_window_input]["UpperLeftCorner"][0]
+ lrx = metadatas[reference_window_input]["LowerRightCorner"][0]
+ lry = metadatas[reference_window_input]["LowerRightCorner"][1]
+ uly = metadatas[reference_window_input]["UpperLeftCorner"][1]
+ elif window_rule == "specify":
pass
# TODO : it is when the user explicitly specifies the bounding box -> add some arguments in the function
- elif window_rule == 'union':
+ elif window_rule == "union":
pass
# TODO : it is when the user wants the final bounding box to be the union of all bounding box
# It should replace any 'outside' pixel by some NoData -> add `fillvalue` argument in the function
# Applying this bounding box to all inputs
- logger.info('Cropping all images to extent Upper Left (%s, %s), Lower Right (%s, %s)', ulx, uly, lrx, lry)
+ logger.info(
+ "Cropping all images to extent Upper Left (%s, %s), Lower Right (%s, %s)",
+ ulx,
+ uly,
+ lrx,
+ lry,
+ )
new_inputs = []
for inp in inputs:
try:
params = {
- 'in': inp, 'mode': 'extent', 'mode.extent.unit': 'phy',
- 'mode.extent.ulx': ulx, 'mode.extent.uly': lry, # bug in OTB <= 7.3 :
- 'mode.extent.lrx': lrx, 'mode.extent.lry': uly, # ULY/LRY are inverted
+ "in": inp,
+ "mode": "extent",
+ "mode.extent.unit": "phy",
+ "mode.extent.ulx": ulx,
+ "mode.extent.uly": lry, # bug in OTB <= 7.3 :
+ "mode.extent.lrx": lrx,
+ "mode.extent.lry": uly, # ULY/LRY are inverted
}
- new_input = App('ExtractROI', params)
+ new_input = App("ExtractROI", params)
# TODO: OTB 7.4 fixes this bug, how to handle different versions of OTB?
new_inputs.append(new_input)
# Potentially update the reference inputs for later resampling
- if str(inp) == str(reference_pixel_size_input): # we use comparison of string because calling '=='
+ if str(inp) == str(
+ reference_pixel_size_input
+ ): # we use comparison of string because calling '=='
# on pyotb objects implicitly calls BandMathX application, which is not desirable
reference_pixel_size_input = new_input
except RuntimeError as e:
- logger.error('Cannot define the processing area for input %s: %s', inp, e)
+ logger.error(
+ "Cannot define the processing area for input %s: %s", inp, e
+ )
raise
inputs = new_inputs
# Update metadatas
- metadatas = {input: input.app.GetImageMetaData('out') for input in inputs}
+ metadatas = {input: input.app.GetImageMetaData("out") for input in inputs}
# Get a metadata of an arbitrary image. This is just to compare later with other images
any_metadata = next(iter(metadatas.values()))
# Handling different pixel sizes
- if not all(metadata['GeoTransform'][1] == any_metadata['GeoTransform'][1]
- and metadata['GeoTransform'][5] == any_metadata['GeoTransform'][5]
- for metadata in metadatas.values()):
+ if not all(
+ metadata["GeoTransform"][1] == any_metadata["GeoTransform"][1]
+ and metadata["GeoTransform"][5] == any_metadata["GeoTransform"][5]
+ for metadata in metadatas.values()
+ ):
# Retrieving the pixel size that will be common for all inputs
- if pixel_size_rule == 'minimal':
+ if pixel_size_rule == "minimal":
# selecting the input with the smallest x pixel size
- reference_input = min(metadatas, key=lambda x: metadatas[x]['GeoTransform'][1])
- if pixel_size_rule == 'maximal':
+ reference_input = min(
+ metadatas, key=lambda x: metadatas[x]["GeoTransform"][1]
+ )
+ if pixel_size_rule == "maximal":
# selecting the input with the highest x pixel size
- reference_input = max(metadatas, key=lambda x: metadatas[x]['GeoTransform'][1])
- elif pixel_size_rule == 'same_as_input':
+ reference_input = max(
+ metadatas, key=lambda x: metadatas[x]["GeoTransform"][1]
+ )
+ elif pixel_size_rule == "same_as_input":
reference_input = reference_pixel_size_input
- elif pixel_size_rule == 'specify':
+ elif pixel_size_rule == "specify":
pass
# TODO : when the user explicitly specify the pixel size -> add argument inside the function
- pixel_size = metadatas[reference_input]['GeoTransform'][1]
+ pixel_size = metadatas[reference_input]["GeoTransform"][1]
# Perform resampling on inputs that do not comply with the target pixel size
- logger.info('Resampling all inputs to resolution: %s', pixel_size)
+ logger.info("Resampling all inputs to resolution: %s", pixel_size)
new_inputs = []
for inp in inputs:
- if metadatas[inp]['GeoTransform'][1] != pixel_size:
- superimposed = App('Superimpose', inr=reference_input, inm=inp, interpolator=interpolator)
+ if metadatas[inp]["GeoTransform"][1] != pixel_size:
+ superimposed = App(
+ "Superimpose",
+ inr=reference_input,
+ inm=inp,
+ interpolator=interpolator,
+ )
new_inputs.append(superimposed)
else:
new_inputs.append(inp)
inputs = new_inputs
- metadatas = {inp: inp.app.GetImageMetaData('out') for inp in inputs}
+ metadatas = {inp: inp.app.GetImageMetaData("out") for inp in inputs}
# Final superimposition to be sure to have the exact same image sizes
image_sizes = {}
@@ -451,13 +546,22 @@ def define_processing_area(*args, window_rule: str = 'intersection', pixel_size_
image_sizes[inp] = inp.shape[:2]
# Selecting the most frequent image size. It will be used as reference.
most_common_image_size, _ = Counter(image_sizes.values()).most_common(1)[0]
- same_size_images = [inp for inp, image_size in image_sizes.items() if image_size == most_common_image_size]
+ same_size_images = [
+ inp
+ for inp, image_size in image_sizes.items()
+ if image_size == most_common_image_size
+ ]
# Superimposition for images that do not have the same size as the others
new_inputs = []
for inp in inputs:
if image_sizes[inp] != most_common_image_size:
- superimposed = App('Superimpose', inr=same_size_images[0], inm=inp, interpolator=interpolator)
+ superimposed = App(
+ "Superimpose",
+ inr=same_size_images[0],
+ inm=inp,
+ interpolator=interpolator,
+ )
new_inputs.append(superimposed)
else:
new_inputs.append(inp)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 03200a2..5363937 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -1,12 +1,11 @@
# -*- coding: utf-8 -*-
"""This module helps to ensure we properly initialize pyotb: only in case OTB is found and apps are available."""
+import logging
import os
import sys
-import logging
from pathlib import Path
from shutil import which
-
# Allow user to switch between OTB directories without setting every env variable
OTB_ROOT = os.environ.get("OTB_ROOT")
@@ -15,10 +14,14 @@ OTB_ROOT = os.environ.get("OTB_ROOT")
# then use pyotb.set_logger_level() to adjust logger verbosity
logger = logging.getLogger("pyOTB")
logger_handler = logging.StreamHandler(sys.stdout)
-formatter = logging.Formatter(fmt="%(asctime)s (%(levelname)-4s) [pyOTB] %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
+formatter = logging.Formatter(
+ fmt="%(asctime)s (%(levelname)-4s) [pyOTB] %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
+)
logger_handler.setFormatter(formatter)
# Search for PYOTB_LOGGER_LEVEL, else use OTB_LOGGER_LEVEL as pyOTB level, or fallback to INFO
-LOG_LEVEL = os.environ.get("PYOTB_LOGGER_LEVEL") or os.environ.get("OTB_LOGGER_LEVEL") or "INFO"
+LOG_LEVEL = (
+ os.environ.get("PYOTB_LOGGER_LEVEL") or os.environ.get("OTB_LOGGER_LEVEL") or "INFO"
+)
logger.setLevel(getattr(logging, LOG_LEVEL))
# Here it would be possible to use a different level for a specific handler
# A more verbose one can go to text file while print only errors to stdout
@@ -60,6 +63,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru
try:
set_environment(prefix)
import otbApplication as otb # pylint: disable=import-outside-toplevel
+
return otb
except EnvironmentError as e:
raise SystemExit(f"Failed to import OTB with prefix={prefix}") from e
@@ -71,6 +75,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru
# Here, we can't properly set env variables before OTB import. We assume user did this before running python
# For LD_LIBRARY_PATH problems, use OTB_ROOT instead of PYTHONPATH
import otbApplication as otb # pylint: disable=import-outside-toplevel
+
if "OTB_APPLICATION_PATH" not in os.environ:
lib_dir = __find_lib(otb_module=otb)
apps_path = __find_apps_path(lib_dir)
@@ -79,7 +84,9 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru
except ImportError as e:
pythonpath = os.environ.get("PYTHONPATH")
if not scan:
- raise SystemExit(f"Failed to import OTB with env PYTHONPATH={pythonpath}") from e
+ raise SystemExit(
+ f"Failed to import OTB with env PYTHONPATH={pythonpath}"
+ ) from e
# Else search system
logger.info("Failed to import OTB. Searching for it...")
prefix = __find_otb_root(scan_userdir)
@@ -87,6 +94,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru
try:
set_environment(prefix)
import otbApplication as otb # pylint: disable=import-outside-toplevel
+
return otb
except EnvironmentError as e:
raise SystemExit("Auto setup for OTB env failed. Exiting.") from e
@@ -112,7 +120,7 @@ def set_environment(prefix: str):
if not prefix.exists():
raise FileNotFoundError(str(prefix))
built_from_source = False
- if not (prefix / 'README').exists():
+ if not (prefix / "README").exists():
built_from_source = True
# External libraries
lib_dir = __find_lib(prefix)
@@ -151,7 +159,9 @@ def set_environment(prefix: str):
gdal_data = str(prefix / "share/data")
proj_lib = str(prefix / "share/proj")
else:
- raise EnvironmentError(f"Can't find GDAL location with current OTB prefix '{prefix}' or in /usr")
+ raise EnvironmentError(
+ f"Can't find GDAL location with current OTB prefix '{prefix}' or in /usr"
+ )
os.environ["GDAL_DATA"] = gdal_data
os.environ["PROJ_LIB"] = proj_lib
@@ -168,7 +178,7 @@ def __find_lib(prefix: str = None, otb_module=None):
"""
if prefix is not None:
- lib_dir = prefix / 'lib'
+ lib_dir = prefix / "lib"
if lib_dir.exists():
return lib_dir.absolute()
if otb_module is not None:
@@ -276,33 +286,54 @@ def __suggest_fix_import(error_message: str, prefix: str):
logger.critical("An error occurred while importing OTB Python API")
logger.critical("OTB error message was '%s'", error_message)
if sys.platform == "linux":
- if error_message.startswith('libpython3.'):
- logger.critical("It seems like you need to symlink or recompile python bindings")
- if sys.executable.startswith('/usr/bin'):
- lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so"
- if which('ctest'):
- logger.critical("To recompile python bindings, use 'cd %s ; source otbenv.profile ; "
- "ctest -S share/otb/swig/build_wrapping.cmake -VV'", prefix)
+ if error_message.startswith("libpython3."):
+ logger.critical(
+ "It seems like you need to symlink or recompile python bindings"
+ )
+ if sys.executable.startswith("/usr/bin"):
+ lib = (
+ f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so"
+ )
+ if which("ctest"):
+ logger.critical(
+ "To recompile python bindings, use 'cd %s ; source otbenv.profile ; "
+ "ctest -S share/otb/swig/build_wrapping.cmake -VV'",
+ prefix,
+ )
elif Path(lib).exists():
expect_minor = int(error_message[11])
if expect_minor != sys.version_info.minor:
- logger.critical("Python library version mismatch (OTB was expecting 3.%s) : "
- "a simple symlink may not work, depending on your python version", expect_minor)
+ logger.critical(
+ "Python library version mismatch (OTB was expecting 3.%s) : "
+ "a simple symlink may not work, depending on your python version",
+ expect_minor,
+ )
target_lib = f"{prefix}/lib/libpython3.{expect_minor}.so.rh-python3{expect_minor}-1.0"
logger.critical("Use 'ln -s %s %s'", lib, target_lib)
else:
- logger.critical("You may need to install cmake in order to recompile python bindings")
+ logger.critical(
+ "You may need to install cmake in order to recompile python bindings"
+ )
else:
- logger.critical("Unable to automatically locate python dynamic library of %s", sys.executable)
+ logger.critical(
+ "Unable to automatically locate python dynamic library of %s",
+ sys.executable,
+ )
elif sys.platform == "win32":
if error_message.startswith("DLL load failed"):
if sys.version_info.minor != 7:
- logger.critical("You need Python 3.5 (OTB releases 6.4 to 7.4) or Python 3.7 (since OTB 8)")
+ logger.critical(
+ "You need Python 3.5 (OTB releases 6.4 to 7.4) or Python 3.7 (since OTB 8)"
+ )
else:
- logger.critical("It seems that your env variables aren't properly set,"
- " first use 'call otbenv.bat' then try to import pyotb once again")
+ logger.critical(
+ "It seems that your env variables aren't properly set,"
+ " first use 'call otbenv.bat' then try to import pyotb once again"
+ )
docs_link = "https://www.orfeo-toolbox.org/CookBook/Installation.html"
- logger.critical("You can verify installation requirements for your OS at %s", docs_link)
+ logger.critical(
+ "You can verify installation requirements for your OS at %s", docs_link
+ )
# Since helpers is the first module to be inititialized, this will prevent pyotb to run if OTB is not found
diff --git a/pyproject.toml b/pyproject.toml
index ac4d3df..6f86ad0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -5,14 +5,18 @@ build-backend = "setuptools.build_meta"
[project]
name = "pyotb"
description = "Library to enable easy use of the Orfeo ToolBox (OTB) in Python"
-authors = [{name = "Rémi Cresson", email = "remi.cresson@inrae.fr"}, {name = "Nicolas Narçon"}, {name = "Vincent Delbar"}]
+authors = [
+ { name = "Rémi Cresson", email = "remi.cresson@inrae.fr" },
+ { name = "Nicolas Narçon" },
+ { name = "Vincent Delbar" },
+]
requires-python = ">=3.7"
keywords = ["gis", "remote sensing", "otb", "orfeotoolbox", "orfeo toolbox"]
dependencies = ["numpy>=1.16"]
readme = "README.md"
license = { file = "LICENSE" }
dynamic = ["version"]
-classifiers=[
+classifiers = [
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
@@ -37,22 +41,23 @@ repository = "https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb"
packages = ["pyotb"]
[tool.setuptools.dynamic]
-version = {attr = "pyotb.__version__"}
+version = { attr = "pyotb.__version__" }
[tool.pylint]
-max-line-length = 120
+max-line-length = 88
max-module-lines = 2000
good-names = ["x", "y", "i", "j", "k", "e"]
disable = [
"fixme",
+ "line-too-long",
"too-many-locals",
"too-many-branches",
"too-many-statements",
- "too-many-instance-attributes"
+ "too-many-instance-attributes",
]
[tool.pydocstyle]
convention = "google"
[tool.black]
-line-length = 120
+line-length = 88
--
GitLab
From 5eddf64d71e60b86b6a9895d1f0177cc52e37f66 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 1 Jun 2023 18:36:38 +0000
Subject: [PATCH 194/399] Refac CI
---
.gitlab-ci.yml | 125 ++++++++++++++++++++--------------------------
pyotb/__init__.py | 2 +-
setup.py | 3 --
3 files changed, 54 insertions(+), 76 deletions(-)
delete mode 100644 setup.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index be29725..4d53e04 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,15 +1,23 @@
-workflow:
- rules:
- - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_REF_NAME =~ /master/
-
default:
image: $CI_REGISTRY/orfeotoolbox/otb-build-env/otb-ubuntu-native-develop-headless:20.04
tags:
- light
+ interruptible: true
+
+variables:
+ PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+
+cache:
+ key: $CI_COMMIT_REF_SLUG
+ paths:
+ - .cache/pip
+
+workflow:
+ rules:
+ - if: $CI_PIPELINE_SOURCE == "merge_request_event"
stages:
- Static Analysis
- - Documentation test
- Tests
- Ship
@@ -18,33 +26,11 @@ stages:
.static_analysis:
stage: Static Analysis
allow_failure: true
-
-codespell:
rules:
- changes:
- - pyotb/*.py
- - tests/*.py
- - README.md
- extends: .static_analysis
- before_script:
- - pip install codespell
- script:
- - codespell {pyotb,tests,doc,README.md}
-
-flake8:
- rules:
- - changes:
- - pyotb/*.py
- extends: .static_analysis
- before_script:
- - pip install flake8
- script:
- - flake8 --max-line-length=120 $PWD/pyotb --ignore=F403,E402,F401,W503,W504
+ - pyotb/*.py
pydocstyle:
- rules:
- - changes:
- - pyotb/*.py
extends: .static_analysis
before_script:
- pip install pydocstyle tomli
@@ -52,83 +38,74 @@ pydocstyle:
- pydocstyle $PWD/pyotb
pylint:
- rules:
- - changes:
- - pyotb/*.py
extends: .static_analysis
before_script:
- pip install pylint
script:
- pylint $PWD/pyotb
-# ---------------------------------- Documentation ----------------------------------
-
-.docs:
+codespell:
+ extends: .static_analysis
+ rules:
+ - changes:
+ - .gitlab-ci.yml
+ - pyotb/*.py
+ - tests/*.py
+ - doc/*.md
+ - README.md
before_script:
- - apt-get update && apt-get -y install virtualenv
- - virtualenv doc_env
- - source doc_env/bin/activate
- - pip install -U pip
- - pip install -U -r doc/doc_requirements.txt
-
-pages_test:
- stage: Documentation test
- extends: .docs
- except:
- - master
- when: manual
+ - pip install codespell
script:
- - mkdocs build --site-dir public_test
- artifacts:
- paths:
- - public_test
+ - codespell {pyotb,tests,doc,README.md}
-# -------------------------------------- Tests --------------------------------------
+# -------------------------------------- Tests --------------------------------------
.tests:
stage: Tests
+ allow_failure: false
rules:
- changes:
- - pyotb/*.py
- - tests/*.py
- allow_failure: false
+ - .gitlab-ci.yml
+ - pyotb/*.py
+ - tests/*.py
variables:
OTB_ROOT: /opt/otb
LD_LIBRARY_PATH: /opt/otb/lib
OTB_LOGGER_LEVEL: INFO
PYOTB_LOGGER_LEVEL: DEBUG
- IMAGE_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif?inline=false
- TEST_INPUT_IMAGE: tests/image.tif
artifacts:
reports:
junit: test-*.xml
before_script:
- - wget $IMAGE_URL -O $TEST_INPUT_IMAGE
- - pip install pytest requests
+ - pip install pytest
+ - pip install .
test_core:
extends: .tests
script:
- - python3 -m pytest --color=yes --junitxml=test-core.xml tests/test_core.py
+ - pytest --color=yes --junitxml=test-core.xml tests/test_core.py
test_numpy:
extends: .tests
script:
- - python3 -m pytest --color=yes --junitxml=test-numpy.xml tests/test_numpy.py
+ - pytest --color=yes --junitxml=test-numpy.xml tests/test_numpy.py
test_pipeline:
- #when: manual
extends: .tests
script:
- - python3 -m pytest --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py
+ - pytest --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py
-# -------------------------------------- Ship ---------------------------------------
+# -------------------------------------- Ship ---------------------------------------
-pages:
+docs:
stage: Ship
- extends: .docs
- only:
- - master
+ when: manual
+ before_script:
+ - apt update && apt install -y python3.8-venv
+ - python3 -m venv docs_venv
+ - source docs_venv/bin/activate
+ - python3 -m pip install -U pip
+ - python3 -m pip install -r doc/doc_requirements.txt
script:
- mkdocs build --site-dir public
artifacts:
@@ -137,11 +114,15 @@ pages:
pypi:
stage: Ship
+ when: manual
only:
- - master
+ - tags
+ - master
+ - develop
+
before_script:
- - apt update && apt install -y python3.8-venv
- - python3 -m pip install --upgrade build twine
+ - apt update && apt install -y python3.8-venv
+ - pip install --upgrade build twine
script:
- - python3 -m build
- - python3 -m twine upload --repository-url https://upload.pypi.org/legacy/ --non-interactive -u __token__ -p $pypi_token dist/*
+ - python3 -m build
+ - python3 -m twine upload --non-interactive -u __token__ -p $pypi_token dist/*
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index c9fe258..65943d1 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
"""This module provides convenient python wrapping of otbApplications."""
-__version__ = "2.0.0"
+__version__ = "2.0.0.dev1"
from .helpers import logger, set_logger_level
from .apps import *
diff --git a/setup.py b/setup.py
deleted file mode 100644
index 6068493..0000000
--- a/setup.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from setuptools import setup
-
-setup()
--
GitLab
From 110c28c6106b3764dc547ac08e9374ab165a04eb Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 1 Jun 2023 19:14:08 +0000
Subject: [PATCH 195/399] Change CI trigger rule with branch name OR merge
request event
---
.gitlab-ci.yml | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4d53e04..77cfdca 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,7 +14,7 @@ cache:
workflow:
rules:
- - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+ - if: $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "develop" || $CI_PIPELINE_SOURCE == "merge_request_event"
stages:
- Static Analysis
@@ -99,7 +99,10 @@ test_pipeline:
docs:
stage: Ship
- when: manual
+ #when: manual
+ only:
+ - master
+ - develop
before_script:
- apt update && apt install -y python3.8-venv
- python3 -m venv docs_venv
@@ -114,9 +117,8 @@ docs:
pypi:
stage: Ship
- when: manual
+ #when: manual
only:
- - tags
- master
- develop
--
GitLab
From afc8290e9eac6e8c0adc3655e5912f1608c79638 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 1 Jun 2023 21:25:39 +0000
Subject: [PATCH 196/399] Resolve "CI trigger is not right"
---
.gitlab-ci.yml | 18 ++++++++++--------
pyotb/__init__.py | 2 +-
2 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 77cfdca..dea3721 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,7 +14,11 @@ cache:
workflow:
rules:
- - if: $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME == "develop" || $CI_PIPELINE_SOURCE == "merge_request_event"
+ - if: $CI_COMMIT_TAG
+ - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+ - if: $CI_COMMIT_REF_PROTECTED && $CI_OPEN_MERGE_REQUESTS
+ when: never
+ - if: $CI_COMMIT_REF_PROTECTED
stages:
- Static Analysis
@@ -99,10 +103,9 @@ test_pipeline:
docs:
stage: Ship
- #when: manual
+ when: manual
only:
- - master
- - develop
+ - tags
before_script:
- apt update && apt install -y python3.8-venv
- python3 -m venv docs_venv
@@ -117,14 +120,13 @@ docs:
pypi:
stage: Ship
- #when: manual
+ when: manual
only:
- - master
- - develop
+ - tags
before_script:
- apt update && apt install -y python3.8-venv
- - pip install --upgrade build twine
+ - pip install build twine
script:
- python3 -m build
- python3 -m twine upload --non-interactive -u __token__ -p $pypi_token dist/*
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index 65943d1..5191fd6 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
"""This module provides convenient python wrapping of otbApplications."""
-__version__ = "2.0.0.dev1"
+__version__ = "2.0.0.dev2"
from .helpers import logger, set_logger_level
from .apps import *
--
GitLab
From 75385e348d279cd4268052364a8be3cecee21d68 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 1 Jun 2023 23:51:45 +0200
Subject: [PATCH 197/399] CI: test token
---
.gitlab-ci.yml | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index dea3721..22b7127 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,11 +14,9 @@ cache:
workflow:
rules:
- - if: $CI_COMMIT_TAG
- - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+ - if: $CI_COMMIT_REF_PROTECTED|| $CI_COMMIT_TAG || $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_COMMIT_REF_PROTECTED && $CI_OPEN_MERGE_REQUESTS
when: never
- - if: $CI_COMMIT_REF_PROTECTED
stages:
- Static Analysis
--
GitLab
From 77f3e7c3b873522dfa53fb30057f8e9bdc711ea7 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 1 Jun 2023 23:54:37 +0200
Subject: [PATCH 198/399] CI: test rules on protected branch
---
.gitlab-ci.yml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 22b7127..dea3721 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,9 +14,11 @@ cache:
workflow:
rules:
- - if: $CI_COMMIT_REF_PROTECTED|| $CI_COMMIT_TAG || $CI_PIPELINE_SOURCE == "merge_request_event"
+ - if: $CI_COMMIT_TAG
+ - if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_COMMIT_REF_PROTECTED && $CI_OPEN_MERGE_REQUESTS
when: never
+ - if: $CI_COMMIT_REF_PROTECTED
stages:
- Static Analysis
--
GitLab
From f372db40498cbc49409930477ac93bf679ed058c Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 2 Jun 2023 00:00:54 +0200
Subject: [PATCH 199/399] CI: auto ship on tags, build docs only for protected
branch
---
.gitlab-ci.yml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index dea3721..39da3b0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -103,9 +103,9 @@ test_pipeline:
docs:
stage: Ship
- when: manual
only:
- - tags
+ - master
+ - develop
before_script:
- apt update && apt install -y python3.8-venv
- python3 -m venv docs_venv
@@ -120,7 +120,7 @@ docs:
pypi:
stage: Ship
- when: manual
+ # when: manual
only:
- tags
--
GitLab
From 8549252ea645e773d2cff2c0d9374208a457a147 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 2 Jun 2023 11:52:38 +0200
Subject: [PATCH 200/399] ENH: pixel_type is 2d argument in write()
---
pyotb/core.py | 2 +-
tests/test_core.py | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index dfe2598..06da0c7 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -729,9 +729,9 @@ class App(OTBObject):
def write(
self,
path: str | Path | dict[str, str] = None,
- ext_fname: str = "",
pixel_type: dict[str, str] | str = None,
preserve_dtype: bool = False,
+ ext_fname: str = "",
**kwargs,
) -> bool:
"""Set output pixel type and write the output raster files.
diff --git a/tests/test_core.py b/tests/test_core.py
index f766c58..99b21de 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -246,8 +246,8 @@ def test_ndvi_comparison():
ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (INPUT[:, :, -1] + INPUT[:, :, 0])
ndvi_indices = pyotb.RadiometricIndices(INPUT, {"list": ["Vegetation:NDVI"], "channels.red": 1, "channels.nir": 4})
assert ndvi_bandmath.exp == "((im1b4 - im1b1) / (im1b4 + im1b1))"
- assert ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", pixel_type="float")
- assert ndvi_indices.write("/tmp/ndvi_indices.tif", pixel_type="float")
+ assert ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", "float")
+ assert ndvi_indices.write("/tmp/ndvi_indices.tif", "float")
compared = pyotb.CompareImages({"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"})
assert (compared["count"], compared["mse"]) == (0, 0)
--
GitLab
From 814efc6714058e7ad9add98af1df58532062f31f Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 2 Jun 2023 11:54:16 +0200
Subject: [PATCH 201/399] DOC: update README since pixel_type kwarg is not
required
---
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.md b/README.md
index d72b294..5447959 100644
--- a/README.md
+++ b/README.md
@@ -43,7 +43,7 @@ roi = ndvi[20:586, 9:572]
# Pipeline execution
# The actual computation happens here !
-roi.write("output.tif", pixel_type="float")
+roi.write("output.tif", "float")
```
pyotb's objects also enable easy interoperability with
--
GitLab
From 0b5a84e8c5c70798c74b396d1811155d787fb577 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 6 Jun 2023 08:02:23 +0000
Subject: [PATCH 202/399] CI: last changes since workflow update
---
.gitlab-ci.yml | 28 ++++++++++++++--------------
doc/examples/nodata_mean.md | 1 -
doc/installation.md | 5 +----
3 files changed, 15 insertions(+), 19 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 39da3b0..49211fa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -23,6 +23,7 @@ workflow:
stages:
- Static Analysis
- Tests
+ - Documentation
- Ship
# -------------------------------- Static analysis --------------------------------
@@ -52,11 +53,8 @@ codespell:
extends: .static_analysis
rules:
- changes:
- - .gitlab-ci.yml
- - pyotb/*.py
- - tests/*.py
- - doc/*.md
- - README.md
+ - "**/*.py"
+ - "**/*.md"
before_script:
- pip install codespell
script:
@@ -69,9 +67,7 @@ codespell:
allow_failure: false
rules:
- changes:
- - .gitlab-ci.yml
- - pyotb/*.py
- - tests/*.py
+ - "**/*.py"
variables:
OTB_ROOT: /opt/otb
LD_LIBRARY_PATH: /opt/otb/lib
@@ -99,13 +95,16 @@ test_pipeline:
script:
- pytest --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py
-# -------------------------------------- Ship ---------------------------------------
+# -------------------------------------- Docs ---------------------------------------
docs:
- stage: Ship
- only:
- - master
- - develop
+ stage: Documentation
+ # when: manual
+ rules:
+ - changes:
+ - mkdocs.yml
+ - doc/*
+ - pyotb/*.py
before_script:
- apt update && apt install -y python3.8-venv
- python3 -m venv docs_venv
@@ -118,12 +117,13 @@ docs:
paths:
- public
+# -------------------------------------- Ship ---------------------------------------
+
pypi:
stage: Ship
# when: manual
only:
- tags
-
before_script:
- apt update && apt install -y python3.8-venv
- pip install build twine
diff --git a/doc/examples/nodata_mean.md b/doc/examples/nodata_mean.md
index 9ec4900..d3818ce 100644
--- a/doc/examples/nodata_mean.md
+++ b/doc/examples/nodata_mean.md
@@ -23,4 +23,3 @@ mean.write('ndvi_annual_mean.tif')
```
Note that no actual computation is executed before the last line where the result is written to disk.
-
diff --git a/doc/installation.md b/doc/installation.md
index de5c0ae..404d7bd 100644
--- a/doc/installation.md
+++ b/doc/installation.md
@@ -6,7 +6,7 @@ Requirements:
- Orfeo ToolBox binaries (follow these
[instructions](https://www.orfeo-toolbox.org/CookBook/Installation.html))
- Orfeo ToolBox python binding (follow these
- [instructions](https://www.orfeo-toolbox.org/CookBook/Installation.html))
+ [instructions](https://www.orfeo-toolbox.org/CookBook/Installation.html#python-bindings))
## Install with pip
@@ -26,6 +26,3 @@ pip install -e ".[dev]"
If you need compatibility with python3.6, install `pyotb<2.0` and for
python3.5 use `pyotb==1.2.2`.
-
-For Python>=3.6, latest version available is pyotb 1.5.0.
-For Python 3.5, latest version available is pyotb 1.2.2
--
GitLab
From 0b1ae1e0a5d2c1e455a0f235bfe1cbbb18d0f1c1 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 27 Jun 2023 16:04:58 +0200
Subject: [PATCH 203/399] DOC: Lot of docfixes
---
doc/MISC.md | 20 ++++--
doc/comparison_otb.md | 135 +++++++++++++++++++++++++++---------
doc/examples/nodata_mean.md | 20 ++++--
doc/examples/pleiades.md | 31 +++++++--
doc/features.md | 10 ++-
doc/functions.md | 57 ++++++++++-----
doc/index.md | 9 ++-
doc/interaction.md | 79 +++++++++++++++------
doc/managing_loggers.md | 15 ++--
doc/otb_versions.md | 12 ++--
doc/quickstart.md | 91 ++++++++++++++++++------
doc/troubleshooting.md | 18 +++--
12 files changed, 365 insertions(+), 132 deletions(-)
diff --git a/doc/MISC.md b/doc/MISC.md
index 2a92b03..6be696f 100644
--- a/doc/MISC.md
+++ b/doc/MISC.md
@@ -1,10 +1,12 @@
## Miscellaneous: Work with images with different footprints / resolutions
-OrfeoToolBox provides a handy `Superimpose` application that enables the projection of an image into the geometry of another one.
+OrfeoToolBox provides a handy `Superimpose` application that enables the
+projection of an image into the geometry of another one.
In pyotb, a function has been created to handle more than 2 images.
-Let's consider the case where we have 3 images with different resolutions and different footprints :
+Let's consider the case where we have 3 images with different resolutions and
+different footprints :
@@ -12,7 +14,9 @@ Let's consider the case where we have 3 images with different resolutions and di
import pyotb
# transforming filepaths to pyotb objects
-s2_image, vhr_image, labels = pyotb.Input('image_10m.tif'), pyotb.Input('image_60cm.tif'), pyotb.Input('land_cover_2m.tif')
+s2_image = pyotb.Input('image_10m.tif')
+vhr_image = pyotb.Input('image_60cm.tif')
+labels = pyotb.Input('land_cover_2m.tif')
print(s2_image.shape) # (286, 195, 4)
print(vhr_image.shape) # (2048, 2048, 3)
@@ -30,9 +34,13 @@ Here is the final result :
The piece of code to achieve this :
```python
-s2_image, vhr_image, labels = pyotb.define_processing_area(s2_image, vhr_image, labels, window_rule='intersection',
- pixel_size_rule='same_as_input',
- reference_pixel_size_input=labels, interpolator='bco')
+s2_image, vhr_image, labels = pyotb.define_processing_area(
+ s2_image, vhr_image, labels,
+ window_rule='intersection',
+ pixel_size_rule='same_as_input',
+ reference_pixel_size_input=labels,
+ interpolator='bco'
+)
print(s2_image.shape) # (657, 520, 4)
print(vhr_image.shape) # (657, 520, 3)
diff --git a/doc/comparison_otb.md b/doc/comparison_otb.md
index a733678..29d1e89 100644
--- a/doc/comparison_otb.md
+++ b/doc/comparison_otb.md
@@ -2,17 +2,31 @@
### Single application execution
-Using OTB, the code would be like :
+<table>
+<tr>
+<th> OTB </th>
+<th> pyotb </th>
+</tr>
+<tr>
+<td>
```python
import otbApplication
input_path = 'my_image.tif'
-resampled = otbApplication.Registry.CreateApplication('RigidTransformResample')
+resampled = otbApplication.Registry.CreateApplication(
+ 'RigidTransformResample'
+)
resampled.SetParameterString('in', input_path)
resampled.SetParameterString('interpolator', 'linear')
-resampled.SetParameterFloat('transform.type.id.scalex', 0.5)
-resampled.SetParameterFloat('transform.type.id.scaley', 0.5)
+resampled.SetParameterFloat(
+ 'transform.type.id.scalex',
+ 0.5
+)
+resampled.SetParameterFloat(
+ 'transform.type.id.scaley',
+ 0.5
+)
resampled.SetParameterString('out', 'output.tif')
resampled.SetParameterOutputImagePixelType(
'out', otbApplication.ImagePixelType_uint16
@@ -21,7 +35,8 @@ resampled.SetParameterOutputImagePixelType(
resampled.ExecuteAndWriteOutput()
```
-Using pyotb:
+</td>
+<td>
```python
import pyotb
@@ -36,6 +51,10 @@ resampled = pyotb.RigidTransformResample({
resampled.write('output.tif', pixel_type='uint16')
```
+</td>
+</tr>
+</table>
+
### In-memory connections
<table>
@@ -126,61 +145,105 @@ app3.write(
### Arithmetic operations
-Every pyotb object supports arithmetic operations, such as addition, subtraction, comparison...
-Consider an example where we want to perform the arithmetic operation `image1 * image2 - 2*image3`
+Every pyotb object supports arithmetic operations, such as addition,
+subtraction, comparison...
+Consider an example where we want to perform the arithmetic operation
+`image1 * image2 - 2*image3`.
-Using OTB, the following code works for 3-bands images :
+<table>
+<tr>
+<th> OTB </th>
+<th> pyotb </th>
+</tr>
+<tr>
+<td>
```python
import otbApplication
-bmx = otbApplication.Registry.CreateApplication('BandMathX')
-bmx.SetParameterStringList('il', ['image1.tif', 'image2.tif', 'image3.tif']) # all images are 3-bands
-exp = 'im1b1*im2b1 - 2*im3b1; im1b2*im2b2 - 2*im3b2; im1b3*im2b3 - 2*im3b3'
+bmx = otbApplication.Registry.CreateApplication(
+ 'BandMathX'
+)
+bmx.SetParameterStringList(
+ 'il',
+ ['image1.tif', 'image2.tif', 'image3.tif']
+) # all images are 3-bands
+exp = ('im1b1*im2b1-2*im3b1; '
+ 'im1b2*im2b2-2*im3b2; '
+ 'im1b3*im2b3-2*im3b3')
bmx.SetParameterString('exp', exp)
bmx.SetParameterString('out', 'output.tif')
-bmx.SetParameterOutputImagePixelType('out', otbApplication.ImagePixelType_uint8)
+bmx.SetParameterOutputImagePixelType(
+ 'out',
+ otbApplication.ImagePixelType_uint8
+)
bmx.ExecuteAndWriteOutput()
```
-With pyotb, the following works with images of any number of bands :
+In OTB, this code works for 3-bands images.
+
+</td>
+<td>
```python
import pyotb
# transforming filepaths to pyotb objects
-input1, input2, input3 = pyotb.Input('image1.tif'), pyotb.Input('image2.tif') , pyotb.Input('image3.tif')
+input1 = pyotb.Input('image1.tif')
+input2 = pyotb.Input('image2.tif')
+input3 = pyotb.Input('image3.tif')
res = input1 * input2 - 2 * input2
res.write('output.tif', pixel_type='uint8')
```
+In pyotb,this code works with images of any number of bands.
+
+</td>
+</tr>
+</table>
+
### Slicing
-Using OTB, for selection bands or ROI, the code looks like:
+<table>
+<tr>
+<th> OTB </th>
+<th> pyotb </th>
+</tr>
+<tr>
+<td>
+
```python
import otbApplication
-# selecting first 3 bands
-extracted = otbApplication.Registry.CreateApplication('ExtractROI')
-extracted.SetParameterString('in', 'my_image.tif')
-extracted.SetParameterStringList('cl', ['Channel1', 'Channel2', 'Channel3'])
-extracted.Execute()
-
-# selecting 1000x1000 subset
-extracted = otbApplication.Registry.CreateApplication('ExtractROI')
-extracted.SetParameterString('in', 'my_image.tif')
-extracted.SetParameterString('mode', 'extent')
-extracted.SetParameterString('mode.extent.unit', 'pxl')
-extracted.SetParameterFloat('mode.extent.ulx', 0)
-extracted.SetParameterFloat('mode.extent.uly', 0)
-extracted.SetParameterFloat('mode.extent.lrx', 999)
-extracted.SetParameterFloat('mode.extent.lry', 999)
-extracted.Execute()
+# first 3 channels
+app = otbApplication.Registry.CreateApplication(
+ 'ExtractROI'
+)
+app.SetParameterString('in', 'my_image.tif')
+app.SetParameterStringList(
+ 'cl',
+ ['Channel1', 'Channel2', 'Channel3']
+)
+app.Execute()
+
+# 1000x1000 roi
+app = otbApplication.Registry.CreateApplication(
+ 'ExtractROI'
+)
+app.SetParameterString('in', 'my_image.tif')
+app.SetParameterString('mode', 'extent')
+app.SetParameterString('mode.extent.unit', 'pxl')
+app.SetParameterFloat('mode.extent.ulx', 0)
+app.SetParameterFloat('mode.extent.uly', 0)
+app.SetParameterFloat('mode.extent.lrx', 999)
+app.SetParameterFloat('mode.extent.lry', 999)
+app.Execute()
```
-Instead, using pyotb:
+</td>
+<td>
```python
import pyotb
@@ -188,6 +251,10 @@ import pyotb
# transforming filepath to pyotb object
inp = pyotb.Input('my_image.tif')
-extracted = inp[:, :, :3] # selecting first 3 bands
-extracted = inp[:1000, :1000] # selecting 1000x1000 subset
+extracted = inp[:, :, :3] # first 3 channels
+extracted = inp[:1000, :1000] # 1000x1000 roi
```
+
+</td>
+</tr>
+</table>
\ No newline at end of file
diff --git a/doc/examples/nodata_mean.md b/doc/examples/nodata_mean.md
index d3818ce..288be6e 100644
--- a/doc/examples/nodata_mean.md
+++ b/doc/examples/nodata_mean.md
@@ -1,7 +1,9 @@
### Compute the mean of several rasters, taking into account NoData
-Let's consider we have at disposal 73 NDVI rasters for a year, where clouds have been masked with NoData (nodata value of -10 000 for example).
+Let's consider we have at disposal 73 NDVI rasters for a year, where clouds
+have been masked with NoData (nodata value of -10 000 for example).
-Goal: compute the mean across time (keeping the spatial dimension) of the NDVIs, excluding cloudy pixels. Piece of code to achieve that:
+Goal: compute the mean across time (keeping the spatial dimension) of the
+NDVIs, excluding cloudy pixels. Piece of code to achieve that:
```python
import pyotb
@@ -13,13 +15,21 @@ ndvis = [pyotb.Input(path) for path in ndvi_paths]
summed = sum([pyotb.where(ndvi != nodata, ndvi, 0) for ndvi in ndvis])
# Printing the generated BandMath expression
-print(summed.exp) # this returns a very long exp: "0 + ((im1b1 != -10000) ? im1b1 : 0) + ((im2b1 != -10000) ? im2b1 : 0) + ... + ((im73b1 != -10000) ? im73b1 : 0)"
+print(summed.exp)
+# this returns a very long exp:
+# "0 + ((im1b1 != -10000) ? im1b1 : 0) + ((im2b1 != -10000) ? im2b1 : 0) + ...
+# ... + ((im73b1 != -10000) ? im73b1 : 0)"
# For each pixel location, getting the count of valid pixels
count = sum([pyotb.where(ndvi == nodata, 0, 1) for ndvi in ndvis])
-mean = summed / count # BandMath exp of this is very long: "(0 + ((im1b1 != -10000) ? im1b1 : 0) + ... + ((im73b1 != -10000) ? im73b1 : 0)) / (0 + ((im1b1 == -10000) ? 0 : 1) + ... + ((im73b1 == -10000) ? 0 : 1))"
+mean = summed / count
+# BandMath exp of this is very long:
+# "(0 + ((im1b1 != -10000) ? im1b1 : 0) + ...
+# + ((im73b1 != -10000) ? im73b1 : 0)) / (0 + ((im1b1 == -10000) ? 0 : 1) + ...
+# + ((im73b1 == -10000) ? 0 : 1))"
mean.write('ndvi_annual_mean.tif')
```
-Note that no actual computation is executed before the last line where the result is written to disk.
+Note that no actual computation is executed before the last line where the
+result is written to disk.
diff --git a/doc/examples/pleiades.md b/doc/examples/pleiades.md
index 32d0447..eb3f1f2 100644
--- a/doc/examples/pleiades.md
+++ b/doc/examples/pleiades.md
@@ -6,15 +6,34 @@ import pyotb
srtm = '/media/data/raster/nasa/srtm_30m'
geoid = '/media/data/geoid/egm96.grd'
-pan = pyotb.OpticalCalibration('IMG_PHR1A_P_001/DIM_PHR1A_P_201509011347379_SEN_1791374101-001.XML', level='toa')
-ms = pyotb.OpticalCalibration('IMG_PHR1A_MS_002/DIM_PHR1A_MS_201509011347379_SEN_1791374101-002.XML', level='toa')
+pan = pyotb.OpticalCalibration(
+ 'IMG_PHR1A_P_001/DIM_PHR1A_P_201509011347379_SEN_1791374101-001.XML',
+ level='toa'
+)
+ms = pyotb.OpticalCalibration(
+ 'IMG_PHR1A_MS_002/DIM_PHR1A_MS_201509011347379_SEN_1791374101-002.XML',
+ level='toa'
+)
-pan_ortho = pyotb.OrthoRectification({'io.in': pan, 'elev.dem': srtm, 'elev.geoid': geoid})
-ms_ortho = pyotb.OrthoRectification({'io.in': ms, 'elev.dem': srtm, 'elev.geoid': geoid})
+pan_ortho = pyotb.OrthoRectification({
+ 'io.in': pan,
+ 'elev.dem': srtm,
+ 'elev.geoid': geoid
+})
+ms_ortho = pyotb.OrthoRectification({
+ 'io.in': ms,
+ 'elev.dem': srtm,
+ 'elev.geoid': geoid
+})
-pxs = pyotb.BundleToPerfectSensor(inp=pan_ortho, inxs=ms_ortho, method='bayes', mode='default')
+pxs = pyotb.BundleToPerfectSensor(
+ inp=pan_ortho,
+ inxs=ms_ortho,
+ method='bayes',
+ mode='default'
+)
exfn = '?gdal:co:COMPRESS=DEFLATE&gdal:co:PREDICTOR=2&gdal:co:BIGTIFF=YES'
# Here we trigger every app in the pipeline and the process is blocked until result is written to disk
-pxs.write('pxs_image.tif', pixel_type='uint16', filename_extension=exfn)
+pxs.write('pxs_image.tif', pixel_type='uint16', ext_fname=exfn)
```
diff --git a/doc/features.md b/doc/features.md
index cc2c39e..2058af5 100644
--- a/doc/features.md
+++ b/doc/features.md
@@ -1,7 +1,9 @@
## Arithmetic operations
-Every pyotb object supports arithmetic operations, such as addition, subtraction, comparison...
-Consider an example where we want to compute a vegeteation mask from NDVI, i.e. the arithmetic operation `(nir - red) / (nir + red) > 0.3`
+Every pyotb object supports arithmetic operations, such as addition,
+subtraction, comparison...
+Consider an example where we want to compute a vegeteation mask from NDVI,
+i.e. the arithmetic operation `(nir - red) / (nir + red) > 0.3`
With pyotb, one can simply do :
@@ -12,7 +14,9 @@ import pyotb
nir, red = pyotb.Input('nir.tif'), pyotb.Input('red.tif')
res = (nir - red) / (nir + red) > 0.3
-print(res.exp) # prints the BandMath expression: "((im1b1 - im2b1) / (im1b1 + im2b1)) > 0.3 ? 1 : 0"
+# Prints the BandMath expression:
+# "((im1b1 - im2b1) / (im1b1 + im2b1)) > 0.3 ? 1 : 0"
+print(res.exp)
res.write('vegetation_mask.tif', pixel_type='uint8')
```
diff --git a/doc/functions.md b/doc/functions.md
index 070ba0b..1e82c58 100644
--- a/doc/functions.md
+++ b/doc/functions.md
@@ -1,26 +1,47 @@
-Some functions have been written, entirely based on OTB, to mimic the behavior of some well-known numpy functions.
+Some functions have been written, entirely based on OTB, to mimic the behavior
+of some well-known numpy functions.
+
## pyotb.where
+
Equivalent of `numpy.where`.
-It is the equivalent of the muparser syntax `condition ? x : y` that can be used in OTB's BandMath.
+It is the equivalent of the muparser syntax `condition ? x : y` that can be
+used in OTB's BandMath.
```python
import pyotb
# transforming filepaths to pyotb objects
-labels, image1, image2 = pyotb.Input('labels.tif'), pyotb.Input('image1.tif') , pyotb.Input('image2.tif')
+labels = pyotb.Input('labels.tif')
+image1 = pyotb.Input('image1.tif')
+image2 = pyotb.Input('image2.tif')
# If labels = 1, returns image1. Else, returns image2
-res = pyotb.where(labels == 1, image1, image2) # this would also work: pyotb.where(labels == 1, 'image1.tif', 'image2.tif')
+res = pyotb.where(labels == 1, image1, image2)
+# this would also work: `pyotb.where(labels == 1, 'image1.tif', 'image2.tif')`
-# A more complex example
-# If labels = 1, returns image1. If labels = 2, returns image2. If labels = 3, returns 3. Else 0
-res = pyotb.where(labels == 1, image1,
- pyotb.where(labels == 2, image2,
- pyotb.where(labels == 3, 3, 0)))
+# A more complex example:
+# - If labels = 1 --> returns image1,
+# - If labels = 2 --> returns image2,
+# - If labels = 3 --> returns 3.0,
+# - Else, returns 0.0
+res = pyotb.where(
+ labels == 1,
+ image1,
+ pyotb.where(
+ labels == 2,
+ image2,
+ pyotb.where(
+ labels == 3,
+ 3.0,
+ 0.0
+ )
+ )
+)
```
## pyotb.clip
+
Equivalent of `numpy.clip`. Clip (limit) the values in a raster to a range.
```python
@@ -30,18 +51,20 @@ res = pyotb.clip('my_image.tif', 0, 255) # clips the values between 0 and 255
```
## pyotb.all
+
Equivalent of `numpy.all`.
-For only one image, this function checks that all bands of the image are True (i.e. !=0) and outputs
-a singleband boolean raster.
-For several images, this function checks that all images are True (i.e. !=0) and outputs
-a boolean raster, with as many bands as the inputs.
+For only one image, this function checks that all bands of the image are True
+(i.e. !=0) and outputs a single band boolean raster.
+For several images, this function checks that all images are True (i.e. !=0)
+and outputs a boolean raster, with as many bands as the inputs.
## pyotb.any
+
Equivalent of `numpy.any`.
-For only one image, this function checks that at least one band of the image is True (i.e. !=0) and outputs
-a singleband boolean raster.
-For several images, this function checks that at least one of the images is True (i.e. !=0) and outputs
-a boolean raster, with as many bands as the inputs.
\ No newline at end of file
+For only one image, this function checks that at least one band of the image
+is True (i.e. !=0) and outputs a single band boolean raster.
+For several images, this function checks that at least one of the images is
+True (i.e. !=0) and outputs a boolean raster, with as many bands as the inputs.
\ No newline at end of file
diff --git a/doc/index.md b/doc/index.md
index fa29918..448d139 100644
--- a/doc/index.md
+++ b/doc/index.md
@@ -1,11 +1,13 @@
# Pyotb: Orfeo Toolbox for Python
-pyotb is a Python extension of Orfeo Toolbox. It has been built on top of the existing Python API of OTB, in order
+pyotb is a Python extension of Orfeo Toolbox. It has been built on top of the
+existing Python API of OTB, in order
to make OTB more Python friendly.
# Table of Contents
## Get started
+
- [Installation](installation.md)
- [How to use pyotb](quickstart.md)
- [Useful features](features.md)
@@ -13,10 +15,12 @@ to make OTB more Python friendly.
- [Interaction with Python libraries (numpy, rasterio, tensorflow)](interaction.md)
## Examples
+
- [Pleiades data processing](examples/pleiades.md)
- [Computing the mean of several rasters with NoData](examples/nodata_mean.md)
## Advanced use
+
- [Comparison between pyotb and OTB native library](comparison_otb.md)
- [OTB versions](otb_versions.md)
- [Managing loggers](managing_loggers.md)
@@ -24,3 +28,6 @@ to make OTB more Python friendly.
## API
+
+- See the API reference. If you have any doubts or questions, feel free to ask
+on github or gitlab!
\ No newline at end of file
diff --git a/doc/interaction.md b/doc/interaction.md
index 217dbaa..45da657 100644
--- a/doc/interaction.md
+++ b/doc/interaction.md
@@ -6,8 +6,14 @@ pyotb objects can be exported to numpy array.
import pyotb
import numpy as np
-calibrated = pyotb.OpticalCalibration('image.tif', level='toa') # this is a pyotb object
-arr = np.asarray(calibrated) # same as calibrated.to_numpy()
+# The following is a pyotb object
+calibrated = pyotb.OpticalCalibration('image.tif', level='toa')
+
+# The following is a numpy array
+arr = np.asarray(calibrated)
+
+# Note that the following is equivalent:
+arr = calibrated.to_numpy()
```
## Interaction with Numpy
@@ -20,21 +26,26 @@ For example:
import pyotb
import numpy as np
-inp = pyotb.Input('image.tif') # this is a pyotb object
+# The following is a pyotb object
+inp = pyotb.Input('image.tif')
-# Creating a numpy array of noise
-white_noise = np.random.normal(0, 50, size=inp.shape) # this is a numpy object
+# Creating a numpy array of noise. The following is a numpy object
+white_noise = np.random.normal(0, 50, size=inp.shape)
# Adding the noise to the image
-noisy_image = inp + white_noise # magic: this is a pyotb object that has the same georeference as input.
- # `np.add(inp, white_noise)` would have worked the same
+noisy_image = inp + white_noise
+# Magically, this is a pyotb object that has the same geo-reference as `inp`.
+# Note the `np.add(inp, white_noise)` would have worked the same
+
+# Finally we can write the result like any pyotb object
noisy_image.write('image_plus_noise.tif')
```
Limitations :
- The whole image is loaded into memory
-- The georeference can not be modified. Thus, numpy operations can not change the image or pixel size
+- The georeference can not be modified. Thus, numpy operations can not change
+the image or pixel size
## Export to rasterio
@@ -48,15 +59,23 @@ import rasterio
from scipy import ndimage
# Pansharpening + NDVI + creating bare soils mask
-pxs = pyotb.BundleToPerfectSensor(inp='panchromatic.tif', inxs='multispectral.tif')
-ndvi = pyotb.RadiometricIndices({'in': pxs, 'channels.red': 3, 'channels.nir': 4, 'list': 'Vegetation:NDVI'})
+pxs = pyotb.BundleToPerfectSensor(
+ inp='panchromatic.tif',
+ inxs='multispectral.tif'
+)
+ndvi = pyotb.RadiometricIndices({
+ 'in': pxs,
+ 'channels.red': 3,
+ 'channels.nir': 4,
+ 'list': 'Vegetation:NDVI'
+})
bare_soil_mask = (ndvi < 0.3)
# Exporting the result as array & profile usable by rasterio
mask_array, profile = bare_soil_mask.to_rasterio()
-# Doing something in Python that is not possible with OTB, e.g. gathering the contiguous groups of pixels
-# with an integer index
+# Doing something in Python that is not possible with OTB, e.g. gathering
+# the contiguous groups of pixels with an integer index
labeled_mask_array, nb_groups = ndimage.label(mask_array)
# Writing the result to disk
@@ -64,14 +83,18 @@ with rasterio.open('labeled_bare_soil.tif', 'w', **profile) as f:
f.write(labeled_mask_array)
```
-This way of exporting pyotb objects is more flexible that exporting to numpy, as the user gets the `profile` dictionary.
-If the georeference or pixel size is modified, the user can update the `profile` accordingly.
+This way of exporting pyotb objects is more flexible that exporting to numpy,
+as the user gets the `profile` dictionary.
+If the georeference or pixel size is modified, the user can update the
+`profile` accordingly.
## Interaction with Tensorflow
-We saw that numpy operations had some limitations. To bypass those limitations, it is possible to use some Tensorflow operations on pyotb objects.
+We saw that numpy operations had some limitations. To bypass those
+limitations, it is possible to use some Tensorflow operations on pyotb objects.
-You need a working installation of OTBTF >=3.0 for this and then the code is like this:
+You need a working installation of OTBTF >=3.0 for this and then the code is
+like this:
```python
import pyotb
@@ -82,29 +105,39 @@ def scalar_product(x1, x2):
return tf.reduce_sum(tf.multiply(x1, x2), axis=-1)
# Compute the scalar product
-res = pyotb.run_tf_function(scalar_product)('image1.tif', 'image2.tif') # magic: this is a pyotb object
+res = pyotb.run_tf_function(scalar_product)('image1.tif', 'image2.tif')
+
+# Magically, `res` is a pyotb object
res.write('scalar_product.tif')
```
-For some easy syntax, one can use `pyotb.run_tf_function` as a function decorator, such as:
+For some easy syntax, one can use `pyotb.run_tf_function` as a function
+decorator, such as:
```python
import pyotb
-@pyotb.run_tf_function # The decorator enables the use of pyotb objects as inputs/output of the function
+# The `pyotb.run_tf_function` decorator enables the use of pyotb objects as
+# inputs/output of the function
+@pyotb.run_tf_function
def scalar_product(x1, x2):
import tensorflow as tf
return tf.reduce_sum(tf.multiply(x1, x2), axis=-1)
-res = scalar_product('image1.tif', 'image2.tif') # magic: this is a pyotb object
+res = scalar_product('image1.tif', 'image2.tif')
+# Magically, `res` is a pyotb object
```
Advantages :
-- The process supports streaming, hence the whole image is **not** loaded into memory
+- The process supports streaming, hence the whole image is **not** loaded into
+memory
- Can be integrated in OTB pipelines
Limitations :
-- It is not possible to use the tensorflow python API inside a script where OTBTF is used because of compilation issues
- between Tensorflow and OTBTF, i.e. `import tensorflow` doesn't work in a script where OTBTF apps have been initialized
+- For OTBTF versions < 4.0.0, it is not possible to use the tensorflow python
+API inside a script where OTBTF is used because of libraries clashing between
+Tensorflow and OTBTF, i.e. `import tensorflow` doesn't work in a script where
+OTBTF apps have been initialized. This is why we recommend to use latest OTBTF
+versions
diff --git a/doc/managing_loggers.md b/doc/managing_loggers.md
index 31c960b..5c2d80b 100644
--- a/doc/managing_loggers.md
+++ b/doc/managing_loggers.md
@@ -1,22 +1,27 @@
## Managing loggers
-Several environment variables are used in order to adjust logger level and behaviour. It should be set before importing pyotb.
+Several environment variables are used in order to adjust logger level and
+behaviour. It should be set before importing pyotb.
- `OTB_LOGGER_LEVEL` : used to set the default OTB logger level.
-- `PYOTB_LOGGER_LEVEL` : used to set the pyotb logger level. if not set, `OTB_LOGGER_LEVEL` will be used.
+- `PYOTB_LOGGER_LEVEL` : used to set the pyotb logger level. if not set,
+- `OTB_LOGGER_LEVEL` will be used.
If none of those two variables is set, the logger level will be set to 'INFO'.
Available levels are : DEBUG, INFO, WARNING, ERROR, CRITICAL
-You may also change the logger level after import (for pyotb only) with the function `set_logger_level`.
+You may also change the logger level after import (for pyotb only) with the
+function `set_logger_level`.
```python
import pyotb
pyotb.set_logger_level('DEBUG')
```
-Bonus : in some cases, yo may want to silence the GDAL driver logger (for example you will see a lot of errors when reading GML files with OGR).
-One useful trick is to redirect these logs to a file. This can be done using the variable `CPL_LOG`.
+Bonus : in some cases, yo may want to silence the GDAL driver logger (for
+example you will see a lot of errors when reading GML files with OGR).
+One useful trick is to redirect these logs to a file. This can be done using
+the variable `CPL_LOG`.
## Named applications in logs
diff --git a/doc/otb_versions.md b/doc/otb_versions.md
index 7eeb5cf..50f30ee 100644
--- a/doc/otb_versions.md
+++ b/doc/otb_versions.md
@@ -1,6 +1,7 @@
## System with multiple OTB versions
-If you want to quickly switch between OTB versions, or override the default system version, you may use the `OTB_ROOT` env variable :
+If you want to quickly switch between OTB versions, or override the default
+system version, you may use the `OTB_ROOT` env variable :
```python
import os
@@ -14,7 +15,8 @@ import pyotb
2022-06-14 13:59:04 (INFO) [pyOTB] Successfully loaded 126 OTB applications
```
-If you try to import pyotb without having set environment, it will try to find any OTB version installed on your system:
+If you try to import pyotb without having set environment, it will try to find
+any OTB version installed on your system:
```python
import pyotb
@@ -38,11 +40,13 @@ Here is the path precedence for this automatic env configuration :
OR (for windows) : C:/Program Files
```
-N.B. : in case `otbApplication` is found in `PYTHONPATH` (and if `OTB_ROOT` was not set), the OTB which the python API is linked to will be used.
+N.B. : in case `otbApplication` is found in `PYTHONPATH` (and if `OTB_ROOT`
+was not set), the OTB which the python API is linked to will be used.
## Fresh OTB installation
-If you've just installed OTB binaries in a Linux environment, you may encounter an error at first import, pyotb will help you fix it :
+If you've just installed OTB binaries in a Linux environment, you may
+encounter an error at first import, pyotb will help you fix it :
```python
import pyotb
diff --git a/doc/quickstart.md b/doc/quickstart.md
index 294d512..915b3f3 100644
--- a/doc/quickstart.md
+++ b/doc/quickstart.md
@@ -1,73 +1,120 @@
## Quickstart: running an OTB application with pyotb
-pyotb has been written so that it is more convenient to run an application in Python.
+
+pyotb has been written so that it is more convenient to run an application in
+Python.
You can pass the parameters of an application as a dictionary :
```python
import pyotb
-resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'transform.type.id.scaley': 0.5,
- 'interpolator': 'linear', 'transform.type.id.scalex': 0.5})
+resampled = pyotb.RigidTransformResample({
+ 'in': 'my_image.tif',
+ 'transform.type.id.scaley': 0.5,
+ 'interpolator': 'linear',
+ 'transform.type.id.scalex': 0.5
+})
```
-Note that pyotb has a 'lazy' evaluation: it only performs operation when it is needed, i.e. results are written to disk.
+Note that pyotb has a 'lazy' evaluation: it only performs operation when it is
+needed, i.e. results are written to disk.
Thus, the previous line doesn't trigger the application.
-To actually trigger the application execution, you need to write the result to disk:
+To actually trigger the application execution, you need to write the result to
+disk:
```python
resampled.write('output.tif') # this is when the application actually runs
```
### Using Python keyword arguments
-It is also possible to use the Python keyword arguments notation for passing the parameters:
+
+It is also possible to use the Python keyword arguments notation for passing
+the parameters:
+
```python
output = pyotb.SuperImpose(inr='reference_image.tif', inm='image.tif')
```
+
is equivalent to:
+
```python
output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'})
```
-Limitations : for this notation, python doesn't accept the parameter `in` or any parameter that contains a `.`. E.g., it is not possible to use `pyotb.RigidTransformResample(in=input_path...)` or `pyotb.VectorDataExtractROI(io.vd=vector_path...)`.
+Limitations : for this notation, python doesn't accept the parameter `in` or
+any parameter that contains a `.`. E.g., it is not possible to use
+`pyotb.RigidTransformResample(in=input_path...)` or
+`pyotb.VectorDataExtractROI(io.vd=vector_path...)`.
## In-memory connections
+
The big asset of pyotb is the ease of in-memory connections between apps.
-Let's start from our previous example. Consider the case where one wants to apply optical calibration and binary morphological dilatation
+Let's start from our previous example. Consider the case where one wants to
+apply optical calibration and binary morphological dilatation
following the undersampling.
Using pyotb, you can pass the output of an app as input of another app :
+
```python
import pyotb
-resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear',
- 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5})
-
-calibrated = pyotb.OpticalCalibration({'in': resampled, 'level': 'toa'})
+resampled = pyotb.RigidTransformResample({
+ 'in': 'my_image.tif',
+ 'interpolator': 'linear',
+ 'transform.type.id.scaley': 0.5,
+ 'transform.type.id.scalex': 0.5
+})
+
+calibrated = pyotb.OpticalCalibration({
+ 'in': resampled,
+ 'level': 'toa'
+})
+
+dilated = pyotb.BinaryMorphologicalOperation({
+ 'in': calibrated,
+ 'out': 'output.tif',
+ 'filter': 'dilate',
+ 'structype': 'ball',
+ 'xradius': 3, 'yradius': 3
+})
-dilated = pyotb.BinaryMorphologicalOperation({'in': calibrated, 'out': 'output.tif', 'filter': 'dilate',
- 'structype': 'ball', 'xradius': 3, 'yradius': 3})
dilated.write('result.tif')
```
## Writing the result of an app
+
Any pyotb object can be written to disk using the `write` method, e.g. :
```python
import pyotb
-resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear',
- 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5})
+resampled = pyotb.RigidTransformResample({
+ 'in': 'my_image.tif',
+ 'interpolator': 'linear',
+ 'transform.type.id.scaley': 0.5,
+ 'transform.type.id.scalex': 0.5
+})
+
# Here you can set optionally pixel type and extended filename variables
-resampled.write({'out': 'output.tif'}, pixel_type='uint16', filename_extension='?nodata=65535')
+resampled.write(
+ {'out': 'output.tif'},
+ pixel_type='uint16',
+ ext_fname='?nodata=65535'
+)
```
-Another possibility for writing results is to set the output parameter when initializing the application:
+Another possibility for writing results is to set the output parameter when
+initializing the application:
+
```python
import pyotb
-resampled = pyotb.RigidTransformResample({'in': 'my_image.tif', 'interpolator': 'linear', 'out': 'output.tif',
- 'transform.type.id.scaley': 0.5, 'transform.type.id.scalex': 0.5})
-# Here you can set optionally pixel type and extended filename variables
-resampled.write(pixel_type='uint16', filename_extension='?nodata=65535')
+resampled = pyotb.RigidTransformResample({
+ 'in': 'my_image.tif',
+ 'interpolator': 'linear',
+ 'out': 'output.tif',
+ 'transform.type.id.scaley': 0.5,
+ 'transform.type.id.scalex': 0.5
+})
```
\ No newline at end of file
diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md
index e96e3e7..62cfafd 100644
--- a/doc/troubleshooting.md
+++ b/doc/troubleshooting.md
@@ -1,14 +1,20 @@
-## Troubleshooting: known limitations
+## Troubleshooting: known limitations with old versions
+
+!!! Info
+
+ All defects described below have been fixed since OTB 8.1.2 and pyotb 2.0.0
### Failure of intermediate writing
-When chaining applications in-memory, there may be some problems when writing intermediate results, depending on the order
+When chaining applications in-memory, there may be some problems when writing
+intermediate results, depending on the order
the writings are requested. Some examples can be found below:
#### Example of failures involving slicing
-For some applications (non-exhaustive know list: OpticalCalibration, DynamicConvert, BandMath), we can face unexpected
- failures when using channels slicing
+For some applications (non-exhaustive know list: OpticalCalibration,
+DynamicConvert, BandMath), we can face unexpected failures when using channels
+slicing
```python
import pyotb
@@ -55,8 +61,8 @@ one_band.write('one_band.tif')
#### Example of failures involving arithmetic operation
-One can meet errors when using arithmetic operations at the end of a pipeline when DynamicConvert, BandMath or
-OpticalCalibration is involved:
+One can meet errors when using arithmetic operations at the end of a pipeline
+when DynamicConvert, BandMath or OpticalCalibration is involved:
```python
import pyotb
--
GitLab
From bd7163b6e7f0a07018cb0b103f501d3995bad57b Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 27 Jun 2023 16:23:32 +0200
Subject: [PATCH 204/399] DOC: update mkdocs.yml
---
mkdocs.yml | 3 +++
1 file changed, 3 insertions(+)
diff --git a/mkdocs.yml b/mkdocs.yml
index 5654279..29d51d4 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -59,6 +59,7 @@ extra_css:
use_directory_urls: false # this creates some pyotb/core.html pages instead of pyotb/core/index.html
markdown_extensions:
+ - admonition
- toc:
permalink: true
title: On this page
@@ -67,6 +68,8 @@ markdown_extensions:
anchor_linenums: true
- pymdownx.inlinehilite
- pymdownx.snippets
+ - pymdownx.details
+ - pymdownx.superfences
# Rest of the navigation..
site_name: "pyotb documentation: a Python extension of OTB"
--
GitLab
From ed3bdf2eabe37d1033f2404c6123b0bd79391b45 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 27 Jun 2023 16:23:43 +0200
Subject: [PATCH 205/399] DOC: update troubleshooting
---
doc/troubleshooting.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md
index 62cfafd..f8a4739 100644
--- a/doc/troubleshooting.md
+++ b/doc/troubleshooting.md
@@ -1,6 +1,6 @@
## Troubleshooting: known limitations with old versions
-!!! Info
+!!! note
All defects described below have been fixed since OTB 8.1.2 and pyotb 2.0.0
--
GitLab
From 8c777f26dc63d2451fc9a0ab320d55976ad04332 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 27 Jun 2023 16:32:56 +0200
Subject: [PATCH 206/399] Doc: shorten authors info
---
AUTHORS.md | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/AUTHORS.md b/AUTHORS.md
index 7ac46df..6518258 100644
--- a/AUTHORS.md
+++ b/AUTHORS.md
@@ -2,9 +2,9 @@
## Initial codebase
-* Nicolas NARÇON - IT engineer @ ESA - Rome (Italy)
+* Nicolas NARÇON (INRAE, now ESA)
## Current maintainers
-* Rémi CRESSON - RS research engineer @ INRAe - Montpellier (France)
-* Vincent DELBAR - GIS engineer @ La TeleScop - Montpellier (France)
+* Rémi CRESSON (INRAE)
+* Vincent DELBAR (La TeleScop)
--
GitLab
From fe592d755c12554d4f1e096e6b979fdac393e7a4 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 27 Jun 2023 16:33:13 +0200
Subject: [PATCH 207/399] Doc: add highlights (warning/note)
---
doc/examples/pleiades.md | 6 ++++--
doc/interaction.md | 24 ++++++++++++++----------
doc/troubleshooting.md | 2 +-
3 files changed, 19 insertions(+), 13 deletions(-)
diff --git a/doc/examples/pleiades.md b/doc/examples/pleiades.md
index eb3f1f2..120a554 100644
--- a/doc/examples/pleiades.md
+++ b/doc/examples/pleiades.md
@@ -1,5 +1,6 @@
### Process raw Pleiades data
-This is a common case of Pleiades data preprocessing : optical calibration -> orthorectification -> pansharpening
+This is a common case of Pleiades data preprocessing :
+*optical calibration -> orthorectification -> pansharpening*
```python
import pyotb
@@ -34,6 +35,7 @@ pxs = pyotb.BundleToPerfectSensor(
)
exfn = '?gdal:co:COMPRESS=DEFLATE&gdal:co:PREDICTOR=2&gdal:co:BIGTIFF=YES'
-# Here we trigger every app in the pipeline and the process is blocked until result is written to disk
+# Here we trigger every app in the pipeline and the process is blocked until
+# result is written to disk
pxs.write('pxs_image.tif', pixel_type='uint16', ext_fname=exfn)
```
diff --git a/doc/interaction.md b/doc/interaction.md
index 45da657..e4c73fb 100644
--- a/doc/interaction.md
+++ b/doc/interaction.md
@@ -41,11 +41,13 @@ noisy_image = inp + white_noise
noisy_image.write('image_plus_noise.tif')
```
-Limitations :
+!!! warning
-- The whole image is loaded into memory
-- The georeference can not be modified. Thus, numpy operations can not change
-the image or pixel size
+ Limitations :
+
+ - The whole image is loaded into memory
+ - The georeference can not be modified. Thus, numpy operations can not change
+ the image or pixel size
## Export to rasterio
@@ -134,10 +136,12 @@ Advantages :
memory
- Can be integrated in OTB pipelines
-Limitations :
+!!! warning
-- For OTBTF versions < 4.0.0, it is not possible to use the tensorflow python
-API inside a script where OTBTF is used because of libraries clashing between
-Tensorflow and OTBTF, i.e. `import tensorflow` doesn't work in a script where
-OTBTF apps have been initialized. This is why we recommend to use latest OTBTF
-versions
+ Limitations :
+
+ - For OTBTF versions < 4.0.0, it is not possible to use the tensorflow
+ python API inside a script where OTBTF is used because of libraries
+ clashing between Tensorflow and OTBTF, i.e. `import tensorflow` doesn't
+ work in a script where OTBTF apps have been initialized. This is why we
+ recommend to use latest OTBTF versions
diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md
index f8a4739..0025913 100644
--- a/doc/troubleshooting.md
+++ b/doc/troubleshooting.md
@@ -4,7 +4,7 @@
All defects described below have been fixed since OTB 8.1.2 and pyotb 2.0.0
-### Failure of intermediate writing
+### Failure of intermediate writing (otb < 8.1, pyotb < 1.5.4)
When chaining applications in-memory, there may be some problems when writing
intermediate results, depending on the order
--
GitLab
From 001a9cae44c81e82d5a3d8d26cfa86eec019f9fc Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 27 Jun 2023 16:19:22 +0000
Subject: [PATCH 208/399] Apply 1 suggestion(s) to 1 file(s)
---
doc/managing_loggers.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/managing_loggers.md b/doc/managing_loggers.md
index 5c2d80b..7220af4 100644
--- a/doc/managing_loggers.md
+++ b/doc/managing_loggers.md
@@ -18,7 +18,7 @@ import pyotb
pyotb.set_logger_level('DEBUG')
```
-Bonus : in some cases, yo may want to silence the GDAL driver logger (for
+Bonus : in some cases, you may want to silence the GDAL driver logger (for
example you will see a lot of errors when reading GML files with OGR).
One useful trick is to redirect these logs to a file. This can be done using
the variable `CPL_LOG`.
--
GitLab
From 7aa5940c69f1a6e1bb054d301d14fbab73b4cdf6 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 27 Jun 2023 16:19:36 +0000
Subject: [PATCH 209/399] Apply 1 suggestion(s) to 1 file(s)
---
doc/quickstart.md | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/doc/quickstart.md b/doc/quickstart.md
index 915b3f3..5e07e43 100644
--- a/doc/quickstart.md
+++ b/doc/quickstart.md
@@ -42,9 +42,18 @@ output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'})
```
Limitations : for this notation, python doesn't accept the parameter `in` or
-any parameter that contains a `.`. E.g., it is not possible to use
-`pyotb.RigidTransformResample(in=input_path...)` or
-`pyotb.VectorDataExtractROI(io.vd=vector_path...)`.
+any parameter that contains a dots (e.g. `io.in)`.
+For `in` and other main input parameters of an OTB app, you may simply pass
+the value as first argument, pyotb will guess the parameter name.
+For parameters that contains dots, you can either use a dictionary, or replace dots (`.`) with underscores (`_`) as follow :
+
+```python
+resampled = pyotb.RigidTransformResample(
+ 'my_image.tif',
+ interpolator = 'linear',
+ transform_type_id_scaley = 0.5,
+ transform_type_id_scalex = 0.5
+)
## In-memory connections
--
GitLab
From 3cc6378de5cddfc4b5a7977373a1de6490c4b3c1 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 27 Jun 2023 16:39:36 +0000
Subject: [PATCH 210/399] Fix CI trigger
---
.gitlab-ci.yml | 8 ++++----
pyotb/__init__.py | 2 +-
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 49211fa..a5d80fd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,11 +14,11 @@ cache:
workflow:
rules:
- - if: $CI_COMMIT_TAG
- - if: $CI_PIPELINE_SOURCE == "merge_request_event"
- - if: $CI_COMMIT_REF_PROTECTED && $CI_OPEN_MERGE_REQUESTS
+ - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"
when: never
- - if: $CI_COMMIT_REF_PROTECTED
+ - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+ - if: $CI_COMMIT_TAG
+ - if: $CI_COMMIT_REF_PROTECTED == "true"
stages:
- Static Analysis
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index 5191fd6..b3bc21c 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
"""This module provides convenient python wrapping of otbApplications."""
-__version__ = "2.0.0.dev2"
+__version__ = "2.0.0.dev3"
from .helpers import logger, set_logger_level
from .apps import *
--
GitLab
From dbeca6c2944fa6821e209c0527f5ac0582d8fec5 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 27 Jun 2023 21:30:32 +0200
Subject: [PATCH 211/399] CI: add metadata tests
---
.gitlab-ci.yml | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a5d80fd..bfad764 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -95,6 +95,11 @@ test_pipeline:
script:
- pytest --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py
+test_metadata:
+ extends: .tests
+ script:
+ - pytest --color=yes --junitxml=test-metadata.xml tests/test_metadata.py
+
# -------------------------------------- Docs ---------------------------------------
docs:
--
GitLab
From b970b1af79588d2be593adb2a72384eb0b53d370 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 27 Jun 2023 21:30:46 +0200
Subject: [PATCH 212/399] ADD: metadata tests
---
tests/test_metadata.py | 11 +++++++++++
1 file changed, 11 insertions(+)
create mode 100644 tests/test_metadata.py
diff --git a/tests/test_metadata.py b/tests/test_metadata.py
new file mode 100644
index 0000000..5d42a44
--- /dev/null
+++ b/tests/test_metadata.py
@@ -0,0 +1,11 @@
+from tests_data import INPUT
+import pyotb
+
+INPUT2 = pyotb.Input(
+ "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/"
+ "47/Q/RU/2021/12/S2B_47QRU_20211227_0_L2A/B04.tif"
+)
+
+def test_metadata():
+ assert "ProjectionRef", "TIFFTAG_SOFTWARE" in INPUT.metadata
+ assert "ProjectionRef", "OVR_RESAMPLING_ALG" in INPUT2.metadata
--
GitLab
From c5a15613e4709aa0a4da3769a0a1936f6c466368 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 27 Jun 2023 21:31:20 +0200
Subject: [PATCH 213/399] ENH: metadata property to return IMD and MDD
---
pyotb/core.py | 32 ++++++++++++++++++++++++++++++--
1 file changed, 30 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 06da0c7..4f5a724 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -39,8 +39,36 @@ class OTBObject(ABC):
@property
def metadata(self) -> dict[str, (str, float, list[float])]:
- """Return first output image metadata dictionary."""
- return dict(self.app.GetMetadataDictionary(self.output_image_key))
+ """Return the concatenation of the first output image metadata
+ dictionary and the metadata dictionary."""
+
+ # Image Metadata only when there are values
+ otb_imd = self.app.GetImageMetadata(self.output_image_key)
+ cats = ["Num", "Str", "L1D", "Time"]
+ imd = {
+ key: getattr(otb_imd, f"get_{cat.lower()}")(key)
+ for cat in cats
+ for key in getattr(otb_imd, f"GetKeyList{cat}")().split(" ")
+ if getattr(otb_imd, "has")(key)
+ }
+
+ # Metadata dictionary
+ # Replace items like {"metadata_1": "TIFFTAG_SOFTWARE=CSinG - 13
+ # SEPTEMBRE 2012"} with {"TIFFTAG_SOFTWARE": "CSinG - 13 SEPTEMBRE
+ # 2012"}
+ mdd = dict(self.app.GetMetadataDictionary(self.output_image_key))
+ new_mdd = {
+ splits[0].strip()
+ if (replace := (k.lower().startswith("metadata_") and
+ len(splits := v.split("=")) == 2)) else k:
+ splits[1].strip() if replace else v
+ for k, v in mdd.items()
+ }
+
+ return {
+ **new_mdd,
+ **imd
+ }
@property
def dtype(self) -> np.dtype:
--
GitLab
From 9cd14d7f01366df5e67e3aad0e876329065c4200 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 27 Jun 2023 21:46:28 +0200
Subject: [PATCH 214/399] CI: move metadata tests inside core tests
---
.gitlab-ci.yml | 5 -----
tests/test_metadata.py | 11 -----------
2 files changed, 16 deletions(-)
delete mode 100644 tests/test_metadata.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bfad764..a5d80fd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -95,11 +95,6 @@ test_pipeline:
script:
- pytest --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py
-test_metadata:
- extends: .tests
- script:
- - pytest --color=yes --junitxml=test-metadata.xml tests/test_metadata.py
-
# -------------------------------------- Docs ---------------------------------------
docs:
diff --git a/tests/test_metadata.py b/tests/test_metadata.py
deleted file mode 100644
index 5d42a44..0000000
--- a/tests/test_metadata.py
+++ /dev/null
@@ -1,11 +0,0 @@
-from tests_data import INPUT
-import pyotb
-
-INPUT2 = pyotb.Input(
- "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/"
- "47/Q/RU/2021/12/S2B_47QRU_20211227_0_L2A/B04.tif"
-)
-
-def test_metadata():
- assert "ProjectionRef", "TIFFTAG_SOFTWARE" in INPUT.metadata
- assert "ProjectionRef", "OVR_RESAMPLING_ALG" in INPUT2.metadata
--
GitLab
From 69b31abc40d88b7944130d5694b0ce1e1f993e15 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 27 Jun 2023 21:46:40 +0200
Subject: [PATCH 215/399] ADD: move metadata tests inside core tests
---
tests/test_core.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/tests/test_core.py b/tests/test_core.py
index 99b21de..52cdc97 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -1,6 +1,5 @@
import pytest
-import pyotb
from tests_data import *
@@ -78,7 +77,12 @@ def test_data():
def test_metadata():
- assert INPUT.metadata["Metadata_1"] == "TIFFTAG_SOFTWARE=CSinG - 13 SEPTEMBRE 2012"
+ INPUT2 = pyotb.Input(
+ "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/"
+ "47/Q/RU/2021/12/S2B_47QRU_20211227_0_L2A/B04.tif"
+ )
+ assert "ProjectionRef", "TIFFTAG_SOFTWARE" in INPUT.metadata
+ assert "ProjectionRef", "OVR_RESAMPLING_ALG" in INPUT2.metadata
def test_nonraster_property():
--
GitLab
From f3bcefd9bfbbb2adc46074a74fd94b3725b1345c Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 27 Jun 2023 21:53:07 +0200
Subject: [PATCH 216/399] DOC: metadata property doc
---
pyotb/core.py | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 4f5a724..05198ef 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -39,10 +39,13 @@ class OTBObject(ABC):
@property
def metadata(self) -> dict[str, (str, float, list[float])]:
- """Return the concatenation of the first output image metadata
- dictionary and the metadata dictionary."""
+ """Return metadata.
- # Image Metadata only when there are values
+ The returned dict results from the concatenation of the first output
+ image metadata dictionary and the metadata dictionary.
+
+ """
+ # Image Metadata
otb_imd = self.app.GetImageMetadata(self.output_image_key)
cats = ["Num", "Str", "L1D", "Time"]
imd = {
--
GitLab
From 82f3ab70f906e3479d0138fb3183da97c349c527 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 27 Jun 2023 22:02:53 +0200
Subject: [PATCH 217/399] DOC: metadata property doc
---
pyotb/core.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 05198ef..daa6b60 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -41,8 +41,8 @@ class OTBObject(ABC):
def metadata(self) -> dict[str, (str, float, list[float])]:
"""Return metadata.
- The returned dict results from the concatenation of the first output
- image metadata dictionary and the metadata dictionary.
+ The returned dict results from the concatenation of the first output
+ image metadata dictionary and the metadata dictionary.
"""
# Image Metadata
--
GitLab
From 35bfb6ab95fdbf7cb48ea4a8e743dd7df8bf6851 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 27 Jun 2023 22:06:48 +0200
Subject: [PATCH 218/399] DOC: metadata property doc
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index daa6b60..676b7c5 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -44,7 +44,7 @@ class OTBObject(ABC):
The returned dict results from the concatenation of the first output
image metadata dictionary and the metadata dictionary.
- """
+ """
# Image Metadata
otb_imd = self.app.GetImageMetadata(self.output_image_key)
cats = ["Num", "Str", "L1D", "Time"]
--
GitLab
From e03489e79b820316cff21105108a6af02e73777a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr>
Date: Wed, 28 Jun 2023 08:58:13 +0000
Subject: [PATCH 219/399] Apply 1 suggestion(s) to 1 file(s)
---
pyotb/core.py | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 676b7c5..7675858 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -60,13 +60,13 @@ class OTBObject(ABC):
# SEPTEMBRE 2012"} with {"TIFFTAG_SOFTWARE": "CSinG - 13 SEPTEMBRE
# 2012"}
mdd = dict(self.app.GetMetadataDictionary(self.output_image_key))
- new_mdd = {
- splits[0].strip()
- if (replace := (k.lower().startswith("metadata_") and
- len(splits := v.split("=")) == 2)) else k:
- splits[1].strip() if replace else v
- for k, v in mdd.items()
- }
+ new_mdd = {}
+ for key, val in mdd.items():
+ splits = val.split("=")
+ if key.lower().startswith("metadata_") and len(splits) == 2:
+ new_mdd[splits[0].strip()] = splits[1].strip()
+ else:
+ new_mdd[key] = val
return {
**new_mdd,
--
GitLab
From bb736175a863c44e53994455e84c9991924353ba Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 28 Jun 2023 14:09:42 +0200
Subject: [PATCH 220/399] ADD: process metadata values other that str
---
pyotb/core.py | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 7675858..040968d 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -62,11 +62,14 @@ class OTBObject(ABC):
mdd = dict(self.app.GetMetadataDictionary(self.output_image_key))
new_mdd = {}
for key, val in mdd.items():
- splits = val.split("=")
- if key.lower().startswith("metadata_") and len(splits) == 2:
- new_mdd[splits[0].strip()] = splits[1].strip()
- else:
- new_mdd[key] = val
+ new_key = key
+ new_val = val
+ if isinstance(val, str):
+ splits = val.split("=")
+ if key.lower().startswith("metadata_") and len(splits) == 2:
+ new_key = splits[0].strip()
+ new_val = splits[1].strip()
+ new_mdd[new_key] = new_val
return {
**new_mdd,
--
GitLab
From 491242716abc3baaf0222f690ee8b43c51758c06 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 28 Jun 2023 14:37:04 +0200
Subject: [PATCH 221/399] ADD: process metadata values other that str
---
tests/test_core.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/tests/test_core.py b/tests/test_core.py
index 52cdc97..eab42d4 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -84,6 +84,12 @@ def test_metadata():
assert "ProjectionRef", "TIFFTAG_SOFTWARE" in INPUT.metadata
assert "ProjectionRef", "OVR_RESAMPLING_ALG" in INPUT2.metadata
+ # Metadata with numeric values (e.g. TileHintX)
+ fp = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/" \
+ "Data/Input/radarsat2/RADARSAT2_ALTONA_300_300_VV.tif?inline=false"
+ app = pyotb.BandMath({"il": [fp], "exp": "im1b1"})
+ assert "TileHintX" in app.metadata
+
def test_nonraster_property():
with pytest.raises(TypeError):
--
GitLab
From 8fa6e5e7dfededfff8286b4862db70b647687b12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr>
Date: Wed, 28 Jun 2023 13:34:13 +0000
Subject: [PATCH 222/399] Extended filenames as dict
---
pyotb/core.py | 44 ++++++++++++++++++++++----------
tests/test_core.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 93 insertions(+), 13 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 040968d..1582079 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -810,19 +810,37 @@ class App(OTBObject):
# Append filename extension to filenames
if ext_fname:
- logger.debug(
- "%s: using extended filename for outputs: %s", self.name, ext_fname
- )
- if not ext_fname.startswith("?"):
- ext_fname = "?&" + ext_fname
- elif not ext_fname.startswith("?&"):
- ext_fname = "?&" + ext_fname[1:]
- for key, value in kwargs.items():
- if (
- self._out_param_types[key] == otb.ParameterType_OutputImage
- and "?" not in value
- ):
- parameters[key] = value + ext_fname
+ if not isinstance(ext_fname, (dict, str)):
+ raise ValueError("Extended filename must be a str or a dict")
+ def _str2dict(ext_str):
+ """Function that converts str to dict."""
+ splits = [pair.split("=") for pair in ext_str.split("&")]
+ return dict(split for split in splits if len(split) == 2)
+
+ if isinstance(ext_fname, str):
+ ext_fname = _str2dict(ext_fname)
+
+ logger.debug("%s: extended filename for all outputs:", self.name)
+ for key, ext in ext_fname.items():
+ logger.debug("%s: %s", key, ext)
+
+ for key, filepath in kwargs.items():
+ if self._out_param_types[key] == otb.ParameterType_OutputImage:
+ new_ext_fname = ext_fname.copy()
+
+ # grab already set extended filename key/values
+ if "?&" in filepath:
+ filepath, already_set_ext = filepath.split("?&", 1)
+ # extensions in filepath prevail over `new_ext_fname`
+ new_ext_fname.update(_str2dict(already_set_ext))
+
+ # transform dict to str
+ ext_fname_str = "&".join([
+ f"{key}={value}"
+ for key, value in new_ext_fname.items()
+ ])
+ parameters[key] = f"{filepath}?&{ext_fname_str}"
+
# Manage output pixel types
data_types = {}
if pixel_type:
diff --git a/tests/test_core.py b/tests/test_core.py
index eab42d4..5207b80 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -121,6 +121,68 @@ def test_write():
INPUT["out"].filepath.unlink()
+def test_ext_fname():
+ def _check(expected: str, key: str = "out", app = INPUT.app):
+ fn = app.GetParameterString(key)
+ assert "?&" in fn
+ assert fn.split("?&", 1)[1] == expected
+
+ assert INPUT.write("/tmp/test_write.tif", ext_fname="nodata=0")
+ _check("nodata=0")
+ assert INPUT.write("/tmp/test_write.tif", ext_fname={"nodata": "0"})
+ _check("nodata=0")
+ assert INPUT.write("/tmp/test_write.tif", ext_fname={"nodata": 0})
+ _check("nodata=0")
+ assert INPUT.write(
+ "/tmp/test_write.tif",
+ ext_fname={
+ "nodata": 0,
+ "gdal:co:COMPRESS": "DEFLATE"
+ }
+ )
+ _check("nodata=0&gdal:co:COMPRESS=DEFLATE")
+ assert INPUT.write(
+ "/tmp/test_write.tif",
+ ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE"
+ )
+ _check("nodata=0&gdal:co:COMPRESS=DEFLATE")
+ assert INPUT.write(
+ "/tmp/test_write.tif?&box=0:0:10:10",
+ ext_fname={
+ "nodata": "0",
+ "gdal:co:COMPRESS": "DEFLATE",
+ "box": "0:0:20:20"
+ }
+ )
+ # Check that the bbox is the one specified in the filepath, not the one
+ # specified in `ext_filename`
+ _check("nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:10:10")
+ assert INPUT.write(
+ "/tmp/test_write.tif?&box=0:0:10:10",
+ ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:20:20"
+ )
+ _check("nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:10:10")
+
+ INPUT["out"].filepath.unlink()
+
+ mss = pyotb.MeanShiftSmoothing(FILEPATH)
+ mss.write(
+ {
+ "fout": "/tmp/test_ext_fn_fout.tif?&nodata=1",
+ "foutpos": "/tmp/test_ext_fn_foutpos.tif?&nodata=2"
+ },
+ ext_fname={
+ "nodata": 0,
+ "gdal:co:COMPRESS": "DEFLATE"
+ }
+ )
+ _check("nodata=1&gdal:co:COMPRESS=DEFLATE", key="fout", app=mss.app)
+ _check("nodata=2&gdal:co:COMPRESS=DEFLATE", key="foutpos", app=mss.app)
+ mss["fout"].filepath.unlink()
+ mss["foutpos"].filepath.unlink()
+
+
+
def test_frozen_app_write():
app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True)
assert app.write("/tmp/test_frozen_app_write.tif")
--
GitLab
From d811ee0959387080ded681dfd23e4c79626bc2ed Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 28 Jun 2023 15:08:29 +0000
Subject: [PATCH 223/399] Avoid storing full license in package metadata
---
pyotb/__init__.py | 2 +-
pyproject.toml | 10 +++++-----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index b3bc21c..3e09a63 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
"""This module provides convenient python wrapping of otbApplications."""
-__version__ = "2.0.0.dev3"
+__version__ = "2.0.0.dev4"
from .helpers import logger, set_logger_level
from .apps import *
diff --git a/pyproject.toml b/pyproject.toml
index 6f86ad0..bc2c576 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,15 +6,15 @@ build-backend = "setuptools.build_meta"
name = "pyotb"
description = "Library to enable easy use of the Orfeo ToolBox (OTB) in Python"
authors = [
- { name = "Rémi Cresson", email = "remi.cresson@inrae.fr" },
- { name = "Nicolas Narçon" },
- { name = "Vincent Delbar" },
+ {name="Rémi Cresson", email="remi.cresson@inrae.fr"},
+ {name="Nicolas Narçon"},
+ {name="Vincent Delbar"},
]
requires-python = ">=3.7"
keywords = ["gis", "remote sensing", "otb", "orfeotoolbox", "orfeo toolbox"]
dependencies = ["numpy>=1.16"]
readme = "README.md"
-license = { file = "LICENSE" }
+license = {text="Apache-2.0"}
dynamic = ["version"]
classifiers = [
"Programming Language :: Python :: 3",
@@ -41,7 +41,7 @@ repository = "https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb"
packages = ["pyotb"]
[tool.setuptools.dynamic]
-version = { attr = "pyotb.__version__" }
+version = {attr="pyotb.__version__"}
[tool.pylint]
max-line-length = 88
--
GitLab
From ebf7646f4d8d143459c2ab2980676acac4c03857 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 29 Jun 2023 23:27:13 +0200
Subject: [PATCH 224/399] ADD: test for bug #106
---
tests/test_core.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/tests/test_core.py b/tests/test_core.py
index 5207b80..5f65ca0 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -203,6 +203,10 @@ def test_frozen_output_write():
assert app["out"].write("/tmp/test_frozen_app_write.tif")
app["out"].filepath.unlink()
+def test_parameters():
+ app = pyotb.OrthoRectification(FILEPATH)
+ assert isinstance(app.parameters["map"], str)
+ assert app.parameters["map"] == "utm"
def test_output_in_arg():
info = pyotb.ReadImageInfo(INPUT["out"])
--
GitLab
From ef04e7415115aa32960c454fb335938c5399a57c Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 29 Jun 2023 23:31:31 +0200
Subject: [PATCH 225/399] FIX: try to fix #106
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 1582079..08877d4 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -558,7 +558,7 @@ class App(OTBObject):
@property
def parameters(self):
"""Return used OTB application parameters."""
- return {**self._auto_parameters, **self.app.GetParameters(), **self._settings}
+ return {**self.app.GetParameters(), **self._auto_parameters, **self._settings}
@property
def exports_dic(self) -> dict[str, dict]:
--
GitLab
From 645556c76f95a272c287dfb5007e78cbb96305d1 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 30 Jun 2023 10:29:05 +0200
Subject: [PATCH 226/399] ENH: remove key from App._auto_parameters when set by
user
---
pyotb/core.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/pyotb/core.py b/pyotb/core.py
index 08877d4..ff0905b 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -691,6 +691,8 @@ class App(OTBObject):
self._settings[key] = obj
if key in self.outputs:
self.outputs[key].filepath = obj
+ if key in self._auto_parameters:
+ del self._auto_parameters[key]
def propagate_dtype(self, target_key: str = None, dtype: int = None):
"""Propagate a pixel type from main input to every outputs, or to a target output key only.
--
GitLab
From 3ea66cf97cb1b764b2fc2b479dafa1478492eb41 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 30 Jun 2023 10:52:02 +0200
Subject: [PATCH 227/399] FIX: duplicated test_parameters function + try update
param
---
tests/test_core.py | 22 +++++++++++++---------
1 file changed, 13 insertions(+), 9 deletions(-)
diff --git a/tests/test_core.py b/tests/test_core.py
index 5f65ca0..0c511e7 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -8,6 +8,19 @@ def test_parameters():
assert INPUT.parameters
assert INPUT.parameters["in"] == FILEPATH
assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304)
+ app = pyotb.OrthoRectification(INPUT)
+ assert isinstance(app.parameters["map"], str)
+ assert app.parameters["map"] == "utm"
+ assert "map" in app._auto_parameters
+ app.set_parameters({"map": "epsg", "map.epsg.code": 2154})
+ assert app.parameters["map"] == "epsg"
+ assert "map" in app._settings and "map" not in app._auto_parameters
+ assert app.parameters["map.epsg.code"] == app.app.GetParameters()["map.epsg.code"]
+
+
+def test_param_with_underscore():
+ app = pyotb.OrthoRectification(io_in=INPUT, map_epsg_code=2154)
+ assert app.parameters["map.epsg.code"] == 2154
def test_input_vsi():
@@ -36,11 +49,6 @@ def test_input_vsi_from_user():
pyotb.Input("/vsicurl/" + FILEPATH)
-def test_param_with_underscore():
- app = pyotb.OrthoRectification(io_in=INPUT, map_epsg_code=2154)
- assert app.parameters["map.epsg.code"] == 2154
-
-
def test_wrong_key():
with pytest.raises(KeyError):
pyotb.BandMath(INPUT, expression="im1b1")
@@ -203,10 +211,6 @@ def test_frozen_output_write():
assert app["out"].write("/tmp/test_frozen_app_write.tif")
app["out"].filepath.unlink()
-def test_parameters():
- app = pyotb.OrthoRectification(FILEPATH)
- assert isinstance(app.parameters["map"], str)
- assert app.parameters["map"] == "utm"
def test_output_in_arg():
info = pyotb.ReadImageInfo(INPUT["out"])
--
GitLab
From a7197dc945f4c7d26dbd5f67c7eb71f0122f47f8 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 1 Jul 2023 12:00:16 +0200
Subject: [PATCH 228/399] ENH: __sync_parameters skip unused choices param keys
---
pyotb/core.py | 30 ++++++++++++++++++++++--------
1 file changed, 22 insertions(+), 8 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index ff0905b..e8ecfcc 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -521,15 +521,17 @@ class App(OTBObject):
# Param keys and types
self.parameters_keys = tuple(self.app.GetParametersKeys())
self._all_param_types = {
- k: self.app.GetParameterType(k) for k in self.parameters_keys
+ key: self.app.GetParameterType(key) for key in self.parameters_keys
}
- types = (
- otb.ParameterType_OutputImage,
- otb.ParameterType_OutputVectorData,
- otb.ParameterType_OutputFilename,
- )
self._out_param_types = {
- k: v for k, v in self._all_param_types.items() if v in types
+ key: val
+ for key, val in self._all_param_types.items()
+ if val in self.OUTPUT_PARAM_TYPES
+ }
+ self._key_choices = {
+ key: [f"{key}.{choice}" for choice in self.app.GetChoiceKeys(key)]
+ for key in self.parameters_keys
+ if self.app.GetParameterType(key) == otb.ParameterType_Choice
}
# Init, execute and write (auto flush only when output param was provided)
if args or kwargs:
@@ -951,8 +953,20 @@ class App(OTBObject):
def __sync_parameters(self):
"""Save OTB parameters in _settings, data and outputs dict, for a list of keys or all parameters."""
+ skip = [
+ k for k in self.parameters_keys if k.split(".")[-1] in ("ram", "default")
+ ]
+ # Prune unused choices child params
+ for key in self._key_choices:
+ choices = self._key_choices[key].copy()
+ choices.remove(f"{key}.{self.app.GetParameterValue(key)}")
+ skip.extend(
+ [k for k in self.parameters_keys if k.startswith(tuple(choices))]
+ )
+
+ self._auto_parameters.clear()
for key in self.parameters_keys:
- if not self.app.HasValue(key):
+ if key in skip or key in self._settings or not self.app.HasValue(key):
continue
value = self.app.GetParameterValue(key)
# TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken
--
GitLab
From 280d06e648af2a2f55ef162d134dcdd2d987258f Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 1 Jul 2023 12:01:09 +0200
Subject: [PATCH 229/399] FIX: __sync_parameters find every auto parameters
---
pyotb/core.py | 22 ++++++++++------------
1 file changed, 10 insertions(+), 12 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index e8ecfcc..97bfca1 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -969,20 +969,18 @@ class App(OTBObject):
if key in skip or key in self._settings or not self.app.HasValue(key):
continue
value = self.app.GetParameterValue(key)
- # TODO: here we *should* use self.app.IsParameterEnabled, but it seems broken
- if isinstance(value, otb.ApplicationProxy) and self.app.HasAutomaticValue(
- key
+ # Here we should use AND self.app.IsParameterEnabled(key) but it's broken
+ if not self.app.GetParameterRole(key) and (
+ self.app.HasAutomaticValue(key) or self.app.IsParameterEnabled(key)
):
- try:
- value = str(
- value
- ) # some default str values like "mode" or "interpolator"
- self._auto_parameters[key] = value
- continue
- except RuntimeError:
- continue # grouped parameters
+ if isinstance(value, otb.ApplicationProxy):
+ try:
+ value = str(value)
+ except RuntimeError:
+ continue # root of param group
+ self._auto_parameters[key] = value
# Save static output data (ReadImageInfo, ComputeImageStatistics, etc.)
- elif self.app.GetParameterRole(key) == 1 and bool(value) or value == 0:
+ elif self.app.GetParameterRole(key) == 1 and (bool(value) or value == 0):
if isinstance(value, str):
try:
value = literal_eval(value)
--
GitLab
From 32f74e618bca40033d39c5ba2f4a1abbde6934f3 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 1 Jul 2023 12:20:54 +0200
Subject: [PATCH 230/399] ENH: sync params, and avoir calling
App.app.GetParameters in property
---
pyotb/core.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 97bfca1..81b56da 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -546,6 +546,8 @@ class App(OTBObject):
self.execute()
if any(key in self._settings for key in self._out_param_types):
self.flush()
+ else:
+ self.__sync_parameters() # since not called during execute()
@property
def name(self) -> str:
@@ -560,7 +562,7 @@ class App(OTBObject):
@property
def parameters(self):
"""Return used OTB application parameters."""
- return {**self.app.GetParameters(), **self._auto_parameters, **self._settings}
+ return {**self._auto_parameters, **self._settings}
@property
def exports_dic(self) -> dict[str, dict]:
@@ -762,6 +764,8 @@ class App(OTBObject):
)
self._time_start = perf_counter()
self.app.ExecuteAndWriteOutput()
+ self.__sync_parameters()
+ self.frozen = False
self._time_end = perf_counter()
def write(
--
GitLab
From 1e90bc7f74c4118cb0924eafb729a0569ede0227 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 1 Jul 2023 12:21:28 +0200
Subject: [PATCH 231/399] STYLE: black autoformat + comments about types since
we use enums
---
pyotb/core.py | 23 ++++++-----------------
1 file changed, 6 insertions(+), 17 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 81b56da..7f38a5c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -71,10 +71,7 @@ class OTBObject(ABC):
new_val = splits[1].strip()
new_mdd[new_key] = new_val
- return {
- **new_mdd,
- **imd
- }
+ return {**new_mdd, **imd}
@property
def dtype(self) -> np.dtype:
@@ -443,27 +440,19 @@ class App(OTBObject):
"""Base class that gathers common operations for any OTB application."""
INPUT_IMAGE_TYPES = [
- # Images only
otb.ParameterType_InputImage,
otb.ParameterType_InputImageList,
]
INPUT_PARAM_TYPES = INPUT_IMAGE_TYPES + [
- # Vectors
otb.ParameterType_InputVectorData,
otb.ParameterType_InputVectorDataList,
- # Filenames
otb.ParameterType_InputFilename,
otb.ParameterType_InputFilenameList,
]
- OUTPUT_IMAGE_TYPES = [
- # Images only
- otb.ParameterType_OutputImage
- ]
+ OUTPUT_IMAGE_TYPES = [otb.ParameterType_OutputImage]
OUTPUT_PARAM_TYPES = OUTPUT_IMAGE_TYPES + [
- # Vectors
otb.ParameterType_OutputVectorData,
- # Filenames
otb.ParameterType_OutputFilename,
]
@@ -820,6 +809,7 @@ class App(OTBObject):
if ext_fname:
if not isinstance(ext_fname, (dict, str)):
raise ValueError("Extended filename must be a str or a dict")
+
def _str2dict(ext_str):
"""Function that converts str to dict."""
splits = [pair.split("=") for pair in ext_str.split("&")]
@@ -843,10 +833,9 @@ class App(OTBObject):
new_ext_fname.update(_str2dict(already_set_ext))
# transform dict to str
- ext_fname_str = "&".join([
- f"{key}={value}"
- for key, value in new_ext_fname.items()
- ])
+ ext_fname_str = "&".join(
+ [f"{key}={value}" for key, value in new_ext_fname.items()]
+ )
parameters[key] = f"{filepath}?&{ext_fname_str}"
# Manage output pixel types
--
GitLab
From 6fd7101ac275cfd4ba4157013c673920ad1cfb9c Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 1 Jul 2023 12:27:28 +0200
Subject: [PATCH 232/399] FIX: always try convert to str if otbApplicationProxy
---
pyotb/core.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 7f38a5c..afd5787 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -962,15 +962,15 @@ class App(OTBObject):
if key in skip or key in self._settings or not self.app.HasValue(key):
continue
value = self.app.GetParameterValue(key)
+ if isinstance(value, otb.ApplicationProxy):
+ try:
+ value = str(value)
+ except RuntimeError:
+ continue
# Here we should use AND self.app.IsParameterEnabled(key) but it's broken
if not self.app.GetParameterRole(key) and (
self.app.HasAutomaticValue(key) or self.app.IsParameterEnabled(key)
):
- if isinstance(value, otb.ApplicationProxy):
- try:
- value = str(value)
- except RuntimeError:
- continue # root of param group
self._auto_parameters[key] = value
# Save static output data (ReadImageInfo, ComputeImageStatistics, etc.)
elif self.app.GetParameterRole(key) == 1 and (bool(value) or value == 0):
--
GitLab
From c6374d4cd0264a1df9b18d56af0b8173291b0527 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 1 Jul 2023 12:43:54 +0200
Subject: [PATCH 233/399] FIX: skip empty collections or strings for
_auto_parameters or data
---
pyotb/core.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index afd5787..1738cbf 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -967,13 +967,15 @@ class App(OTBObject):
value = str(value)
except RuntimeError:
continue
+ if not (bool(value) or value == 0):
+ continue
# Here we should use AND self.app.IsParameterEnabled(key) but it's broken
- if not self.app.GetParameterRole(key) and (
+ if self.app.GetParameterRole(key) == 0 and (
self.app.HasAutomaticValue(key) or self.app.IsParameterEnabled(key)
):
self._auto_parameters[key] = value
# Save static output data (ReadImageInfo, ComputeImageStatistics, etc.)
- elif self.app.GetParameterRole(key) == 1 and (bool(value) or value == 0):
+ elif self.app.GetParameterRole(key) == 1:
if isinstance(value, str):
try:
value = literal_eval(value)
--
GitLab
From c8ab03e3c46aa5aa0c19545daf595b4e48d9bf3e Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 1 Jul 2023 12:44:07 +0200
Subject: [PATCH 234/399] TEST: pytest verbose
---
.gitlab-ci.yml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a5d80fd..3e7f69d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -83,17 +83,17 @@ codespell:
test_core:
extends: .tests
script:
- - pytest --color=yes --junitxml=test-core.xml tests/test_core.py
+ - pytest -vv --color=yes --junitxml=test-core.xml tests/test_core.py
test_numpy:
extends: .tests
script:
- - pytest --color=yes --junitxml=test-numpy.xml tests/test_numpy.py
+ - pytest -vv --color=yes --junitxml=test-numpy.xml tests/test_numpy.py
test_pipeline:
extends: .tests
script:
- - pytest --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py
+ - pytest -vv --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py
# -------------------------------------- Docs ---------------------------------------
--
GitLab
From 48ee49e216c58f15fea07984900c321ed5c4088f Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 2 Jul 2023 11:58:23 +0200
Subject: [PATCH 235/399] CI: add badge for develop to README, instead of
master
---
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 5447959..d51ba17 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
# pyotb: a pythonic extension of Orfeo Toolbox
[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/releases)
-[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/master)
+[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop)
[](https://pyotb.readthedocs.io/en/master/)
**pyotb** wraps the [Orfeo Toolbox](https://www.orfeo-toolbox.org/) (OTB)
--
GitLab
From 0fb7370181ecbe95a08df9fa5f8d007383ecf9be Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 2 Jul 2023 11:59:36 +0200
Subject: [PATCH 236/399] ENH: update comment and docstring for
__sync_parameters
---
pyotb/core.py | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 1738cbf..decd8aa 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -550,7 +550,7 @@ class App(OTBObject):
@property
def parameters(self):
- """Return used OTB application parameters."""
+ """Return used application parameters: automatic values or set by user."""
return {**self._auto_parameters, **self._settings}
@property
@@ -739,7 +739,7 @@ class App(OTBObject):
self.frozen = False
self._time_end = perf_counter()
logger.debug("%s: execution ended", self.name)
- self.__sync_parameters() # this is required for apps like ReadImageInfo or ComputeImagesStatistics
+ self.__sync_parameters()
def flush(self):
"""Flush data to disk, this is when WriteOutput is actually called."""
@@ -945,7 +945,11 @@ class App(OTBObject):
self.app.SetParameterValue(key, obj)
def __sync_parameters(self):
- """Save OTB parameters in _settings, data and outputs dict, for a list of keys or all parameters."""
+ """Save app parameters in _auto_parameters or data dict.
+
+ This is always called during init or after execution, to ensure the
+ parameters property of the App is in sync with the otb.Application instance.
+ """
skip = [
k for k in self.parameters_keys if k.split(".")[-1] in ("ram", "default")
]
--
GitLab
From 182d05946be0346c4e6493a019f5bbb03c07f41d Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 2 Jul 2023 12:13:23 +0200
Subject: [PATCH 237/399] CI: update pipeline test data with missing parameter
values
---
tests/serialized_apps.json | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/tests/serialized_apps.json b/tests/serialized_apps.json
index 424afba..938958c 100644
--- a/tests/serialized_apps.json
+++ b/tests/serialized_apps.json
@@ -3,6 +3,9 @@
"name": "ManageNoData",
"parameters": {
"mode": "buildmask",
+ "mode.buildmask.inv": 1.0,
+ "mode.buildmask.outv": 0.0,
+ "usenan": false,
"in": {
"name": "OrthoRectification",
"parameters": {
@@ -19,8 +22,10 @@
"outputs.lry": 5493909.5,
"outputs.isotropic": true,
"opt.gridspacing": 4.0,
+ "opt.rpc": 10,
"outputs.mode": "auto",
"interpolator": "bco",
+ "interpolator.bco.radius": 2,
"io.in": {
"name": "BandMath",
"parameters": {
@@ -54,8 +59,10 @@
"outputs.lry": 5493909.5,
"outputs.isotropic": true,
"opt.gridspacing": 4.0,
+ "opt.rpc": 10,
"outputs.mode": "auto",
"interpolator": "bco",
+ "interpolator.bco.radius": 2,
"io.in": {
"name": "BandMath",
"parameters": {
@@ -71,6 +78,9 @@
"name": "ManageNoData",
"parameters": {
"mode": "buildmask",
+ "mode.buildmask.inv": 1.0,
+ "mode.buildmask.outv": 0.0,
+ "usenan": false,
"in": {
"name": "OrthoRectification",
"parameters": {
@@ -87,8 +97,10 @@
"outputs.lry": 5493909.5,
"outputs.isotropic": true,
"opt.gridspacing": 4.0,
+ "opt.rpc": 10,
"outputs.mode": "auto",
"interpolator": "bco",
+ "interpolator.bco.radius": 2,
"io.in": {
"name": "BandMath",
"parameters": {
--
GitLab
From 576305c8715bb6db69051684e01b8b138bd0731e Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 2 Jul 2023 12:42:22 +0200
Subject: [PATCH 238/399] FIX: sync_parameters make sure to keep False bool
param values
---
pyotb/core.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index decd8aa..94da09b 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -971,7 +971,8 @@ class App(OTBObject):
value = str(value)
except RuntimeError:
continue
- if not (bool(value) or value == 0):
+ # Keep False or 0 values, but make sure to skip empty collections or str
+ if hasattr(value, "__iter__") and not value:
continue
# Here we should use AND self.app.IsParameterEnabled(key) but it's broken
if self.app.GetParameterRole(key) == 0 and (
--
GitLab
From c99f0131dcf6d8b5dec3127e968b44e7257ad1ab Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 3 Jul 2023 09:42:01 +0200
Subject: [PATCH 239/399] ADD: test for autoparameters
---
tests/test_core.py | 30 ++++++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
diff --git a/tests/test_core.py b/tests/test_core.py
index 0c511e7..2e07b96 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -360,6 +360,36 @@ def test_summarize_strip_output():
assert summary["parameters"][key] == expected, \
f"Failed for input {inp}, output {out}, args {extra_args}"
+def test_summarize_consistency():
+ app_fns = [
+ lambda inp: pyotb.ExtractROI(
+ {"in": inp, "startx": 10, "starty": 10, "sizex": 50, "sizey": 50}
+ ),
+ lambda inp: pyotb.ManageNoData({"in": inp, "mode": "changevalue"}),
+ lambda inp: pyotb.DynamicConvert({"in": inp}),
+ lambda inp: pyotb.Mosaic({"il": [inp]}),
+ lambda inp: pyotb.BandMath({"il": [inp], "exp": "im1b1 + 1"}),
+ lambda inp: pyotb.BandMathX({"il": [inp], "exp": "im1"}),
+ lambda inp: pyotb.OrthoRectification({"io.in": inp}),
+ ]
+ def _test(app_fn):
+ """
+ Here we create 2 summaries:
+ - summary of the app before write()
+ - summary of the app after write()
+ Then we check that both only differ with the output parameter
+ """
+ app = app_fn(inp=FILEPATH)
+ out_file = "/dev/shm/out.tif"
+ out_key = app.output_image_key
+ summary_wo_wrt = pyotb.summarize(app)
+ app.write(out_file)
+ summay_w_wrt = pyotb.summarize(app)
+ app[out_key].filepath.unlink()
+ summary_wo_wrt["parameters"].update({out_key: out_file})
+ assert summary_wo_wrt == summay_w_wrt
+ for app_fn in app_fns:
+ _test(app_fn)
def test_pipeline_simple():
# BandMath -> OrthoRectification -> ManageNoData
--
GitLab
From 45467540327f5c4985c02626dd88a3008e4c3a6d Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 5 Jul 2023 22:20:00 +0200
Subject: [PATCH 240/399] WIP: depreciation
---
pyotb/__init__.py | 12 ++++-
pyotb/core.py | 65 ++++++++++++++++++++++++
pyotb/depreciation.py | 114 ++++++++++++++++++++++++++++++++++++++++++
3 files changed, 190 insertions(+), 1 deletion(-)
create mode 100644 pyotb/depreciation.py
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index 3e09a63..e6d3a97 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -4,7 +4,17 @@ __version__ = "2.0.0.dev4"
from .helpers import logger, set_logger_level
from .apps import *
-from .core import App, Input, Output, get_nbchannels, get_pixel_type, summarize
+from .core import (
+ App,
+ Input,
+ Output,
+ get_nbchannels,
+ get_pixel_type,
+ summarize,
+ OTBObject,
+ otbObject,
+)
+
from .functions import ( # pylint: disable=redefined-builtin
all,
any,
diff --git a/pyotb/core.py b/pyotb/core.py
index 94da09b..08c9fb5 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -11,7 +11,9 @@ from typing import Any
import numpy as np
import otbApplication as otb # pylint: disable=import-error
+import pyotb
from .helpers import logger
+from .depreciation import deprecated_alias, warning_msg, deprecated_attr
class OTBObject(ABC):
@@ -32,6 +34,14 @@ class OTBObject(ABC):
def output_image_key(self) -> str:
"""Return the name of a parameter key associated to the main output image of the object."""
+ @property
+ @deprecated_attr(replacement="output_image_key")
+ def output_param(self) -> str:
+ """
+ Return the name of a parameter key associated to the main output image
+ of the object (deprecated).
+ """
+
@property
@abstractmethod
def exports_dic(self) -> dict[str, dict]:
@@ -109,6 +119,21 @@ class OTBObject(ABC):
origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2
return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y
+
+ def summarize(self, *args, **kwargs):
+ """Summarize the app with `pyotb.summarize()`.
+
+ Args:
+ *args: args
+ **kwargs: keyword args
+
+ Returns:
+ app summary
+
+ """
+ return pyotb.summarize(self, *args, **kwargs)
+
+
def get_info(self) -> dict[str, (str, float, list[float])]:
"""Return a dict output of ReadImageInfo for the first image output."""
return App("ReadImageInfo", self, quiet=True).data
@@ -393,6 +418,30 @@ class OTBObject(ABC):
"""
return id(self)
+ def __getattr__(self, item: str):
+ """
+ Provides depreciation of old methods to access the OTB application
+ values.
+ ```
+
+ Args:
+ item: attribute name
+
+ Returns:
+ attribute from self.app
+
+ """
+ getattr(self.app, item)
+ raise DeprecationWarning(
+ "Since pyotb 2.0.0, OTBObject instances have stopped to "
+ "forward attributes to their own internal otbApplication "
+ "instance. `App.app` can be used to call otbApplications "
+ f"methods. Attribute was: \"{item}\". Hint: maybe try "
+ f"`pyotb_app.app.{item}` instead of `pyotb_app.{item}`?"
+ )
+
+
+
def __getitem__(self, key) -> Any | list[float] | float | Slicer:
"""Override the default __getitem__ behaviour.
@@ -436,6 +485,14 @@ class OTBObject(ABC):
return f"<pyotb.{self.__class__.__name__} object, id {id(self)}>"
+class otbObject(OTBObject): # pylint: noqa
+ def __init_subclass__(cls):
+ warning_msg(
+ "Since pyotb 2.0.0, otbObject has been renamed OTBObject. "
+ "otbObject will be removed definitively in future releases."
+ )
+
+
class App(OTBObject):
"""Base class that gathers common operations for any OTB application."""
@@ -757,6 +814,7 @@ class App(OTBObject):
self.frozen = False
self._time_end = perf_counter()
+ @deprecated_alias(filename_extension="ext_fname")
def write(
self,
path: str | Path | dict[str, str] = None,
@@ -1431,6 +1489,8 @@ class Output(OTBObject):
_filepath: str | Path = None
+
+ @deprecated_alias(app="pyotb_app", output_parameter_key="param_key")
def __init__(
self,
pyotb_app: App,
@@ -1463,6 +1523,11 @@ class Output(OTBObject):
"""Reference to the parent pyotb otb.Application instance."""
return self.parent_pyotb_app.app
+ @property
+ @deprecated_attr(replacement="parent_pyotb_app")
+ def pyotb_app(self) -> App:
+ """Reference to the parent pyotb App (deprecated)."""
+
@property
def exports_dic(self) -> dict[str, dict]:
"""Returns internal _exports_dic object that contains numpy array exports."""
diff --git a/pyotb/depreciation.py b/pyotb/depreciation.py
new file mode 100644
index 0000000..87d1d77
--- /dev/null
+++ b/pyotb/depreciation.py
@@ -0,0 +1,114 @@
+"""Helps with deprecated classes and methods.
+
+Taken from https://stackoverflow.com/questions/49802412/how-to-implement-
+deprecation-in-python-with-argument-alias
+"""
+from typing import Callable, Dict, Any
+import functools
+import warnings
+
+
+WARN = '\033[91m'
+ENDC = '\033[0m'
+OKAY = '\033[92m'
+
+def warning_msg(message: str):
+ """
+ Shows a warning message.
+
+ Args:
+ message: message
+
+ """
+ warnings.warn(
+ message=message,
+ category=DeprecationWarning,
+ stacklevel=3,
+ )
+def deprecated_alias(**aliases: str) -> Callable:
+ """
+ Decorator for deprecated function and method arguments.
+
+ Use as follows:
+
+ @deprecated_alias(old_arg='new_arg')
+ def myfunc(new_arg):
+ ...
+
+ Args:
+ **aliases: aliases
+
+ Returns:
+ wrapped function
+
+ """
+ def deco(f: Callable):
+ @functools.wraps(f)
+ def wrapper(*args, **kwargs):
+ rename_kwargs(f.__name__, kwargs, aliases)
+ return f(*args, **kwargs)
+
+ return wrapper
+
+ return deco
+
+
+def rename_kwargs(
+ func_name: str,
+ kwargs: Dict[str, Any],
+ aliases: Dict[str, str]
+):
+ """
+ Helper function for deprecating function arguments.
+
+ Args:
+ func_name: function
+ kwargs: keyword args
+ aliases: aliases
+
+ """
+ for alias, new in aliases.items():
+ if alias in kwargs:
+ if new in kwargs:
+ raise TypeError(
+ f"{func_name} received both {alias} and {new} as arguments!"
+ f" {alias} is deprecated, use {new} instead."
+ )
+ message = (
+ f"{WARN}`{alias}`{ENDC} is deprecated as an argument to "
+ f"`{func_name}`; use {OKAY}`{new}`{ENDC} instead."
+ )
+ warning_msg(message)
+ kwargs[new] = kwargs.pop(alias)
+
+def deprecated_attr(replacement: str) -> Callable:
+ """
+ Decorator for deprecated attr.
+
+ Use as follows:
+
+ @deprecated_attr(replacement='new_attr')
+ def old_attr(...):
+ ...
+
+ Args:
+ replacement: name of the new attr (method or attribute)
+
+ Returns:
+ wrapped function
+
+ """
+ def deco(attr: Any):
+ @functools.wraps(attr)
+ def wrapper(self, *args, **kwargs):
+ warning_msg(
+ f"{WARN}`{attr.__name__}`{ENDC} will be removed in future "
+ f"releases. Please replace {WARN}`{attr.__name__}`{ENDC} with "
+ f"{OKAY}`{replacement}`{ENDC}."
+ )
+ g = getattr(self, replacement)
+ return g(*args, **kwargs) if isinstance(g, Callable) else g
+
+ return wrapper
+
+ return deco
--
GitLab
From 09920054c29588786a7a0b6db0b5bc55b775118f Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 5 Jul 2023 22:31:23 +0200
Subject: [PATCH 241/399] WIP: depreciation
---
pyotb/core.py | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 08c9fb5..298389d 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -11,7 +11,6 @@ from typing import Any
import numpy as np
import otbApplication as otb # pylint: disable=import-error
-import pyotb
from .helpers import logger
from .depreciation import deprecated_alias, warning_msg, deprecated_attr
@@ -124,14 +123,14 @@ class OTBObject(ABC):
"""Summarize the app with `pyotb.summarize()`.
Args:
- *args: args
- **kwargs: keyword args
+ *args: args for `pyotb.summarize()`
+ **kwargs: keyword args for `pyotb.summarize()`
Returns:
- app summary
+ app summary, same as `pyotb.summarize()`
"""
- return pyotb.summarize(self, *args, **kwargs)
+ return summarize(self, *args, **kwargs)
def get_info(self) -> dict[str, (str, float, list[float])]:
--
GitLab
From 7ef93cd298da8d26280ffaf3ea710641cbeb9c70 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 5 Jul 2023 22:42:13 +0200
Subject: [PATCH 242/399] WIP: depreciation
---
pyotb/core.py | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 298389d..9ceab97 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -431,12 +431,19 @@ class OTBObject(ABC):
"""
getattr(self.app, item)
+ note = ""
+ if item.startswith("GetParameter"):
+ note = (
+ "Note: `pyotb_app.app.GetParameterValue('paramname')` can be "
+ "shorten with `pyotb_app['paramname']` to access parameters "
+ "values."
+ )
raise DeprecationWarning(
"Since pyotb 2.0.0, OTBObject instances have stopped to "
"forward attributes to their own internal otbApplication "
"instance. `App.app` can be used to call otbApplications "
f"methods. Attribute was: \"{item}\". Hint: maybe try "
- f"`pyotb_app.app.{item}` instead of `pyotb_app.{item}`?"
+ f"`pyotb_app.app.{item}` instead of `pyotb_app.{item}`? {note}"
)
--
GitLab
From 64d2dc7d83f7dcdd964ee760b83f629e808f6d0c Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 6 Jul 2023 11:07:10 +0200
Subject: [PATCH 243/399] WIP: depreciation
---
pyotb/core.py | 57 ++++++++++++++++++++++++++-----------------
pyotb/depreciation.py | 12 +++------
2 files changed, 39 insertions(+), 30 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 9ceab97..3a284bc 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -36,10 +36,7 @@ class OTBObject(ABC):
@property
@deprecated_attr(replacement="output_image_key")
def output_param(self) -> str:
- """
- Return the name of a parameter key associated to the main output image
- of the object (deprecated).
- """
+ """Return the name of a parameter key associated to the main output image of the object (deprecated)."""
@property
@abstractmethod
@@ -120,7 +117,7 @@ class OTBObject(ABC):
def summarize(self, *args, **kwargs):
- """Summarize the app with `pyotb.summarize()`.
+ """Call `pyotb.summarize()` on itself.
Args:
*args: args for `pyotb.summarize()`
@@ -418,33 +415,47 @@ class OTBObject(ABC):
return id(self)
def __getattr__(self, item: str):
- """
- Provides depreciation of old methods to access the OTB application
- values.
- ```
+ """Provides depreciation of old methods to access the OTB application values.
+
+ This function will be removed completely in future releases.
Args:
item: attribute name
- Returns:
- attribute from self.app
"""
- getattr(self.app, item)
- note = ""
+ note = (
+ "Since pyotb 2.0.0, OTBObject instances have stopped to forward "
+ "attributes to their own internal otbApplication instance. "
+ "`App.app` can be used to call otbApplications methods. "
+ )
+
+ if item[0].isupper():
+ # Because otbApplication instances methods names start with an
+ # upper case
+ note += (
+ f"Maybe try `pyotb_app.app.{item}` instead of "
+ f"`pyotb_app.{item}`? "
+ )
+
+ if item[0].islower():
+ # Because in pyotb 1.5.4, applications outputs were added as
+ # attributes of the instance
+ note += (
+ "Note: `pyotb_app.paramname` is no longer supported. Starting "
+ "from pytob 2.0.0, `pyotb_app['paramname']` can be used to "
+ "access parameters values. "
+ )
+
if item.startswith("GetParameter"):
- note = (
+ note += (
"Note: `pyotb_app.app.GetParameterValue('paramname')` can be "
"shorten with `pyotb_app['paramname']` to access parameters "
"values."
)
- raise DeprecationWarning(
- "Since pyotb 2.0.0, OTBObject instances have stopped to "
- "forward attributes to their own internal otbApplication "
- "instance. `App.app` can be used to call otbApplications "
- f"methods. Attribute was: \"{item}\". Hint: maybe try "
- f"`pyotb_app.app.{item}` instead of `pyotb_app.{item}`? {note}"
- )
+ warning_msg(note)
+ raise AttributeError
+
@@ -491,8 +502,10 @@ class OTBObject(ABC):
return f"<pyotb.{self.__class__.__name__} object, id {id(self)}>"
-class otbObject(OTBObject): # pylint: noqa
+class otbObject(OTBObject): # noqa
+ """Class for depreciation of otbObject since pyotb 2.0.0. Will be removed in future releases."""
def __init_subclass__(cls):
+ """Show a warning for depreciation."""
warning_msg(
"Since pyotb 2.0.0, otbObject has been renamed OTBObject. "
"otbObject will be removed definitively in future releases."
diff --git a/pyotb/depreciation.py b/pyotb/depreciation.py
index 87d1d77..213aa56 100644
--- a/pyotb/depreciation.py
+++ b/pyotb/depreciation.py
@@ -13,8 +13,7 @@ ENDC = '\033[0m'
OKAY = '\033[92m'
def warning_msg(message: str):
- """
- Shows a warning message.
+ """Shows a warning message.
Args:
message: message
@@ -26,8 +25,7 @@ def warning_msg(message: str):
stacklevel=3,
)
def deprecated_alias(**aliases: str) -> Callable:
- """
- Decorator for deprecated function and method arguments.
+ """Decorator for deprecated function and method arguments.
Use as follows:
@@ -58,8 +56,7 @@ def rename_kwargs(
kwargs: Dict[str, Any],
aliases: Dict[str, str]
):
- """
- Helper function for deprecating function arguments.
+ """Helper function for deprecating function arguments.
Args:
func_name: function
@@ -82,8 +79,7 @@ def rename_kwargs(
kwargs[new] = kwargs.pop(alias)
def deprecated_attr(replacement: str) -> Callable:
- """
- Decorator for deprecated attr.
+ """Decorator for deprecated attr.
Use as follows:
--
GitLab
From 240ce351b146b7b7ee0cfd43415a95ccafaff22c Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 6 Jul 2023 11:15:14 +0200
Subject: [PATCH 244/399] WIP: depreciation
---
pyotb/core.py | 4 ++--
pyotb/depreciation.py | 12 ++++++------
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 3a284bc..e35c564 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -117,7 +117,7 @@ class OTBObject(ABC):
def summarize(self, *args, **kwargs):
- """Call `pyotb.summarize()` on itself.
+ """Recursively summarize parameters and parents.
Args:
*args: args for `pyotb.summarize()`
@@ -502,7 +502,7 @@ class OTBObject(ABC):
return f"<pyotb.{self.__class__.__name__} object, id {id(self)}>"
-class otbObject(OTBObject): # noqa
+class otbObject(OTBObject): # noqa, pylint: disable=invalid-name
"""Class for depreciation of otbObject since pyotb 2.0.0. Will be removed in future releases."""
def __init_subclass__(cls):
"""Show a warning for depreciation."""
diff --git a/pyotb/depreciation.py b/pyotb/depreciation.py
index 213aa56..0234249 100644
--- a/pyotb/depreciation.py
+++ b/pyotb/depreciation.py
@@ -40,11 +40,11 @@ def deprecated_alias(**aliases: str) -> Callable:
wrapped function
"""
- def deco(f: Callable):
- @functools.wraps(f)
+ def deco(func: Callable):
+ @functools.wraps(func)
def wrapper(*args, **kwargs):
- rename_kwargs(f.__name__, kwargs, aliases)
- return f(*args, **kwargs)
+ rename_kwargs(func.__name__, kwargs, aliases)
+ return func(*args, **kwargs)
return wrapper
@@ -102,8 +102,8 @@ def deprecated_attr(replacement: str) -> Callable:
f"releases. Please replace {WARN}`{attr.__name__}`{ENDC} with "
f"{OKAY}`{replacement}`{ENDC}."
)
- g = getattr(self, replacement)
- return g(*args, **kwargs) if isinstance(g, Callable) else g
+ out = getattr(self, replacement)
+ return out(*args, **kwargs) if isinstance(out, Callable) else out
return wrapper
--
GitLab
From 82c93bdb6e8ff537460ad123f1feb7338520afff Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 6 Jul 2023 11:58:06 +0200
Subject: [PATCH 245/399] REFAC: rename warning_msg --> depreciation_warning
---
pyotb/__init__.py | 11 +----------
pyotb/core.py | 6 +++---
pyotb/depreciation.py | 6 +++---
3 files changed, 7 insertions(+), 16 deletions(-)
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index e6d3a97..62508f5 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -4,16 +4,7 @@ __version__ = "2.0.0.dev4"
from .helpers import logger, set_logger_level
from .apps import *
-from .core import (
- App,
- Input,
- Output,
- get_nbchannels,
- get_pixel_type,
- summarize,
- OTBObject,
- otbObject,
-)
+from .core import App, Input, Output, get_nbchannels, get_pixel_type, summarize, OTBObject, otbObject
from .functions import ( # pylint: disable=redefined-builtin
all,
diff --git a/pyotb/core.py b/pyotb/core.py
index e35c564..b7ee57a 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -12,7 +12,7 @@ import numpy as np
import otbApplication as otb # pylint: disable=import-error
from .helpers import logger
-from .depreciation import deprecated_alias, warning_msg, deprecated_attr
+from .depreciation import deprecated_alias, depreciation_warning, deprecated_attr
class OTBObject(ABC):
@@ -453,7 +453,7 @@ class OTBObject(ABC):
"shorten with `pyotb_app['paramname']` to access parameters "
"values."
)
- warning_msg(note)
+ depreciation_warning(note)
raise AttributeError
@@ -506,7 +506,7 @@ class otbObject(OTBObject): # noqa, pylint: disable=invalid-name
"""Class for depreciation of otbObject since pyotb 2.0.0. Will be removed in future releases."""
def __init_subclass__(cls):
"""Show a warning for depreciation."""
- warning_msg(
+ depreciation_warning(
"Since pyotb 2.0.0, otbObject has been renamed OTBObject. "
"otbObject will be removed definitively in future releases."
)
diff --git a/pyotb/depreciation.py b/pyotb/depreciation.py
index 0234249..859ba60 100644
--- a/pyotb/depreciation.py
+++ b/pyotb/depreciation.py
@@ -12,7 +12,7 @@ WARN = '\033[91m'
ENDC = '\033[0m'
OKAY = '\033[92m'
-def warning_msg(message: str):
+def depreciation_warning(message: str):
"""Shows a warning message.
Args:
@@ -75,7 +75,7 @@ def rename_kwargs(
f"{WARN}`{alias}`{ENDC} is deprecated as an argument to "
f"`{func_name}`; use {OKAY}`{new}`{ENDC} instead."
)
- warning_msg(message)
+ depreciation_warning(message)
kwargs[new] = kwargs.pop(alias)
def deprecated_attr(replacement: str) -> Callable:
@@ -97,7 +97,7 @@ def deprecated_attr(replacement: str) -> Callable:
def deco(attr: Any):
@functools.wraps(attr)
def wrapper(self, *args, **kwargs):
- warning_msg(
+ depreciation_warning(
f"{WARN}`{attr.__name__}`{ENDC} will be removed in future "
f"releases. Please replace {WARN}`{attr.__name__}`{ENDC} with "
f"{OKAY}`{replacement}`{ENDC}."
--
GitLab
From 2a8c3414e37512ac0aadd2e82b1ff2c8331787db Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 6 Jul 2023 12:53:49 +0200
Subject: [PATCH 246/399] DOC: correct typing of ext_fname
---
pyotb/core.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 94da09b..405e736 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -762,7 +762,7 @@ class App(OTBObject):
path: str | Path | dict[str, str] = None,
pixel_type: dict[str, str] | str = None,
preserve_dtype: bool = False,
- ext_fname: str = "",
+ ext_fname: dict[str, str] | str = None,
**kwargs,
) -> bool:
"""Set output pixel type and write the output raster files.
@@ -773,7 +773,7 @@ class App(OTBObject):
non-standard characters such as a point, e.g. {'io.out':'output.tif'}
- None if output file was passed during App init
ext_fname: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES")
- Will be used for all outputs (Default value = "")
+ Will be used for all outputs (Default value = None)
pixel_type: Can be : - dictionary {out_param_key: pixeltype} when specifying for several outputs
- str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several
outputs, all outputs are written with this unique type.
--
GitLab
From 13cd7bbc1128907bbd53e10de51fe1e126ccd10e Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 6 Jul 2023 12:53:59 +0200
Subject: [PATCH 247/399] DOC: fix missing link
---
doc/index.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/index.md b/doc/index.md
index 448d139..7861878 100644
--- a/doc/index.md
+++ b/doc/index.md
@@ -11,7 +11,7 @@ to make OTB more Python friendly.
- [Installation](installation.md)
- [How to use pyotb](quickstart.md)
- [Useful features](features.md)
-- [Functions](features.md)
+- [Functions](functions.md)
- [Interaction with Python libraries (numpy, rasterio, tensorflow)](interaction.md)
## Examples
--
GitLab
From d21a0d6cd92fc80e9e87b80f205381932abb93b3 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 6 Jul 2023 12:54:14 +0200
Subject: [PATCH 248/399] DOC: enhance quickstart
---
doc/quickstart.md | 118 ++++++++++++++++++++++++++++++++++------------
1 file changed, 89 insertions(+), 29 deletions(-)
diff --git a/doc/quickstart.md b/doc/quickstart.md
index 5e07e43..fa26d45 100644
--- a/doc/quickstart.md
+++ b/doc/quickstart.md
@@ -15,12 +15,11 @@ resampled = pyotb.RigidTransformResample({
})
```
-Note that pyotb has a 'lazy' evaluation: it only performs operation when it is
-needed, i.e. results are written to disk.
-Thus, the previous line doesn't trigger the application.
+For now, `resampled` has not been executed. Indeed, pyotb has a 'lazy'
+evaluation: applications are executed only when required. Generally, like in
+this example, executions happen to write output images to disk.
-To actually trigger the application execution, you need to write the result to
-disk:
+To actually trigger the application execution, `write()` has to be called:
```python
resampled.write('output.tif') # this is when the application actually runs
@@ -28,32 +27,39 @@ resampled.write('output.tif') # this is when the application actually runs
### Using Python keyword arguments
-It is also possible to use the Python keyword arguments notation for passing
-the parameters:
+One can use the Python keyword arguments notation for passing that parameters:
```python
output = pyotb.SuperImpose(inr='reference_image.tif', inm='image.tif')
```
-is equivalent to:
+Which is equivalent to:
```python
output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'})
```
-Limitations : for this notation, python doesn't accept the parameter `in` or
-any parameter that contains a dots (e.g. `io.in)`.
-For `in` and other main input parameters of an OTB app, you may simply pass
-the value as first argument, pyotb will guess the parameter name.
-For parameters that contains dots, you can either use a dictionary, or replace dots (`.`) with underscores (`_`) as follow :
+!!! warning
-```python
-resampled = pyotb.RigidTransformResample(
- 'my_image.tif',
- interpolator = 'linear',
- transform_type_id_scaley = 0.5,
- transform_type_id_scalex = 0.5
-)
+ For this notation, python doesn't accept the parameter `in` or any
+ parameter that contains a dots (e.g. `io.in)`. For `in` or other main
+ input parameters of an OTB application, you may simply pass the value as
+ first argument, pyotb will guess the parameter name. For parameters that
+ contains dots, you can either use a dictionary, or replace dots (`.`)
+ with underscores (`_`). Let's take the example of the `OrthoRectification`
+ application of OTB, with the input image parameter named "io.in":
+
+ Option #1, keyword-arg-free:
+
+ ```python
+ ortho = pyotb.OrthoRectification('my_image.tif')
+ ```
+
+ Option #2, replacing dots with underscores in parameter name:
+
+ ```python
+ ortho = pyotb.OrthoRectification(io_in='my_image.tif')
+ ```
## In-memory connections
@@ -93,26 +99,38 @@ dilated.write('result.tif')
## Writing the result of an app
-Any pyotb object can be written to disk using the `write` method, e.g. :
+Any pyotb object can be written to disk using `write()`.
+
+Let's consider the following pyotb application instance:
```python
import pyotb
-
resampled = pyotb.RigidTransformResample({
'in': 'my_image.tif',
'interpolator': 'linear',
'transform.type.id.scaley': 0.5,
'transform.type.id.scalex': 0.5
})
+```
-# Here you can set optionally pixel type and extended filename variables
-resampled.write(
- {'out': 'output.tif'},
- pixel_type='uint16',
- ext_fname='?nodata=65535'
-)
+We can then write the output of `resampled` as following:
+
+```python
+resampled.write('output.tif')
```
+!!! note
+
+ For applications that have multiple outputs, passing a `dict` of filenames
+ can be considered. Let's take the example of `MeanShiftSmoothing` which
+ has 2 output images:
+
+ ```python
+ import pyotb
+ meanshift = pyotb.MeanShiftSmoothing('my_image.tif')
+ meanshift.write({'fout': 'output_1.tif', 'foutpos': 'output_2.tif'})
+ ```
+
Another possibility for writing results is to set the output parameter when
initializing the application:
@@ -126,4 +144,46 @@ resampled = pyotb.RigidTransformResample({
'transform.type.id.scaley': 0.5,
'transform.type.id.scalex': 0.5
})
-```
\ No newline at end of file
+```
+
+### Pixel type
+
+Setting the pixel type is optional, and can be achieved setting the
+`pixel_type` argument:
+
+```python
+resampled.write('output.tif', pixel_type='uint16')
+```
+
+The value of `pixel_type` corresponds to the name of a pixel type from OTB
+applications (e.g. `'uint8'`, `'float'`, etc).
+
+### Extended filenames
+
+Extended filenames can be passed as `str` or `dict`.
+
+As `str`:
+
+```python
+resampled.write(
+ ...
+ ext_fname='nodata=65535&box=0:0:256:256'
+)
+```
+
+As `dict`:
+
+```python
+resampled.write(
+ ...
+ ext_fname={'nodata': '65535', 'box': '0:0:256:256'}
+)
+```
+
+!!! info
+
+ When `ext_fname` is provided and the output filenames contain already some
+ extended filename pattern, the ones provided in the filenames take
+ priority over the ones passed in `ext_fname`. This allows to fine-grained
+ tune extended filenames for each output, with a common extended filenames
+ keys/values basis.
--
GitLab
From 9dc224ea83cf79c5a213bc68e60f4ffe8dd0f193 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 6 Jul 2023 12:59:16 +0200
Subject: [PATCH 249/399] DOC: edit limitation description
---
doc/interaction.md | 10 +++-------
1 file changed, 3 insertions(+), 7 deletions(-)
diff --git a/doc/interaction.md b/doc/interaction.md
index e4c73fb..a87da8e 100644
--- a/doc/interaction.md
+++ b/doc/interaction.md
@@ -138,10 +138,6 @@ memory
!!! warning
- Limitations :
-
- - For OTBTF versions < 4.0.0, it is not possible to use the tensorflow
- python API inside a script where OTBTF is used because of libraries
- clashing between Tensorflow and OTBTF, i.e. `import tensorflow` doesn't
- work in a script where OTBTF apps have been initialized. This is why we
- recommend to use latest OTBTF versions
+ Due to compilation issues in OTBTF before version 4.0.0, tensorflow and
+ pyotb can't be imported in the same python code. This problem has been
+ fixed in OTBTF 4.0.0.
--
GitLab
From e8d99fa004ba994daf88c11d5c0ccfab1169b184 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 6 Jul 2023 13:01:16 +0200
Subject: [PATCH 250/399] DOC: edit limitation description
---
doc/interaction.md | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/doc/interaction.md b/doc/interaction.md
index a87da8e..0195b70 100644
--- a/doc/interaction.md
+++ b/doc/interaction.md
@@ -43,11 +43,9 @@ noisy_image.write('image_plus_noise.tif')
!!! warning
- Limitations :
-
- The whole image is loaded into memory
- - The georeference can not be modified. Thus, numpy operations can not change
- the image or pixel size
+ - The georeference can not be modified. Thus, numpy operations can not
+ change the image or pixel size
## Export to rasterio
--
GitLab
From 29792c3bb3daa389091ad739dc2a263d034d36a7 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 6 Jul 2023 14:30:07 +0200
Subject: [PATCH 251/399] DOC: use otb stylesheet.css
---
doc/stylesheets/extra.css | 4 ----
mkdocs.yml | 2 +-
2 files changed, 1 insertion(+), 5 deletions(-)
delete mode 100644 doc/stylesheets/extra.css
diff --git a/doc/stylesheets/extra.css b/doc/stylesheets/extra.css
deleted file mode 100644
index c67f70f..0000000
--- a/doc/stylesheets/extra.css
+++ /dev/null
@@ -1,4 +0,0 @@
-/* this is for readthedocs theme */
-.wy-nav-content {
- max-width: 1000px;
-}
\ No newline at end of file
diff --git a/mkdocs.yml b/mkdocs.yml
index 29d51d4..d196b27 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -55,7 +55,7 @@ extra:
- icon: fontawesome/brands/gitlab
link: https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb
extra_css:
- - stylesheets/extra.css
+ - https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/8.1.2-rc1/Documentation/Cookbook/_static/css/otb_theme.css
use_directory_urls: false # this creates some pyotb/core.html pages instead of pyotb/core/index.html
markdown_extensions:
--
GitLab
From f226f2f232d818326b5200e4f7cca15ab3a3e100 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 6 Jul 2023 14:30:24 +0200
Subject: [PATCH 252/399] DOC: nipticks
---
doc/quickstart.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/doc/quickstart.md b/doc/quickstart.md
index fa26d45..8343520 100644
--- a/doc/quickstart.md
+++ b/doc/quickstart.md
@@ -42,12 +42,12 @@ output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'})
!!! warning
For this notation, python doesn't accept the parameter `in` or any
- parameter that contains a dots (e.g. `io.in)`. For `in` or other main
+ parameter that contains a dots (e.g. `io.in`). For `in` or other main
input parameters of an OTB application, you may simply pass the value as
first argument, pyotb will guess the parameter name. For parameters that
contains dots, you can either use a dictionary, or replace dots (`.`)
with underscores (`_`). Let's take the example of the `OrthoRectification`
- application of OTB, with the input image parameter named "io.in":
+ application of OTB, with the input image parameter named `io.in`:
Option #1, keyword-arg-free:
--
GitLab
From 51cc690ac7b3d62863dbafd51f513be7fbc41553 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 6 Jul 2023 14:55:11 +0200
Subject: [PATCH 253/399] DOC: add codehilite
---
mkdocs.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/mkdocs.yml b/mkdocs.yml
index d196b27..273d121 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -59,6 +59,7 @@ extra_css:
use_directory_urls: false # this creates some pyotb/core.html pages instead of pyotb/core/index.html
markdown_extensions:
+ - codehilite
- admonition
- toc:
permalink: true
--
GitLab
From 8dcc384b0ecb008d990233845855ae6eda9a499a Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 6 Jul 2023 14:55:20 +0200
Subject: [PATCH 254/399] DOC: enh
---
doc/quickstart.md | 35 ++++++++++++++++++++++++++---------
1 file changed, 26 insertions(+), 9 deletions(-)
diff --git a/doc/quickstart.md b/doc/quickstart.md
index 8343520..e65c68f 100644
--- a/doc/quickstart.md
+++ b/doc/quickstart.md
@@ -46,8 +46,10 @@ output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'})
input parameters of an OTB application, you may simply pass the value as
first argument, pyotb will guess the parameter name. For parameters that
contains dots, you can either use a dictionary, or replace dots (`.`)
- with underscores (`_`). Let's take the example of the `OrthoRectification`
- application of OTB, with the input image parameter named `io.in`:
+ with underscores (`_`).
+
+ Let's take the example of the `OrthoRectification` application of OTB,
+ with the input image parameter named `io.in`:
Option #1, keyword-arg-free:
@@ -63,13 +65,19 @@ output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'})
## In-memory connections
-The big asset of pyotb is the ease of in-memory connections between apps.
+One nice feature of pyotb is in-memory connection between apps. It relies on
+the so-called [streaming](https://www.orfeo-toolbox.org/CookBook/C++/StreamingAndThreading.html)
+mechanism of OTB, that enables to process huge images with a limited memory
+footprint.
-Let's start from our previous example. Consider the case where one wants to
-apply optical calibration and binary morphological dilatation
-following the undersampling.
+pyotb allows to pass any application's output to another. This enables to
+build pipelines composed of several applications.
-Using pyotb, you can pass the output of an app as input of another app :
+Let's start from our previous example. Consider the case where one wants to
+resample the image, then apply optical calibration and binary morphological
+dilatation. We can write the following code to build a pipeline that will
+generate the output in an end-to-end fashion, without being limited with the
+input image size, without writing temporary files.
```python
import pyotb
@@ -91,12 +99,21 @@ dilated = pyotb.BinaryMorphologicalOperation({
'out': 'output.tif',
'filter': 'dilate',
'structype': 'ball',
- 'xradius': 3, 'yradius': 3
+ 'xradius': 3,
+ 'yradius': 3
})
+```
-dilated.write('result.tif')
+We just have built our first pipeline! At this point, it's all symbolic since
+no computation has been performed. To trigger our pipeline, one must call the
+`write()` method from the pipeline termination:
+
+```python
+dilated.write('output.tif')
```
+In the next section, we will detail how `write()` works.
+
## Writing the result of an app
Any pyotb object can be written to disk using `write()`.
--
GitLab
From 356600626849a294970e52daca988c3a6171fb7a Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 6 Jul 2023 15:03:24 +0200
Subject: [PATCH 255/399] DOC: add codehilite
---
mkdocs.yml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/mkdocs.yml b/mkdocs.yml
index 273d121..e80cfac 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -71,6 +71,8 @@ markdown_extensions:
- pymdownx.snippets
- pymdownx.details
- pymdownx.superfences
+ - pymdownx.highlight:
+ use_pygments: true
# Rest of the navigation..
site_name: "pyotb documentation: a Python extension of OTB"
--
GitLab
From 58e1789e2ab613cce9f0099f0cdf829d929b4d7f Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 6 Jul 2023 15:03:33 +0200
Subject: [PATCH 256/399] DOC: add codehilite
---
doc/doc_requirements.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/doc/doc_requirements.txt b/doc/doc_requirements.txt
index 4f59692..931c3df 100644
--- a/doc/doc_requirements.txt
+++ b/doc/doc_requirements.txt
@@ -5,3 +5,4 @@ mkdocs-gen-files
mkdocs-section-index
mkdocs-literate-nav
mkdocs-mermaid2-plugin
+pygments
--
GitLab
From e4d9a3e42092cd6d004f80ec4412c6037799d6f8 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 7 Jul 2023 07:20:16 +0000
Subject: [PATCH 257/399] Apply 1 suggestion(s) to 1 file(s)
---
pyotb/core.py | 24 ++++++++++--------------
1 file changed, 10 insertions(+), 14 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index b7ee57a..ab6b620 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -430,34 +430,30 @@ class OTBObject(ABC):
"`App.app` can be used to call otbApplications methods. "
)
- if item[0].isupper():
+ if item in dir(self.app):
# Because otbApplication instances methods names start with an
# upper case
note += (
f"Maybe try `pyotb_app.app.{item}` instead of "
f"`pyotb_app.{item}`? "
)
+ if item.startswith("GetParameter"):
+ note += (
+ "Note: `pyotb_app.app.GetParameterValue('paramname')` can be "
+ "shorten with `pyotb_app['paramname']` to access parameters "
+ "values."
+ )
- if item[0].islower():
+ elif item in self.parameters_keys:
# Because in pyotb 1.5.4, applications outputs were added as
# attributes of the instance
note += (
"Note: `pyotb_app.paramname` is no longer supported. Starting "
- "from pytob 2.0.0, `pyotb_app['paramname']` can be used to "
+ "from pyotb 2.0.0, `pyotb_app['paramname']` can be used to "
"access parameters values. "
)
-
- if item.startswith("GetParameter"):
- note += (
- "Note: `pyotb_app.app.GetParameterValue('paramname')` can be "
- "shorten with `pyotb_app['paramname']` to access parameters "
- "values."
- )
depreciation_warning(note)
- raise AttributeError
-
-
-
+ raise AttributeError(f"'{type(self)}' object has no attribute '{item}'")
def __getitem__(self, key) -> Any | list[float] | float | Slicer:
"""Override the default __getitem__ behaviour.
--
GitLab
From 0e49fec8ce3c6f61c91e4f2713c58d32f793ad43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr>
Date: Fri, 7 Jul 2023 07:23:37 +0000
Subject: [PATCH 258/399] Apply 1 suggestion(s) to 1 file(s)
---
pyotb/core.py | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index ab6b620..573929a 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -427,18 +427,19 @@ class OTBObject(ABC):
note = (
"Since pyotb 2.0.0, OTBObject instances have stopped to forward "
"attributes to their own internal otbApplication instance. "
- "`App.app` can be used to call otbApplications methods. "
+ "`App.app` can be used to call otbApplications methods."
)
+ hint = None
if item in dir(self.app):
# Because otbApplication instances methods names start with an
# upper case
- note += (
+ hint = (
f"Maybe try `pyotb_app.app.{item}` instead of "
f"`pyotb_app.{item}`? "
)
if item.startswith("GetParameter"):
- note += (
+ hint += (
"Note: `pyotb_app.app.GetParameterValue('paramname')` can be "
"shorten with `pyotb_app['paramname']` to access parameters "
"values."
@@ -447,13 +448,14 @@ class OTBObject(ABC):
elif item in self.parameters_keys:
# Because in pyotb 1.5.4, applications outputs were added as
# attributes of the instance
- note += (
+ hint = (
"Note: `pyotb_app.paramname` is no longer supported. Starting "
"from pyotb 2.0.0, `pyotb_app['paramname']` can be used to "
"access parameters values. "
)
- depreciation_warning(note)
- raise AttributeError(f"'{type(self)}' object has no attribute '{item}'")
+ if hint:
+ depreciation_warning(f"{note} {hint}")
+ raise AttributeError
def __getitem__(self, key) -> Any | list[float] | float | Slicer:
"""Override the default __getitem__ behaviour.
--
GitLab
From 0956d334b86e93471ec52596b73d03e1e05e815a Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 7 Jul 2023 07:26:15 +0000
Subject: [PATCH 259/399] Apply 1 suggestion(s) to 1 file(s)
---
pyotb/core.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 405e736..f9c8aa6 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -772,14 +772,14 @@ class App(OTBObject):
- dictionary containing key-arguments enumeration. Useful when a key contains
non-standard characters such as a point, e.g. {'io.out':'output.tif'}
- None if output file was passed during App init
- ext_fname: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES")
- Will be used for all outputs (Default value = None)
pixel_type: Can be : - dictionary {out_param_key: pixeltype} when specifying for several outputs
- str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several
outputs, all outputs are written with this unique type.
Valid pixel types are uint8, uint16, uint32, int16, int32, float, double,
cint16, cint32, cfloat, cdouble. (Default value = None)
preserve_dtype: propagate main input pixel type to outputs, in case pixel_type is None
+ ext_fname: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES")
+ Will be used for all outputs (Default value = None)
**kwargs: keyword arguments e.g. out='output.tif'
Returns:
--
GitLab
From 167ed2c63725bd060f76f2d88331c0791aa7df85 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 7 Jul 2023 07:27:42 +0000
Subject: [PATCH 260/399] Apply 1 suggestion(s) to 1 file(s)
---
doc/quickstart.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/quickstart.md b/doc/quickstart.md
index e65c68f..111f283 100644
--- a/doc/quickstart.md
+++ b/doc/quickstart.md
@@ -45,7 +45,7 @@ output = pyotb.SuperImpose({'inr': 'reference_image.tif', 'inm': 'image.tif'})
parameter that contains a dots (e.g. `io.in`). For `in` or other main
input parameters of an OTB application, you may simply pass the value as
first argument, pyotb will guess the parameter name. For parameters that
- contains dots, you can either use a dictionary, or replace dots (`.`)
+ contains dots, you can either use a dictionary, or replace dots (`.`)
with underscores (`_`).
Let's take the example of the `OrthoRectification` application of OTB,
--
GitLab
From ff05e9e5839382ac651141f47e42cd19dd2417f1 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 7 Jul 2023 09:38:41 +0200
Subject: [PATCH 261/399] DOC: wip snippets style
---
mkdocs.yml | 1 -
1 file changed, 1 deletion(-)
diff --git a/mkdocs.yml b/mkdocs.yml
index e80cfac..b24d48c 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -59,7 +59,6 @@ extra_css:
use_directory_urls: false # this creates some pyotb/core.html pages instead of pyotb/core/index.html
markdown_extensions:
- - codehilite
- admonition
- toc:
permalink: true
--
GitLab
From 70acbb8bd11812d31b01d2b36d5e929059e13931 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 7 Jul 2023 10:14:49 +0200
Subject: [PATCH 262/399] DOC: style, closer to OTB docs
---
doc/doc_requirements.txt | 1 -
doc/extra.css | 11 +++++++++++
mkdocs.yml | 3 +--
3 files changed, 12 insertions(+), 3 deletions(-)
create mode 100644 doc/extra.css
diff --git a/doc/doc_requirements.txt b/doc/doc_requirements.txt
index 931c3df..4f59692 100644
--- a/doc/doc_requirements.txt
+++ b/doc/doc_requirements.txt
@@ -5,4 +5,3 @@ mkdocs-gen-files
mkdocs-section-index
mkdocs-literate-nav
mkdocs-mermaid2-plugin
-pygments
diff --git a/doc/extra.css b/doc/extra.css
new file mode 100644
index 0000000..ec4d4bb
--- /dev/null
+++ b/doc/extra.css
@@ -0,0 +1,11 @@
+.rst-content div[class^=highlight] {
+ border: 0px;
+}
+
+.rst-content div[class^=highlight] pre {
+ padding: 0px;
+}
+
+.rst-content pre code {
+ background: #eeffcc;
+}
diff --git a/mkdocs.yml b/mkdocs.yml
index b24d48c..18cc088 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -56,6 +56,7 @@ extra:
link: https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb
extra_css:
- https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/8.1.2-rc1/Documentation/Cookbook/_static/css/otb_theme.css
+ - extra.css
use_directory_urls: false # this creates some pyotb/core.html pages instead of pyotb/core/index.html
markdown_extensions:
@@ -70,8 +71,6 @@ markdown_extensions:
- pymdownx.snippets
- pymdownx.details
- pymdownx.superfences
- - pymdownx.highlight:
- use_pygments: true
# Rest of the navigation..
site_name: "pyotb documentation: a Python extension of OTB"
--
GitLab
From 3a1588266ae92d9bb9669260799dc808d8dfcccd Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 7 Jul 2023 10:35:03 +0200
Subject: [PATCH 263/399] DOC: more compact comparison OTB vs pyotb
---
doc/comparison_otb.md | 168 ++++++++++++++++++++++++++----------------
1 file changed, 105 insertions(+), 63 deletions(-)
diff --git a/doc/comparison_otb.md b/doc/comparison_otb.md
index 29d1e89..0129cb2 100644
--- a/doc/comparison_otb.md
+++ b/doc/comparison_otb.md
@@ -11,28 +11,32 @@
<td>
```python
-import otbApplication
+import otbApplication as otb
input_path = 'my_image.tif'
-resampled = otbApplication.Registry.CreateApplication(
+app = otb.Registry.CreateApplication(
'RigidTransformResample'
)
-resampled.SetParameterString('in', input_path)
-resampled.SetParameterString('interpolator', 'linear')
-resampled.SetParameterFloat(
- 'transform.type.id.scalex',
- 0.5
+app.SetParameterString(
+ 'in', input_path
)
-resampled.SetParameterFloat(
- 'transform.type.id.scaley',
- 0.5
+app.SetParameterString(
+ 'interpolator', 'linear'
)
-resampled.SetParameterString('out', 'output.tif')
-resampled.SetParameterOutputImagePixelType(
- 'out', otbApplication.ImagePixelType_uint16
+app.SetParameterFloat(
+ 'transform.type.id.scalex', 0.5
+)
+app.SetParameterFloat(
+ 'transform.type.id.scaley', 0.5
+)
+app.SetParameterString(
+ 'out', 'output.tif'
+)
+app.SetParameterOutputImagePixelType(
+ 'out', otb.ImagePixelType_uint16
)
-resampled.ExecuteAndWriteOutput()
+app.ExecuteAndWriteOutput()
```
</td>
@@ -41,14 +45,17 @@ resampled.ExecuteAndWriteOutput()
```python
import pyotb
-resampled = pyotb.RigidTransformResample({
+app = pyotb.RigidTransformResample({
'in': 'my_image.tif',
'interpolator': 'linear',
'transform.type.id.scaley': 0.5,
'transform.type.id.scalex': 0.5
})
-resampled.write('output.tif', pixel_type='uint16')
+app.write(
+ 'output.tif',
+ pixel_type='uint16'
+)
```
</td>
@@ -66,42 +73,55 @@ resampled.write('output.tif', pixel_type='uint16')
<td>
```python
-import otbApplication
+import otbApplication as otb
-app1 = otbApplication.Registry.CreateApplication(
+app1 = otb.Registry.CreateApplication(
'RigidTransformResample'
)
-app1.SetParameterString('in', 'my_image.tif')
-app1.SetParameterString('interpolator', 'linear')
+app1.SetParameterString(
+ 'in', 'my_image.tif'
+)
+app1.SetParameterString(
+ 'interpolator', 'linear'
+)
app1.SetParameterFloat(
- 'transform.type.id.scalex',
- 0.5
+ 'transform.type.id.scalex', 0.5
)
app1.SetParameterFloat(
- 'transform.type.id.scaley',
- 0.5
+ 'transform.type.id.scaley', 0.5
)
app1.Execute()
-app2 = otbApplication.Registry.CreateApplication(
+app2 = otb.Registry.CreateApplication(
'OpticalCalibration'
)
app2.ConnectImage('in', app1, 'out')
app2.SetParameterString('level', 'toa')
app2.Execute()
-app3 = otbApplication.Registry.CreateApplication(
+app3 = otb.Registry.CreateApplication(
'BinaryMorphologicalOperation'
)
-app3.ConnectImage('in', app2, 'out')
-app3.SetParameterString('filter', 'dilate')
-app3.SetParameterString('structype', 'ball')
-app3.SetParameterInt('xradius', 3)
-app3.SetParameterInt('yradius', 3)
-app3.SetParameterString('out', 'output.tif')
+app3.ConnectImage(
+ 'in', app2, 'out'
+)
+app3.SetParameterString(
+ 'filter', 'dilate'
+)
+app3.SetParameterString(
+ 'structype', 'ball'
+)
+app3.SetParameterInt(
+ 'xradius', 3
+)
+app3.SetParameterInt(
+ 'yradius', 3
+)
+app3.SetParameterString(
+ 'out', 'output.tif'
+)
app3.SetParameterOutputImagePixelType(
- 'out',
- otbApplication.ImagePixelType_uint16
+ 'out', otb.ImagePixelType_uint16
)
app3.ExecuteAndWriteOutput()
```
@@ -159,28 +179,31 @@ Consider an example where we want to perform the arithmetic operation
<td>
```python
-import otbApplication
+import otbApplication as otb
-bmx = otbApplication.Registry.CreateApplication(
+bmx = otb.Registry.CreateApplication(
'BandMathX'
)
bmx.SetParameterStringList(
'il',
- ['image1.tif', 'image2.tif', 'image3.tif']
-) # all images are 3-bands
+ ['im1.tif', 'im2.tif', 'im3.tif']
+)
exp = ('im1b1*im2b1-2*im3b1; '
'im1b2*im2b2-2*im3b2; '
'im1b3*im2b3-2*im3b3')
bmx.SetParameterString('exp', exp)
-bmx.SetParameterString('out', 'output.tif')
+bmx.SetParameterString(
+ 'out',
+ 'output.tif'
+)
bmx.SetParameterOutputImagePixelType(
- 'out',
- otbApplication.ImagePixelType_uint8
+ 'out',
+ otb.ImagePixelType_uint8
)
bmx.ExecuteAndWriteOutput()
```
-In OTB, this code works for 3-bands images.
+Note: code limited to 3-bands images.
</td>
<td>
@@ -188,16 +211,19 @@ In OTB, this code works for 3-bands images.
```python
import pyotb
-# transforming filepaths to pyotb objects
-input1 = pyotb.Input('image1.tif')
-input2 = pyotb.Input('image2.tif')
-input3 = pyotb.Input('image3.tif')
+# filepaths --> pyotb objects
+in1 = pyotb.Input('im1.tif')
+in2 = pyotb.Input('im2.tif')
+in3 = pyotb.Input('im3.tif')
-res = input1 * input2 - 2 * input2
-res.write('output.tif', pixel_type='uint8')
+res = in1 * in2 - 2 * in3
+res.write(
+ 'output.tif',
+ pixel_type='uint8'
+)
```
-In pyotb,this code works with images of any number of bands.
+Note: works with any number of bands.
</td>
</tr>
@@ -215,13 +241,15 @@ In pyotb,this code works with images of any number of bands.
```python
-import otbApplication
+import otbApplication as otb
# first 3 channels
-app = otbApplication.Registry.CreateApplication(
+app = otb.Registry.CreateApplication(
'ExtractROI'
)
-app.SetParameterString('in', 'my_image.tif')
+app.SetParameterString(
+ 'in', 'my_image.tif'
+)
app.SetParameterStringList(
'cl',
['Channel1', 'Channel2', 'Channel3']
@@ -229,16 +257,30 @@ app.SetParameterStringList(
app.Execute()
# 1000x1000 roi
-app = otbApplication.Registry.CreateApplication(
+app = otb.Registry.CreateApplication(
'ExtractROI'
)
-app.SetParameterString('in', 'my_image.tif')
-app.SetParameterString('mode', 'extent')
-app.SetParameterString('mode.extent.unit', 'pxl')
-app.SetParameterFloat('mode.extent.ulx', 0)
-app.SetParameterFloat('mode.extent.uly', 0)
-app.SetParameterFloat('mode.extent.lrx', 999)
-app.SetParameterFloat('mode.extent.lry', 999)
+app.SetParameterString(
+ 'in', 'my_image.tif'
+)
+app.SetParameterString(
+ 'mode', 'extent'
+)
+app.SetParameterString(
+ 'mode.extent.unit', 'pxl'
+)
+app.SetParameterFloat(
+ 'mode.extent.ulx', 0
+)
+app.SetParameterFloat(
+ 'mode.extent.uly', 0
+)
+app.SetParameterFloat(
+ 'mode.extent.lrx', 999
+)
+app.SetParameterFloat(
+ 'mode.extent.lry', 999
+)
app.Execute()
```
@@ -248,11 +290,11 @@ app.Execute()
```python
import pyotb
-# transforming filepath to pyotb object
+# filepath --> pyotb object
inp = pyotb.Input('my_image.tif')
-extracted = inp[:, :, :3] # first 3 channels
-extracted = inp[:1000, :1000] # 1000x1000 roi
+extracted = inp[:, :, :3] # Bands 1,2,3
+extracted = inp[:1000, :1000] # ROI
```
</td>
--
GitLab
From ef5a1dd024a40cc67fa0e1fe86f168f4d7649ce2 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 7 Jul 2023 10:35:12 +0200
Subject: [PATCH 264/399] DOC: title
---
mkdocs.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/mkdocs.yml b/mkdocs.yml
index 18cc088..5d1e8b8 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -72,8 +72,8 @@ markdown_extensions:
- pymdownx.details
- pymdownx.superfences
-# Rest of the navigation..
-site_name: "pyotb documentation: a Python extension of OTB"
+# Rest of the navigation.
+site_name: "pyotb: a Python extension of OTB"
repo_url: https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb
repo_name: pyotb
docs_dir: doc/
--
GitLab
From bee275a5b23c49ee847d15b58d6b81a4ce0ce96c Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 7 Jul 2023 08:38:56 +0000
Subject: [PATCH 265/399] Apply 1 suggestion(s) to 1 file(s)
---
doc/quickstart.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/quickstart.md b/doc/quickstart.md
index 111f283..fb15188 100644
--- a/doc/quickstart.md
+++ b/doc/quickstart.md
@@ -77,7 +77,7 @@ Let's start from our previous example. Consider the case where one wants to
resample the image, then apply optical calibration and binary morphological
dilatation. We can write the following code to build a pipeline that will
generate the output in an end-to-end fashion, without being limited with the
-input image size, without writing temporary files.
+input image size or writing temporary files.
```python
import pyotb
--
GitLab
From 2c94e26a34421d55728a3241da4c56da731a5aff Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 7 Jul 2023 12:20:42 +0200
Subject: [PATCH 266/399] DOC: fix python syntax highlight
---
mkdocs.yml | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/mkdocs.yml b/mkdocs.yml
index 5d1e8b8..88589b1 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -67,13 +67,15 @@ markdown_extensions:
toc_depth: 1-2
- pymdownx.highlight:
anchor_linenums: true
- - pymdownx.inlinehilite
- - pymdownx.snippets
- pymdownx.details
- - pymdownx.superfences
+ - pymdownx.superfences:
+ custom_fences:
+ - name: python
+ class: python
+ format: !!python/name:pymdownx.superfences.fence_code_format
# Rest of the navigation.
-site_name: "pyotb: a Python extension of OTB"
+site_name: "pyotb: Orfeo ToolBox for Python"
repo_url: https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb
repo_name: pyotb
docs_dir: doc/
--
GitLab
From 8b8389aae60ed8a5cb15c67ac1b01956391e9be1 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 7 Jul 2023 17:34:53 +0200
Subject: [PATCH 267/399] DOC: fix summarize api doc
---
pyotb/core.py | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index b7ee57a..f6e3060 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -502,14 +502,20 @@ class OTBObject(ABC):
return f"<pyotb.{self.__class__.__name__} object, id {id(self)}>"
+def old_otbobj_warn():
+ depreciation_warning(
+ "Since pyotb 2.0.0, otbObject has been renamed OTBObject. "
+ "otbObject will be removed definitively in future releases."
+ )
+
class otbObject(OTBObject): # noqa, pylint: disable=invalid-name
"""Class for depreciation of otbObject since pyotb 2.0.0. Will be removed in future releases."""
def __init_subclass__(cls):
"""Show a warning for depreciation."""
- depreciation_warning(
- "Since pyotb 2.0.0, otbObject has been renamed OTBObject. "
- "otbObject will be removed definitively in future releases."
- )
+ old_otbobj_warn()
+ def __class__(self):
+ old_otbobj_warn()
+ return super().__class__
class App(OTBObject):
@@ -1744,8 +1750,7 @@ def summarize(
useful to remove extended filenames.
Returns:
- nested dictionary with serialized App(s) containing name and
- parameters of an app and its parents
+ nested dictionary with serialized App(s) containing name and parameters of an app and its parents
"""
--
GitLab
From 199a9c196ef8be6dcc41e2c21ece9b8fc7d85e09 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 7 Jul 2023 17:35:42 +0200
Subject: [PATCH 268/399] DOC: fix summarize api doc
---
pyotb/core.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index f9c8aa6..9bcd851 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1660,8 +1660,7 @@ def summarize(
useful to remove extended filenames.
Returns:
- nested dictionary with serialized App(s) containing name and
- parameters of an app and its parents
+ nested dictionary with serialized App(s) containing name and parameters of an app and its parents
"""
--
GitLab
From cabb7f0a7a987b358ad8bfe9fe6af119f7d8c994 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 7 Jul 2023 18:31:54 +0200
Subject: [PATCH 269/399] FIX: AttributeError message
---
pyotb/core.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index d5cca0e..f095125 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -455,7 +455,9 @@ class OTBObject(ABC):
)
if hint:
depreciation_warning(f"{note} {hint}")
- raise AttributeError
+ raise AttributeError(
+ f"'{type(self).__name__}' object has no attribute '{item}'"
+ )
def __getitem__(self, key) -> Any | list[float] | float | Slicer:
"""Override the default __getitem__ behaviour.
--
GitLab
From 62bc96a99137467c53d3d6acec44c9d992489a66 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 7 Jul 2023 18:47:21 +0200
Subject: [PATCH 270/399] FIX: AttributeError message
---
pyotb/core.py | 16 ----------------
1 file changed, 16 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index f095125..1a5859c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -502,22 +502,6 @@ class OTBObject(ABC):
return f"<pyotb.{self.__class__.__name__} object, id {id(self)}>"
-def old_otbobj_warn():
- depreciation_warning(
- "Since pyotb 2.0.0, otbObject has been renamed OTBObject. "
- "otbObject will be removed definitively in future releases."
- )
-
-class otbObject(OTBObject): # noqa, pylint: disable=invalid-name
- """Class for depreciation of otbObject since pyotb 2.0.0. Will be removed in future releases."""
- def __init_subclass__(cls):
- """Show a warning for depreciation."""
- old_otbobj_warn()
- def __class__(self):
- old_otbobj_warn()
- return super().__class__
-
-
class App(OTBObject):
"""Base class that gathers common operations for any OTB application."""
--
GitLab
From 56dea1ac019db8c6db47dd26bf2a3c16f0d6f2cd Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Fri, 7 Jul 2023 18:48:00 +0200
Subject: [PATCH 271/399] FIX: remove otbObject
---
pyotb/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index 62508f5..bf55392 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -4,7 +4,7 @@ __version__ = "2.0.0.dev4"
from .helpers import logger, set_logger_level
from .apps import *
-from .core import App, Input, Output, get_nbchannels, get_pixel_type, summarize, OTBObject, otbObject
+from .core import App, Input, Output, get_nbchannels, get_pixel_type, summarize, OTBObject
from .functions import ( # pylint: disable=redefined-builtin
all,
--
GitLab
From 1490f60a6601886588ea788e6f492f647cc02514 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 7 Jul 2023 17:01:05 +0000
Subject: [PATCH 272/399] PERF: sub-processed application listing is not
required with latest OTBTF
---
pyotb/apps.py | 61 ++++++++-------------------------------------------
1 file changed, 9 insertions(+), 52 deletions(-)
diff --git a/pyotb/apps.py b/pyotb/apps.py
index 6f54b27..cd0c696 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -3,9 +3,6 @@
from __future__ import annotations
import os
-import subprocess
-import sys
-from pathlib import Path
import otbApplication as otb # pylint: disable=import-error
@@ -13,7 +10,7 @@ from .core import App
from .helpers import logger
-def get_available_applications(as_subprocess: bool = False) -> list[str]:
+def get_available_applications() -> tuple[str]:
"""Find available OTB applications.
Args:
@@ -23,53 +20,13 @@ def get_available_applications(as_subprocess: bool = False) -> list[str]:
tuple of available applications
"""
- app_list = ()
- if as_subprocess and sys.executable:
- # Currently, there is an incompatibility between OTBTF and Tensorflow that causes segfault
- # when OTBTF apps are used in a script where tensorflow has already been imported.
- # See https://github.com/remicres/otbtf/issues/28
- # Thus, we run this piece of code in a clean independent `subprocess` that doesn't interact with Tensorflow
- env = os.environ.copy()
- if "PYTHONPATH" not in env:
- env["PYTHONPATH"] = ""
- env["PYTHONPATH"] += ":" + str(Path(otb.__file__).parent)
- env[
- "OTB_LOGGER_LEVEL"
- ] = "CRITICAL" # in order to suppress warnings while listing applications
- pycmd = "import otbApplication; print(otbApplication.Registry.GetAvailableApplications())"
- cmd_args = [sys.executable, "-c", pycmd]
- try:
- params = {"env": env, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE}
- with subprocess.Popen(cmd_args, **params) as process:
- logger.debug("Exec \"%s '%s'\"", " ".join(cmd_args[:-1]), pycmd)
- stdout, stderr = process.communicate()
- stdout, stderr = stdout.decode(), stderr.decode()
- # ast.literal_eval is secure and will raise more handy Exceptions than eval
- from ast import literal_eval # pylint: disable=import-outside-toplevel
-
- app_list = literal_eval(stdout.strip())
- assert isinstance(app_list, (tuple, list))
- except subprocess.SubprocessError:
- logger.debug("Failed to call subprocess")
- except (ValueError, SyntaxError, AssertionError):
- logger.debug(
- "Failed to decode output or convert to tuple:\nstdout=%s\nstderr=%s",
- stdout,
- stderr,
- )
- if not app_list:
- logger.debug(
- "Failed to list applications in an independent process. Falling back to local python import"
- )
- # Find applications using the normal way
- if not app_list:
- app_list = otb.Registry.GetAvailableApplications()
- if not app_list:
- raise SystemExit(
- "Unable to load applications. Set env variable OTB_APPLICATION_PATH and try again."
- )
- logger.info("Successfully loaded %s OTB applications", len(app_list))
- return app_list
+ app_list = otb.Registry.GetAvailableApplications()
+ if app_list:
+ logger.info("Successfully loaded %s OTB applications", len(app_list))
+ return app_list
+ raise SystemExit(
+ "Unable to load applications. Set env variable OTB_APPLICATION_PATH and try again."
+ )
class OTBTFApp(App):
@@ -113,7 +70,7 @@ class OTBTFApp(App):
super().__init__(name, *args, **kwargs)
-AVAILABLE_APPLICATIONS = get_available_applications(as_subprocess=True)
+AVAILABLE_APPLICATIONS = get_available_applications()
# This is to enable aliases of Apps, i.e. using apps like `pyotb.AppName(...)` instead of `pyotb.App("AppName", ...)`
_CODE_TEMPLATE = (
--
GitLab
From 3bedecfc229257cc3908417bf044a25ebcb579ec Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 7 Jul 2023 19:19:01 +0200
Subject: [PATCH 273/399] STYLE: run black
---
pyotb/depreciation.py | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
diff --git a/pyotb/depreciation.py b/pyotb/depreciation.py
index 859ba60..687e6ca 100644
--- a/pyotb/depreciation.py
+++ b/pyotb/depreciation.py
@@ -8,9 +8,10 @@ import functools
import warnings
-WARN = '\033[91m'
-ENDC = '\033[0m'
-OKAY = '\033[92m'
+WARN = "\033[91m"
+ENDC = "\033[0m"
+OKAY = "\033[92m"
+
def depreciation_warning(message: str):
"""Shows a warning message.
@@ -24,6 +25,8 @@ def depreciation_warning(message: str):
category=DeprecationWarning,
stacklevel=3,
)
+
+
def deprecated_alias(**aliases: str) -> Callable:
"""Decorator for deprecated function and method arguments.
@@ -40,6 +43,7 @@ def deprecated_alias(**aliases: str) -> Callable:
wrapped function
"""
+
def deco(func: Callable):
@functools.wraps(func)
def wrapper(*args, **kwargs):
@@ -51,11 +55,7 @@ def deprecated_alias(**aliases: str) -> Callable:
return deco
-def rename_kwargs(
- func_name: str,
- kwargs: Dict[str, Any],
- aliases: Dict[str, str]
-):
+def rename_kwargs(func_name: str, kwargs: Dict[str, Any], aliases: Dict[str, str]):
"""Helper function for deprecating function arguments.
Args:
@@ -78,6 +78,7 @@ def rename_kwargs(
depreciation_warning(message)
kwargs[new] = kwargs.pop(alias)
+
def deprecated_attr(replacement: str) -> Callable:
"""Decorator for deprecated attr.
@@ -94,6 +95,7 @@ def deprecated_attr(replacement: str) -> Callable:
wrapped function
"""
+
def deco(attr: Any):
@functools.wraps(attr)
def wrapper(self, *args, **kwargs):
--
GitLab
From 0d7dc0c4fd47f25d8c7a002f64f4cb085b1556cd Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 7 Jul 2023 19:20:36 +0200
Subject: [PATCH 274/399] STYLE: run black on core.py
---
pyotb/core.py | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 1a5859c..d49581f 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -115,7 +115,6 @@ class OTBObject(ABC):
origin_x, origin_y = origin_x - spacing_x / 2, origin_y - spacing_y / 2
return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y
-
def summarize(self, *args, **kwargs):
"""Recursively summarize parameters and parents.
@@ -129,7 +128,6 @@ class OTBObject(ABC):
"""
return summarize(self, *args, **kwargs)
-
def get_info(self) -> dict[str, (str, float, list[float])]:
"""Return a dict output of ReadImageInfo for the first image output."""
return App("ReadImageInfo", self, quiet=True).data
@@ -435,8 +433,7 @@ class OTBObject(ABC):
# Because otbApplication instances methods names start with an
# upper case
hint = (
- f"Maybe try `pyotb_app.app.{item}` instead of "
- f"`pyotb_app.{item}`? "
+ f"Maybe try `pyotb_app.app.{item}` instead of " f"`pyotb_app.{item}`? "
)
if item.startswith("GetParameter"):
hint += (
@@ -1498,7 +1495,6 @@ class Output(OTBObject):
_filepath: str | Path = None
-
@deprecated_alias(app="pyotb_app", output_parameter_key="param_key")
def __init__(
self,
--
GitLab
From 9d27acfa592f927c443edc63a896e42125db1a79 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 8 Jul 2023 07:16:06 +0000
Subject: [PATCH 275/399] Apply 1 suggestion(s) to 1 file(s)
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index d49581f..c8e98a7 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -433,7 +433,7 @@ class OTBObject(ABC):
# Because otbApplication instances methods names start with an
# upper case
hint = (
- f"Maybe try `pyotb_app.app.{item}` instead of " f"`pyotb_app.{item}`? "
+ f"Maybe try `pyotb_app.app.{item}` instead of `pyotb_app.{item}`? "
)
if item.startswith("GetParameter"):
hint += (
--
GitLab
From b5a3c8097bfc463694ccca9bb58e3a491b837585 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 10 Jul 2023 09:08:31 +0200
Subject: [PATCH 276/399] DOC: add a note for migration in troubleshooting
---
doc/troubleshooting.md | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md
index 0025913..ee3bdf4 100644
--- a/doc/troubleshooting.md
+++ b/doc/troubleshooting.md
@@ -1,4 +1,16 @@
-## Troubleshooting: known limitations with old versions
+# Troubleshooting
+
+## Migration from pyotb 1.5.4 (oct 2022) to 2.x.y
+
+- `otbObject` has ben renamed `OTBObject`
+- use `pyotb_app['paramname']` or `pyotb_app.app.GetParameterValue('paramname')` instead of `pyotb_app.GetParameterValue('paramname')` to access parameter `paramname` value
+- use `pyotb_app['paramname']` instead of `pyotb_app.paramname` to access parameter `paramname` value
+- `App.output_param` has been replaced with `App.output_image_key`
+- `App.write()` argument `filename_extension` has been renamed `ext_fname`
+- `Output.__init__()` arguments `app` and `output_parameter_key` have been renamed `pyotb_app` and `param_key`
+- `Output.pyotb_app` has been renamed `Output.parent_pyotb_app`
+
+## Known limitations with old versions
!!! note
--
GitLab
From 19d682d854045f3ac16ff8e7a7a5cb1390ebdc19 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 10 Jul 2023 09:20:04 +0200
Subject: [PATCH 277/399] DOC: add a note for migration in troubleshooting
---
doc/troubleshooting.md | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md
index ee3bdf4..9cf5b84 100644
--- a/doc/troubleshooting.md
+++ b/doc/troubleshooting.md
@@ -2,13 +2,22 @@
## Migration from pyotb 1.5.4 (oct 2022) to 2.x.y
+List of breaking changes:
+
- `otbObject` has ben renamed `OTBObject`
-- use `pyotb_app['paramname']` or `pyotb_app.app.GetParameterValue('paramname')` instead of `pyotb_app.GetParameterValue('paramname')` to access parameter `paramname` value
-- use `pyotb_app['paramname']` instead of `pyotb_app.paramname` to access parameter `paramname` value
+- `otbObject.get_infos()` has been renamed `OTBObject.get_info()`
+- `otbObject.key_output_image` has been renamed `OTBObject.output_image_key`
+- `otbObject.key_input_image` has been renamed `OTBObject.input_image_key`
+- `otbObject.read_values_at_coords()` has been renamed `OTBObject.get_values_at_coords()`
+- `otbObject.xy_to_rowcol()` has been renamed `OTBObject.get_rowcol_from_xy()`
- `App.output_param` has been replaced with `App.output_image_key`
- `App.write()` argument `filename_extension` has been renamed `ext_fname`
+- `App.save_objects()` has been renamed `App.__sync_parameters()`
+- use `pyotb_app['paramname']` or `pyotb_app.app.GetParameterValue('paramname')` instead of `pyotb_app.GetParameterValue('paramname')` to access parameter `paramname` value
+- use `pyotb_app['paramname']` instead of `pyotb_app.paramname` to access parameter `paramname` value
- `Output.__init__()` arguments `app` and `output_parameter_key` have been renamed `pyotb_app` and `param_key`
- `Output.pyotb_app` has been renamed `Output.parent_pyotb_app`
+- `logicalOperation` has been renamed `LogicalOperation`
## Known limitations with old versions
--
GitLab
From b7b7b9cf017f4cc896d25d045f4feff1fc9ed35f Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 11 Jul 2023 15:06:06 +0000
Subject: [PATCH 278/399] CI: add tests coverage + badge + apply black
---
.gitlab-ci.yml | 20 +++---
README.md | 1 +
pyproject.toml | 2 +-
tests/test_core.py | 152 ++++++++++++++++++++++++++++++++++----------
tests/test_numpy.py | 67 -------------------
5 files changed, 130 insertions(+), 112 deletions(-)
delete mode 100644 tests/test_numpy.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3e7f69d..c57f662 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -68,6 +68,7 @@ codespell:
rules:
- changes:
- "**/*.py"
+ - .gitlab-ci.yml
variables:
OTB_ROOT: /opt/otb
LD_LIBRARY_PATH: /opt/otb/lib
@@ -76,19 +77,18 @@ codespell:
artifacts:
reports:
junit: test-*.xml
+ coverage_report:
+ coverage_format: cobertura
+ path: coverage.xml
before_script:
- - pip install pytest
+ - pip install pytest pytest-cov
- pip install .
test_core:
extends: .tests
+ coverage: '/TOTAL.*\s+(\d+%)$/'
script:
- - pytest -vv --color=yes --junitxml=test-core.xml tests/test_core.py
-
-test_numpy:
- extends: .tests
- script:
- - pytest -vv --color=yes --junitxml=test-numpy.xml tests/test_numpy.py
+ - pytest -vv --color=yes --junitxml=test-core.xml --cov=/usr/local/lib/python3.8/dist-packages/pyotb --cov-report term --cov-report xml:coverage.xml tests/test_core.py
test_pipeline:
extends: .tests
@@ -102,9 +102,9 @@ docs:
# when: manual
rules:
- changes:
- - mkdocs.yml
- - doc/*
- - pyotb/*.py
+ - mkdocs.yml
+ - doc/*
+ - pyotb/*.py
before_script:
- apt update && apt install -y python3.8-venv
- python3 -m venv docs_venv
diff --git a/README.md b/README.md
index d51ba17..6659aab 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,7 @@
[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/releases)
[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop)
+[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop)
[](https://pyotb.readthedocs.io/en/master/)
**pyotb** wraps the [Orfeo Toolbox](https://www.orfeo-toolbox.org/) (OTB)
diff --git a/pyproject.toml b/pyproject.toml
index bc2c576..429ed73 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -30,7 +30,7 @@ classifiers = [
]
[project.optional-dependencies]
-dev = ["pytest", "pylint", "codespell", "pydocstyle", "tomli", "requests"]
+dev = ["pytest", "pytest-cov", "pylint", "codespell", "pydocstyle", "tomli", "requests"]
[project.urls]
documentation = "https://pyotb.readthedocs.io"
diff --git a/tests/test_core.py b/tests/test_core.py
index 2e07b96..bbc35ae 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -1,4 +1,5 @@
import pytest
+import numpy as np
from tests_data import *
@@ -30,7 +31,10 @@ def test_input_vsi():
assert info.parameters["in"] == "https://fake.com/image.tif"
# Compressed remote file
info = pyotb.ReadImageInfo("https://fake.com/image.tif.zip", frozen=True)
- assert info.app.GetParameterValue("in") == "/vsizip//vsicurl/https://fake.com/image.tif.zip"
+ assert (
+ info.app.GetParameterValue("in")
+ == "/vsizip//vsicurl/https://fake.com/image.tif.zip"
+ )
assert info.parameters["in"] == "https://fake.com/image.tif.zip"
# Piped curl --> zip --> tiff
ziped_tif_urls = (
@@ -93,8 +97,10 @@ def test_metadata():
assert "ProjectionRef", "OVR_RESAMPLING_ALG" in INPUT2.metadata
# Metadata with numeric values (e.g. TileHintX)
- fp = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/" \
- "Data/Input/radarsat2/RADARSAT2_ALTONA_300_300_VV.tif?inline=false"
+ fp = (
+ "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/"
+ "Data/Input/radarsat2/RADARSAT2_ALTONA_300_300_VV.tif?inline=false"
+ )
app = pyotb.BandMath({"il": [fp], "exp": "im1b1"})
assert "TileHintX" in app.metadata
@@ -130,7 +136,7 @@ def test_write():
def test_ext_fname():
- def _check(expected: str, key: str = "out", app = INPUT.app):
+ def _check(expected: str, key: str = "out", app=INPUT.app):
fn = app.GetParameterString(key)
assert "?&" in fn
assert fn.split("?&", 1)[1] == expected
@@ -142,32 +148,23 @@ def test_ext_fname():
assert INPUT.write("/tmp/test_write.tif", ext_fname={"nodata": 0})
_check("nodata=0")
assert INPUT.write(
- "/tmp/test_write.tif",
- ext_fname={
- "nodata": 0,
- "gdal:co:COMPRESS": "DEFLATE"
- }
+ "/tmp/test_write.tif", ext_fname={"nodata": 0, "gdal:co:COMPRESS": "DEFLATE"}
)
_check("nodata=0&gdal:co:COMPRESS=DEFLATE")
assert INPUT.write(
- "/tmp/test_write.tif",
- ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE"
+ "/tmp/test_write.tif", ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE"
)
_check("nodata=0&gdal:co:COMPRESS=DEFLATE")
assert INPUT.write(
"/tmp/test_write.tif?&box=0:0:10:10",
- ext_fname={
- "nodata": "0",
- "gdal:co:COMPRESS": "DEFLATE",
- "box": "0:0:20:20"
- }
+ ext_fname={"nodata": "0", "gdal:co:COMPRESS": "DEFLATE", "box": "0:0:20:20"},
)
# Check that the bbox is the one specified in the filepath, not the one
# specified in `ext_filename`
_check("nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:10:10")
assert INPUT.write(
"/tmp/test_write.tif?&box=0:0:10:10",
- ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:20:20"
+ ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:20:20",
)
_check("nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:10:10")
@@ -177,12 +174,9 @@ def test_ext_fname():
mss.write(
{
"fout": "/tmp/test_ext_fn_fout.tif?&nodata=1",
- "foutpos": "/tmp/test_ext_fn_foutpos.tif?&nodata=2"
+ "foutpos": "/tmp/test_ext_fn_foutpos.tif?&nodata=2",
},
- ext_fname={
- "nodata": 0,
- "gdal:co:COMPRESS": "DEFLATE"
- }
+ ext_fname={"nodata": 0, "gdal:co:COMPRESS": "DEFLATE"},
)
_check("nodata=1&gdal:co:COMPRESS=DEFLATE", key="fout", app=mss.app)
_check("nodata=2&gdal:co:COMPRESS=DEFLATE", key="foutpos", app=mss.app)
@@ -190,13 +184,14 @@ def test_ext_fname():
mss["foutpos"].filepath.unlink()
-
def test_frozen_app_write():
app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True)
assert app.write("/tmp/test_frozen_app_write.tif")
app["out"].filepath.unlink()
- app = pyotb.BandMath(INPUT, exp="im1b1", out="/tmp/test_frozen_app_write.tif", frozen=True)
+ app = pyotb.BandMath(
+ INPUT, exp="im1b1", out="/tmp/test_frozen_app_write.tif", frozen=True
+ )
assert app.write()
app["out"].filepath.unlink()
@@ -244,7 +239,9 @@ def test_rational_operators():
meas = func(INPUT)
ref = pyotb.BandMathX({"il": [FILEPATH], "exp": exp})
for i in range(1, 5):
- compared = pyotb.CompareImages({"ref.in": ref, "meas.in": meas, "ref.channel": i, "meas.channel": i})
+ compared = pyotb.CompareImages(
+ {"ref.in": ref, "meas.in": meas, "ref.channel": i, "meas.channel": i}
+ )
assert (compared["count"], compared["mse"]) == (0, 0)
_test(lambda x: x + x, "im1 + im1")
@@ -279,7 +276,10 @@ def test_rational_operators():
def test_operation():
op = INPUT / 255 * 128
- assert op.exp == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)"
+ assert (
+ op.exp
+ == "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)"
+ )
assert op.dtype == "float32"
@@ -296,7 +296,10 @@ def test_binary_mask_where():
# Create binary mask based on several possible values
values = [1, 2, 3, 4]
res = pyotb.where(pyotb.any(INPUT[:, :, 0] == value for value in values), 255, 0)
- assert res.exp == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)"
+ assert (
+ res.exp
+ == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)"
+ )
# Essential apps
@@ -323,18 +326,27 @@ def test_read_values_at_coords():
# BandMath NDVI == RadiometricIndices NDVI ?
def test_ndvi_comparison():
- ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (INPUT[:, :, -1] + INPUT[:, :, 0])
- ndvi_indices = pyotb.RadiometricIndices(INPUT, {"list": ["Vegetation:NDVI"], "channels.red": 1, "channels.nir": 4})
+ ndvi_bandmath = (INPUT[:, :, -1] - INPUT[:, :, [0]]) / (
+ INPUT[:, :, -1] + INPUT[:, :, 0]
+ )
+ ndvi_indices = pyotb.RadiometricIndices(
+ INPUT, {"list": ["Vegetation:NDVI"], "channels.red": 1, "channels.nir": 4}
+ )
assert ndvi_bandmath.exp == "((im1b4 - im1b1) / (im1b4 + im1b1))"
assert ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", "float")
assert ndvi_indices.write("/tmp/ndvi_indices.tif", "float")
- compared = pyotb.CompareImages({"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"})
+ compared = pyotb.CompareImages(
+ {"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"}
+ )
assert (compared["count"], compared["mse"]) == (0, 0)
thresholded_indices = pyotb.where(ndvi_indices >= 0.3, 1, 0)
assert thresholded_indices["exp"] == "((im1b1 >= 0.3) ? 1 : 0)"
thresholded_bandmath = pyotb.where(ndvi_bandmath >= 0.3, 1, 0)
- assert thresholded_bandmath["exp"] == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)"
+ assert (
+ thresholded_bandmath["exp"]
+ == "((((im1b4 - im1b1) / (im1b4 + im1b1)) >= 0.3) ? 1 : 0)"
+ )
def test_summarize_output():
@@ -351,14 +363,16 @@ def test_summarize_strip_output():
(in_fn, out_fn_w_ext, "out", {}, out_fn_w_ext),
(in_fn, out_fn_w_ext, "out", {"strip_output_paths": True}, out_fn),
(in_fn_w_ext, out_fn, "in", {}, in_fn_w_ext),
- (in_fn_w_ext, out_fn, "in", {"strip_input_paths": True}, in_fn)
+ (in_fn_w_ext, out_fn, "in", {"strip_input_paths": True}, in_fn),
]
for inp, out, key, extra_args, expected in baseline:
app = pyotb.ExtractROI({"in": inp, "out": out})
summary = pyotb.summarize(app, **extra_args)
- assert summary["parameters"][key] == expected, \
- f"Failed for input {inp}, output {out}, args {extra_args}"
+ assert (
+ summary["parameters"][key] == expected
+ ), f"Failed for input {inp}, output {out}, args {extra_args}"
+
def test_summarize_consistency():
app_fns = [
@@ -372,6 +386,7 @@ def test_summarize_consistency():
lambda inp: pyotb.BandMathX({"il": [inp], "exp": "im1"}),
lambda inp: pyotb.OrthoRectification({"io.in": inp}),
]
+
def _test(app_fn):
"""
Here we create 2 summaries:
@@ -388,9 +403,12 @@ def test_summarize_consistency():
app[out_key].filepath.unlink()
summary_wo_wrt["parameters"].update({out_key: out_file})
assert summary_wo_wrt == summay_w_wrt
+
for app_fn in app_fns:
_test(app_fn)
+
+@pytest.mark.xfail
def test_pipeline_simple():
# BandMath -> OrthoRectification -> ManageNoData
app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"})
@@ -400,6 +418,7 @@ def test_pipeline_simple():
assert summary == SIMPLE_SERIALIZATION
+@pytest.mark.xfail
def test_pipeline_diamond():
# Diamond graph
app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"})
@@ -408,3 +427,68 @@ def test_pipeline_diamond():
app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"})
summary = pyotb.summarize(app4)
assert summary == COMPLEX_SERIALIZATION
+
+
+# Numpy funcs
+def test_export():
+ INPUT.export()
+ array = INPUT.exports_dic[INPUT.output_image_key]["array"]
+ assert isinstance(array, np.ndarray)
+ assert array.dtype == "uint8"
+ del INPUT.exports_dic["out"]
+
+
+def test_output_export():
+ INPUT["out"].export()
+ assert INPUT["out"].output_image_key in INPUT["out"].exports_dic
+
+
+def test_to_numpy():
+ array = INPUT.to_numpy()
+ assert array.dtype == np.uint8
+ assert array.shape == INPUT.shape
+ assert array.min() == 33
+ assert array.max() == 255
+
+
+def test_to_numpy_sliced():
+ sliced = INPUT[:100, :200, :3]
+ array = sliced.to_numpy()
+ assert array.dtype == np.uint8
+ assert array.shape == (100, 200, 3)
+
+
+def test_convert_to_array():
+ array = np.array(INPUT)
+ assert isinstance(array, np.ndarray)
+ assert INPUT.shape == array.shape
+
+
+def test_pixel_coords_otb_equals_numpy():
+ assert INPUT[19, 7] == list(INPUT.to_numpy()[19, 7])
+
+
+def test_add_noise_array():
+ white_noise = np.random.normal(0, 50, size=INPUT.shape)
+ noisy_image = INPUT + white_noise
+ assert isinstance(noisy_image, pyotb.core.App)
+ assert noisy_image.shape == INPUT.shape
+
+
+def test_to_rasterio():
+ array, profile = INPUT.to_rasterio()
+ assert array.dtype == profile["dtype"] == np.uint8
+ assert array.shape == (4, 304, 251)
+ assert profile["transform"] == (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0)
+
+ # CRS test requires GDAL python bindings
+ try:
+ from osgeo import osr
+
+ crs = osr.SpatialReference()
+ crs.ImportFromEPSG(2154)
+ dest_crs = osr.SpatialReference()
+ dest_crs.ImportFromWkt(profile["crs"])
+ assert dest_crs.IsSame(crs)
+ except ImportError:
+ pass
diff --git a/tests/test_numpy.py b/tests/test_numpy.py
deleted file mode 100644
index d9c3d7b..0000000
--- a/tests/test_numpy.py
+++ /dev/null
@@ -1,67 +0,0 @@
-import numpy as np
-import pyotb
-from tests_data import INPUT
-
-
-def test_export():
- INPUT.export()
- array = INPUT.exports_dic[INPUT.output_image_key]["array"]
- assert isinstance(array, np.ndarray)
- assert array.dtype == "uint8"
- del INPUT.exports_dic["out"]
-
-
-def test_output_export():
- INPUT["out"].export()
- assert INPUT["out"].output_image_key in INPUT["out"].exports_dic
-
-
-def test_to_numpy():
- array = INPUT.to_numpy()
- assert array.dtype == np.uint8
- assert array.shape == INPUT.shape
- assert array.min() == 33
- assert array.max() == 255
-
-
-def test_to_numpy_sliced():
- sliced = INPUT[:100, :200, :3]
- array = sliced.to_numpy()
- assert array.dtype == np.uint8
- assert array.shape == (100, 200, 3)
-
-
-def test_convert_to_array():
- array = np.array(INPUT)
- assert isinstance(array, np.ndarray)
- assert INPUT.shape == array.shape
-
-
-def test_pixel_coords_otb_equals_numpy():
- assert INPUT[19, 7] == list(INPUT.to_numpy()[19, 7])
-
-
-def test_add_noise_array():
- white_noise = np.random.normal(0, 50, size=INPUT.shape)
- noisy_image = INPUT + white_noise
- assert isinstance(noisy_image, pyotb.core.App)
- assert noisy_image.shape == INPUT.shape
-
-
-def test_to_rasterio():
- array, profile = INPUT.to_rasterio()
- assert array.dtype == profile["dtype"] == np.uint8
- assert array.shape == (4, 304, 251)
- assert profile["transform"] == (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0)
-
- # CRS test requires GDAL python bindings
- try:
- from osgeo import osr
-
- crs = osr.SpatialReference()
- crs.ImportFromEPSG(2154)
- dest_crs = osr.SpatialReference()
- dest_crs.ImportFromWkt(profile["crs"])
- assert dest_crs.IsSame(crs)
- except ImportError:
- pass
--
GitLab
From ddd61c281351b0f3df83be50d0a972e45a5c2525 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 12 Jul 2023 13:26:48 +0000
Subject: [PATCH 279/399] ADD: tests with OpticalCalibration
---
.gitlab-ci.yml | 32 +++++--
pyotb/core.py | 4 +-
pyproject.toml | 25 +++--
...alized_apps.json => pipeline_summary.json} | 92 +++++++++----------
tests/test_core.py | 56 ++++++-----
tests/test_pipeline.py | 4 +-
tests/tests_data.py | 12 +--
7 files changed, 127 insertions(+), 98 deletions(-)
rename tests/{serialized_apps.json => pipeline_summary.json} (58%)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c57f662..18cca61 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -74,26 +74,38 @@ codespell:
LD_LIBRARY_PATH: /opt/otb/lib
OTB_LOGGER_LEVEL: INFO
PYOTB_LOGGER_LEVEL: DEBUG
- artifacts:
- reports:
- junit: test-*.xml
- coverage_report:
- coverage_format: cobertura
- path: coverage.xml
+ SPOT_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif
+ PLEIADES_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Baseline/OTB/Images/prTvOrthoRectification_pleiades-1_noDEM.tif
before_script:
- pip install pytest pytest-cov
+
+test_install:
+ extends: .tests
+ script:
- pip install .
-test_core:
+test_module_core:
extends: .tests
+ artifacts:
+ reports:
+ junit: test-module-core.xml
+ coverage_report:
+ coverage_format: cobertura
+ path: coverage.xml
coverage: '/TOTAL.*\s+(\d+%)$/'
script:
- - pytest -vv --color=yes --junitxml=test-core.xml --cov=/usr/local/lib/python3.8/dist-packages/pyotb --cov-report term --cov-report xml:coverage.xml tests/test_core.py
+ - curl -fsLI $SPOT_IMG_URL
+ - curl -fsLI $PLEIADES_IMG_URL
+ - python3 -m pytest -vv --junitxml=test-module-core.xml --cov-report xml:coverage.xml tests/test_core.py
-test_pipeline:
+test_pipeline_permutations:
extends: .tests
+ artifacts:
+ reports:
+ junit: test-pipeline-permutations.xml
script:
- - pytest -vv --color=yes --junitxml=test-pipeline.xml tests/test_pipeline.py
+ - curl -fsLI $SPOT_IMG_URL
+ - python3 -m pytest -vv --junitxml=test-pipeline-permutations.xml tests/test_pipeline.py
# -------------------------------------- Docs ---------------------------------------
diff --git a/pyotb/core.py b/pyotb/core.py
index 18ad041..71a032c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -432,9 +432,7 @@ class OTBObject(ABC):
if item in dir(self.app):
# Because otbApplication instances methods names start with an
# upper case
- hint = (
- f"Maybe try `pyotb_app.app.{item}` instead of `pyotb_app.{item}`? "
- )
+ hint = f"Maybe try `pyotb_app.app.{item}` instead of `pyotb_app.{item}`? "
if item.startswith("GetParameter"):
hint += (
"Note: `pyotb_app.app.GetParameterValue('paramname')` can be "
diff --git a/pyproject.toml b/pyproject.toml
index 429ed73..30804d4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,15 +6,15 @@ build-backend = "setuptools.build_meta"
name = "pyotb"
description = "Library to enable easy use of the Orfeo ToolBox (OTB) in Python"
authors = [
- {name="Rémi Cresson", email="remi.cresson@inrae.fr"},
- {name="Nicolas Narçon"},
- {name="Vincent Delbar"},
+ { name = "Rémi Cresson", email = "remi.cresson@inrae.fr" },
+ { name = "Nicolas Narçon" },
+ { name = "Vincent Delbar" },
]
requires-python = ">=3.7"
keywords = ["gis", "remote sensing", "otb", "orfeotoolbox", "orfeo toolbox"]
dependencies = ["numpy>=1.16"]
readme = "README.md"
-license = {text="Apache-2.0"}
+license = { text = "Apache-2.0" }
dynamic = ["version"]
classifiers = [
"Programming Language :: Python :: 3",
@@ -30,7 +30,15 @@ classifiers = [
]
[project.optional-dependencies]
-dev = ["pytest", "pytest-cov", "pylint", "codespell", "pydocstyle", "tomli", "requests"]
+dev = [
+ "pytest",
+ "pytest-cov",
+ "pylint",
+ "codespell",
+ "pydocstyle",
+ "tomli",
+ "requests",
+]
[project.urls]
documentation = "https://pyotb.readthedocs.io"
@@ -41,7 +49,7 @@ repository = "https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb"
packages = ["pyotb"]
[tool.setuptools.dynamic]
-version = {attr="pyotb.__version__"}
+version = { attr = "pyotb.__version__" }
[tool.pylint]
max-line-length = 88
@@ -61,3 +69,8 @@ convention = "google"
[tool.black]
line-length = 88
+
+[tool.pytest.ini_options]
+minversion = "7.0"
+addopts = "--color=yes --cov=pyotb --no-cov-on-fail --cov-report term"
+testpaths = ["tests"]
diff --git a/tests/serialized_apps.json b/tests/pipeline_summary.json
similarity index 58%
rename from tests/serialized_apps.json
rename to tests/pipeline_summary.json
index 938958c..8c598ae 100644
--- a/tests/serialized_apps.json
+++ b/tests/pipeline_summary.json
@@ -2,39 +2,39 @@
"SIMPLE": {
"name": "ManageNoData",
"parameters": {
+ "usenan": false,
"mode": "buildmask",
"mode.buildmask.inv": 1.0,
"mode.buildmask.outv": 0.0,
- "usenan": false,
"in": {
- "name": "OrthoRectification",
+ "name": "BandMath",
"parameters": {
- "map": "utm",
- "map.utm.zone": 31,
- "map.utm.northhem": true,
- "outputs.ulx": 560000.8125,
- "outputs.uly": 5495732.5,
- "outputs.sizex": 251,
- "outputs.sizey": 304,
- "outputs.spacingx": 5.997312068939209,
- "outputs.spacingy": -5.997312068939209,
- "outputs.lrx": 561506.125,
- "outputs.lry": 5493909.5,
- "outputs.isotropic": true,
- "opt.gridspacing": 4.0,
- "opt.rpc": 10,
- "outputs.mode": "auto",
- "interpolator": "bco",
- "interpolator.bco.radius": 2,
- "io.in": {
- "name": "BandMath",
- "parameters": {
- "il": [
- "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
- ],
- "exp": "im1b1"
+ "il": [
+ {
+ "name": "OrthoRectification",
+ "parameters": {
+ "map": "utm",
+ "map.utm.zone": 31,
+ "map.utm.northhem": true,
+ "outputs.mode": "auto",
+ "outputs.ulx": 560000.8382510637,
+ "outputs.uly": 5495732.692591702,
+ "outputs.sizex": 251,
+ "outputs.sizey": 304,
+ "outputs.spacingx": 5.997312290795521,
+ "outputs.spacingy": -5.997312290795521,
+ "outputs.lrx": 561506.1636360534,
+ "outputs.lry": 5493909.5096553,
+ "outputs.isotropic": true,
+ "interpolator": "bco",
+ "interpolator.bco.radius": 2,
+ "opt.rpc": 10,
+ "opt.gridspacing": 4.0,
+ "io.in": "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
+ }
}
- }
+ ],
+ "exp": "im1b1"
}
}
}
@@ -49,20 +49,20 @@
"map": "utm",
"map.utm.zone": 31,
"map.utm.northhem": true,
- "outputs.ulx": 560000.8125,
- "outputs.uly": 5495732.5,
+ "outputs.mode": "auto",
+ "outputs.ulx": 560000.8382510637,
+ "outputs.uly": 5495732.692591702,
"outputs.sizex": 251,
"outputs.sizey": 304,
- "outputs.spacingx": 5.997312068939209,
- "outputs.spacingy": -5.997312068939209,
- "outputs.lrx": 561506.125,
- "outputs.lry": 5493909.5,
+ "outputs.spacingx": 5.997312290795521,
+ "outputs.spacingy": -5.997312290795521,
+ "outputs.lrx": 561506.1636360534,
+ "outputs.lry": 5493909.5096553,
"outputs.isotropic": true,
- "opt.gridspacing": 4.0,
- "opt.rpc": 10,
- "outputs.mode": "auto",
"interpolator": "bco",
"interpolator.bco.radius": 2,
+ "opt.rpc": 10,
+ "opt.gridspacing": 4.0,
"io.in": {
"name": "BandMath",
"parameters": {
@@ -77,30 +77,30 @@
{
"name": "ManageNoData",
"parameters": {
+ "usenan": false,
"mode": "buildmask",
"mode.buildmask.inv": 1.0,
"mode.buildmask.outv": 0.0,
- "usenan": false,
"in": {
"name": "OrthoRectification",
"parameters": {
"map": "utm",
"map.utm.zone": 31,
"map.utm.northhem": true,
- "outputs.ulx": 560000.8125,
- "outputs.uly": 5495732.5,
+ "outputs.mode": "auto",
+ "outputs.ulx": 560000.8382510637,
+ "outputs.uly": 5495732.692591702,
"outputs.sizex": 251,
"outputs.sizey": 304,
- "outputs.spacingx": 5.997312068939209,
- "outputs.spacingy": -5.997312068939209,
- "outputs.lrx": 561506.125,
- "outputs.lry": 5493909.5,
+ "outputs.spacingx": 5.997312290795521,
+ "outputs.spacingy": -5.997312290795521,
+ "outputs.lrx": 561506.1636360534,
+ "outputs.lry": 5493909.5096553,
"outputs.isotropic": true,
- "opt.gridspacing": 4.0,
- "opt.rpc": 10,
- "outputs.mode": "auto",
"interpolator": "bco",
"interpolator.bco.radius": 2,
+ "opt.rpc": 10,
+ "opt.gridspacing": 4.0,
"io.in": {
"name": "BandMath",
"parameters": {
diff --git a/tests/test_core.py b/tests/test_core.py
index bbc35ae..8d54003 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -4,11 +4,23 @@ import numpy as np
from tests_data import *
-# Input settings
def test_parameters():
+ # Input / ExtractROI
assert INPUT.parameters
- assert INPUT.parameters["in"] == FILEPATH
+ assert INPUT.parameters["in"] == SPOT_IMG_URL
assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304)
+ # ManageNoData
+ app = pyotb.ManageNoData(INPUT)
+ assert "usenan" in app._auto_parameters
+ assert "mode.buildmask.inv" in app._auto_parameters
+ # OpticalCalibration
+ inp = pyotb.Input(PLEIADES_IMG_URL)
+ app = pyotb.OpticalCalibration(inp, level="toa")
+ assert "milli" in app._auto_parameters
+ assert "clamp" in app._auto_parameters
+ assert app._auto_parameters["acqui.year"] == 2012
+ assert app._auto_parameters["acqui.sun.elev"] == 23.836299896240234
+ # OrthoRectification
app = pyotb.OrthoRectification(INPUT)
assert isinstance(app.parameters["map"], str)
assert app.parameters["map"] == "utm"
@@ -50,7 +62,7 @@ def test_input_vsi():
def test_input_vsi_from_user():
# Ensure old way is still working: ExtractROI will raise RuntimeError if a path is malformed
- pyotb.Input("/vsicurl/" + FILEPATH)
+ pyotb.Input("/vsicurl/" + SPOT_IMG_URL)
def test_wrong_key():
@@ -170,7 +182,7 @@ def test_ext_fname():
INPUT["out"].filepath.unlink()
- mss = pyotb.MeanShiftSmoothing(FILEPATH)
+ mss = pyotb.MeanShiftSmoothing(SPOT_IMG_URL)
mss.write(
{
"fout": "/tmp/test_ext_fn_fout.tif?&nodata=1",
@@ -237,7 +249,7 @@ def test_slicer_in_output():
def test_rational_operators():
def _test(func, exp):
meas = func(INPUT)
- ref = pyotb.BandMathX({"il": [FILEPATH], "exp": exp})
+ ref = pyotb.BandMathX({"il": [SPOT_IMG_URL], "exp": exp})
for i in range(1, 5):
compared = pyotb.CompareImages(
{"ref.in": ref, "meas.in": meas, "ref.channel": i, "meas.channel": i}
@@ -248,14 +260,14 @@ def test_rational_operators():
_test(lambda x: x - x, "im1 - im1")
_test(lambda x: x / x, "im1 div im1")
_test(lambda x: x * x, "im1 mult im1")
- _test(lambda x: x + FILEPATH, "im1 + im1")
- _test(lambda x: x - FILEPATH, "im1 - im1")
- _test(lambda x: x / FILEPATH, "im1 div im1")
- _test(lambda x: x * FILEPATH, "im1 mult im1")
- _test(lambda x: FILEPATH + x, "im1 + im1")
- _test(lambda x: FILEPATH - x, "im1 - im1")
- _test(lambda x: FILEPATH / x, "im1 div im1")
- _test(lambda x: FILEPATH * x, "im1 mult im1")
+ _test(lambda x: x + SPOT_IMG_URL, "im1 + im1")
+ _test(lambda x: x - SPOT_IMG_URL, "im1 - im1")
+ _test(lambda x: x / SPOT_IMG_URL, "im1 div im1")
+ _test(lambda x: x * SPOT_IMG_URL, "im1 mult im1")
+ _test(lambda x: SPOT_IMG_URL + x, "im1 + im1")
+ _test(lambda x: SPOT_IMG_URL - x, "im1 - im1")
+ _test(lambda x: SPOT_IMG_URL / x, "im1 div im1")
+ _test(lambda x: SPOT_IMG_URL * x, "im1 mult im1")
_test(lambda x: x + 2, "im1 + {2;2;2;2}")
_test(lambda x: x - 2, "im1 - {2;2;2;2}")
_test(lambda x: x / 2, "0.5 * im1")
@@ -354,8 +366,8 @@ def test_summarize_output():
def test_summarize_strip_output():
- in_fn = FILEPATH
- in_fn_w_ext = FILEPATH + "?&skipcarto=1"
+ in_fn = SPOT_IMG_URL
+ in_fn_w_ext = SPOT_IMG_URL + "?&skipcarto=1"
out_fn = "/tmp/output.tif"
out_fn_w_ext = out_fn + "?&box=10:10:10:10"
@@ -394,7 +406,7 @@ def test_summarize_consistency():
- summary of the app after write()
Then we check that both only differ with the output parameter
"""
- app = app_fn(inp=FILEPATH)
+ app = app_fn(inp=SPOT_IMG_URL)
out_file = "/dev/shm/out.tif"
out_key = app.output_image_key
summary_wo_wrt = pyotb.summarize(app)
@@ -408,20 +420,18 @@ def test_summarize_consistency():
_test(app_fn)
-@pytest.mark.xfail
-def test_pipeline_simple():
+def test_summary_pipeline_simple():
# BandMath -> OrthoRectification -> ManageNoData
- app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"})
- app2 = pyotb.OrthoRectification({"io.in": app1})
+ app1 = pyotb.OrthoRectification({"io.in": SPOT_IMG_URL})
+ app2 = pyotb.BandMath({"il": [app1], "exp": "im1b1"})
app3 = pyotb.ManageNoData({"in": app2})
summary = pyotb.summarize(app3)
assert summary == SIMPLE_SERIALIZATION
-@pytest.mark.xfail
-def test_pipeline_diamond():
+def test_summary_pipeline_diamond():
# Diamond graph
- app1 = pyotb.BandMath({"il": [FILEPATH], "exp": "im1b1"})
+ app1 = pyotb.BandMath({"il": [SPOT_IMG_URL], "exp": "im1b1"})
app2 = pyotb.OrthoRectification({"io.in": app1})
app3 = pyotb.ManageNoData({"in": app2})
app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"})
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index eb3b51f..0b52ea2 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -2,7 +2,7 @@ import os
import itertools
import pytest
import pyotb
-from tests_data import INPUT, FILEPATH
+from tests_data import INPUT, SPOT_IMG_URL
# List of buildings blocks, we can add other pyotb objects here
@@ -79,7 +79,7 @@ def pipeline2str(pipeline):
def make_pipelines_list():
"""Create a list of pipelines using different lengths and blocks"""
- blocks = {FILEPATH: OTBAPPS_BLOCKS, INPUT: ALL_BLOCKS} # for filepath, we can't use Slicer or Operation
+ blocks = {SPOT_IMG_URL: OTBAPPS_BLOCKS, INPUT: ALL_BLOCKS} # for filepath, we can't use Slicer or Operation
pipelines = []
names = []
for inp, blocks in blocks.items():
diff --git a/tests/tests_data.py b/tests/tests_data.py
index 359081d..983b190 100644
--- a/tests/tests_data.py
+++ b/tests/tests_data.py
@@ -1,16 +1,12 @@
import json
from pathlib import Path
-import requests
import pyotb
-FILEPATH = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
-response = requests.get(FILEPATH, timeout=5)
-code = response.status_code
-if code != 200:
- raise requests.HTTPError(f"Unable to fetch remote image, GitLab might be offline (HTTP {code}).")
+SPOT_IMG_URL = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
+PLEIADES_IMG_URL = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Baseline/OTB/Images/prTvOrthoRectification_pleiades-1_noDEM.tif"
-INPUT = pyotb.Input(FILEPATH)
+INPUT = pyotb.Input(SPOT_IMG_URL)
TEST_IMAGE_STATS = {
'out.mean': [79.5505, 109.225, 115.456, 249.349],
'out.min': [33, 64, 91, 47],
@@ -18,7 +14,7 @@ TEST_IMAGE_STATS = {
'out.std': [51.0754, 35.3152, 23.4514, 20.3827]
}
-json_file = Path(__file__).parent / "serialized_apps.json"
+json_file = Path(__file__).parent / "pipeline_summary.json"
with json_file.open("r", encoding="utf-8") as js:
data = json.load(js)
SIMPLE_SERIALIZATION = data["SIMPLE"]
--
GitLab
From 1249e4a981f55d3e816c502f64de1bd1cabc29da Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 3 Aug 2023 20:43:02 +0000
Subject: [PATCH 280/399] CI: Refactor tests
---
.gitlab-ci.yml | 21 +-
tests/pipeline_summary.json | 2 +-
tests/test_core.py | 458 ++++++++++++++++--------------------
tests/test_pipeline.py | 86 +++++--
tests/tests_data.py | 13 +-
5 files changed, 284 insertions(+), 296 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 18cca61..2c6e521 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -61,6 +61,13 @@ codespell:
- codespell {pyotb,tests,doc,README.md}
# -------------------------------------- Tests --------------------------------------
+test_install:
+ stage: Tests
+ only:
+ - tags
+ allow_failure: false
+ script:
+ - pip install .
.tests:
stage: Tests
@@ -72,20 +79,16 @@ codespell:
variables:
OTB_ROOT: /opt/otb
LD_LIBRARY_PATH: /opt/otb/lib
- OTB_LOGGER_LEVEL: INFO
- PYOTB_LOGGER_LEVEL: DEBUG
SPOT_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif
PLEIADES_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Baseline/OTB/Images/prTvOrthoRectification_pleiades-1_noDEM.tif
before_script:
- pip install pytest pytest-cov
-test_install:
- extends: .tests
- script:
- - pip install .
-
test_module_core:
extends: .tests
+ variables:
+ OTB_LOGGER_LEVEL: INFO
+ PYOTB_LOGGER_LEVEL: DEBUG
artifacts:
reports:
junit: test-module-core.xml
@@ -100,6 +103,9 @@ test_module_core:
test_pipeline_permutations:
extends: .tests
+ variables:
+ OTB_LOGGER_LEVEL: WARNING
+ PYOTB_LOGGER_LEVEL: INFO
artifacts:
reports:
junit: test-pipeline-permutations.xml
@@ -111,7 +117,6 @@ test_pipeline_permutations:
docs:
stage: Documentation
- # when: manual
rules:
- changes:
- mkdocs.yml
diff --git a/tests/pipeline_summary.json b/tests/pipeline_summary.json
index 8c598ae..56ebf8d 100644
--- a/tests/pipeline_summary.json
+++ b/tests/pipeline_summary.json
@@ -39,7 +39,7 @@
}
}
},
- "COMPLEX": {
+ "DIAMOND": {
"name": "BandMathX",
"parameters": {
"il": [
diff --git a/tests/test_core.py b/tests/test_core.py
index 8d54003..82173cd 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -4,22 +4,11 @@ import numpy as np
from tests_data import *
-def test_parameters():
+def test_app_parameters():
# Input / ExtractROI
assert INPUT.parameters
assert INPUT.parameters["in"] == SPOT_IMG_URL
assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304)
- # ManageNoData
- app = pyotb.ManageNoData(INPUT)
- assert "usenan" in app._auto_parameters
- assert "mode.buildmask.inv" in app._auto_parameters
- # OpticalCalibration
- inp = pyotb.Input(PLEIADES_IMG_URL)
- app = pyotb.OpticalCalibration(inp, level="toa")
- assert "milli" in app._auto_parameters
- assert "clamp" in app._auto_parameters
- assert app._auto_parameters["acqui.year"] == 2012
- assert app._auto_parameters["acqui.sun.elev"] == 23.836299896240234
# OrthoRectification
app = pyotb.OrthoRectification(INPUT)
assert isinstance(app.parameters["map"], str)
@@ -29,14 +18,37 @@ def test_parameters():
assert app.parameters["map"] == "epsg"
assert "map" in app._settings and "map" not in app._auto_parameters
assert app.parameters["map.epsg.code"] == app.app.GetParameters()["map.epsg.code"]
-
-
-def test_param_with_underscore():
+ # Orthorectification with underscore kwargs
app = pyotb.OrthoRectification(io_in=INPUT, map_epsg_code=2154)
assert app.parameters["map.epsg.code"] == 2154
+ # ManageNoData
+ app = pyotb.ManageNoData(INPUT)
+ assert "usenan" in app._auto_parameters
+ assert "mode.buildmask.inv" in app._auto_parameters
+ # OpticalCalibration
+ app = pyotb.OpticalCalibration(pyotb.Input(PLEIADES_IMG_URL), level="toa")
+ assert "milli" in app._auto_parameters
+ assert "clamp" in app._auto_parameters
+ assert app._auto_parameters["acqui.year"] == 2012
+ assert app._auto_parameters["acqui.sun.elev"] == 23.836299896240234
+
+
+def test_app_properties():
+ assert INPUT.input_key == INPUT.input_image_key == "in"
+ assert INPUT.output_key == INPUT.output_image_key == "out"
+ with pytest.raises(KeyError):
+ pyotb.BandMath(INPUT, expression="im1b1")
+ # Test user can set custom name
+ app = pyotb.App("BandMath", [INPUT], exp="im1b1", name="TestName")
+ assert app.name == "TestName"
+ # Test data dict is not empty
+ app = pyotb.ReadImageInfo(INPUT)
+ assert app.data
+ # Test elapsed time is not null
+ assert 0 < app.elapsed_time < 1
-def test_input_vsi():
+def test_app_input_vsi():
# Simple remote file
info = pyotb.ReadImageInfo("https://fake.com/image.tif", frozen=True)
assert info.app.GetParameterValue("in") == "/vsicurl/https://fake.com/image.tif"
@@ -58,56 +70,27 @@ def test_input_vsi():
for ziped_tif_url in ziped_tif_urls:
info = pyotb.ReadImageInfo(ziped_tif_url)
assert info["sizex"] == 20
-
-
-def test_input_vsi_from_user():
# Ensure old way is still working: ExtractROI will raise RuntimeError if a path is malformed
pyotb.Input("/vsicurl/" + SPOT_IMG_URL)
-def test_wrong_key():
- with pytest.raises(KeyError):
- pyotb.BandMath(INPUT, expression="im1b1")
-
-
-# OTBObject properties
-def test_name():
- app = pyotb.App("BandMath", [INPUT], exp="im1b1", name="TestName")
- assert app.name == "TestName"
-
-
-def test_key_input():
- assert INPUT.input_key == INPUT.input_image_key == "in"
-
-
-def test_key_output():
- assert INPUT.output_image_key == "out"
-
-
-def test_dtype():
+def test_img_properties():
assert INPUT.dtype == "uint8"
-
-
-def test_shape():
assert INPUT.shape == (304, 251, 4)
-
-
-def test_transform():
assert INPUT.transform == (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0)
+ with pytest.raises(TypeError):
+ assert pyotb.ReadImageInfo(INPUT).dtype == "uint8"
-def test_data():
- assert pyotb.ComputeImagesStatistics(INPUT).data == TEST_IMAGE_STATS
-
-
-def test_metadata():
- INPUT2 = pyotb.Input(
+def test_img_metadata():
+ assert "ProjectionRef" in INPUT.metadata
+ assert "TIFFTAG_SOFTWARE" in INPUT.metadata
+ inp2 = pyotb.Input(
"https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/"
"47/Q/RU/2021/12/S2B_47QRU_20211227_0_L2A/B04.tif"
)
- assert "ProjectionRef", "TIFFTAG_SOFTWARE" in INPUT.metadata
- assert "ProjectionRef", "OVR_RESAMPLING_ALG" in INPUT2.metadata
-
+ assert "ProjectionRef" in inp2.metadata
+ assert "OVR_RESAMPLING_ALG" in inp2.metadata
# Metadata with numeric values (e.g. TileHintX)
fp = (
"https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/"
@@ -117,16 +100,24 @@ def test_metadata():
assert "TileHintX" in app.metadata
-def test_nonraster_property():
- with pytest.raises(TypeError):
- assert pyotb.ReadImageInfo(INPUT).dtype == "uint8"
+def test_essential_apps():
+ readimageinfo = pyotb.ReadImageInfo(INPUT, quiet=True)
+ assert (readimageinfo["sizex"], readimageinfo["sizey"]) == (251, 304)
+ assert readimageinfo["numberbands"] == 4
+ computeimagestats = pyotb.ComputeImagesStatistics([INPUT], quiet=True)
+ assert computeimagestats["out.min"] == TEST_IMAGE_STATS["out.min"]
+ slicer_computeimagestats = pyotb.ComputeImagesStatistics(
+ il=[INPUT[:10, :10, 0]], quiet=True
+ )
+ assert slicer_computeimagestats["out.min"] == [180]
-def test_elapsed_time():
- assert 0 < pyotb.ReadImageInfo(INPUT).elapsed_time < 1
+def test_get_statistics():
+ stats_data = pyotb.ComputeImagesStatistics(INPUT).data
+ assert stats_data == TEST_IMAGE_STATS
+ assert INPUT.get_statistics() == TEST_IMAGE_STATS
-# Other functions
def test_get_info():
infos = INPUT.get_info()
assert (infos["sizex"], infos["sizey"]) == (251, 304)
@@ -134,8 +125,9 @@ def test_get_info():
assert infos == bm_infos
-def test_get_statistics():
- assert INPUT.get_statistics() == TEST_IMAGE_STATS
+def test_read_values_at_coords():
+ assert INPUT[0, 0, 0] == 180
+ assert INPUT[10, 20, :] == [207, 192, 172, 255]
def test_xy_to_rowcol():
@@ -143,50 +135,59 @@ def test_xy_to_rowcol():
def test_write():
- assert INPUT.write("/tmp/test_write.tif", ext_fname="nodata=0")
+ assert INPUT.write("/dev/shm/test_write.tif", ext_fname="nodata=0")
INPUT["out"].filepath.unlink()
+ # Frozen
+ frozen_app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True)
+ assert frozen_app.write("/dev/shm/test_frozen_app_write.tif")
+ frozen_app["out"].filepath.unlink()
+ frozen_app_init_with_outfile = pyotb.BandMath(
+ INPUT, exp="im1b1", out="/dev/shm/test_frozen_app_write.tif", frozen=True
+ )
+ assert frozen_app_init_with_outfile.write()
+ frozen_app_init_with_outfile["out"].filepath.unlink()
-def test_ext_fname():
+def test_write_ext_fname():
def _check(expected: str, key: str = "out", app=INPUT.app):
fn = app.GetParameterString(key)
assert "?&" in fn
assert fn.split("?&", 1)[1] == expected
- assert INPUT.write("/tmp/test_write.tif", ext_fname="nodata=0")
+ assert INPUT.write("/dev/shm/test_write.tif", ext_fname="nodata=0")
_check("nodata=0")
- assert INPUT.write("/tmp/test_write.tif", ext_fname={"nodata": "0"})
+ assert INPUT.write("/dev/shm/test_write.tif", ext_fname={"nodata": "0"})
_check("nodata=0")
- assert INPUT.write("/tmp/test_write.tif", ext_fname={"nodata": 0})
+ assert INPUT.write("/dev/shm/test_write.tif", ext_fname={"nodata": 0})
_check("nodata=0")
assert INPUT.write(
- "/tmp/test_write.tif", ext_fname={"nodata": 0, "gdal:co:COMPRESS": "DEFLATE"}
+ "/dev/shm/test_write.tif",
+ ext_fname={"nodata": 0, "gdal:co:COMPRESS": "DEFLATE"},
)
_check("nodata=0&gdal:co:COMPRESS=DEFLATE")
assert INPUT.write(
- "/tmp/test_write.tif", ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE"
+ "/dev/shm/test_write.tif", ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE"
)
_check("nodata=0&gdal:co:COMPRESS=DEFLATE")
assert INPUT.write(
- "/tmp/test_write.tif?&box=0:0:10:10",
+ "/dev/shm/test_write.tif?&box=0:0:10:10",
ext_fname={"nodata": "0", "gdal:co:COMPRESS": "DEFLATE", "box": "0:0:20:20"},
)
# Check that the bbox is the one specified in the filepath, not the one
# specified in `ext_filename`
_check("nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:10:10")
assert INPUT.write(
- "/tmp/test_write.tif?&box=0:0:10:10",
+ "/dev/shm/test_write.tif?&box=0:0:10:10",
ext_fname="nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:20:20",
)
_check("nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:10:10")
-
INPUT["out"].filepath.unlink()
- mss = pyotb.MeanShiftSmoothing(SPOT_IMG_URL)
+ mss = pyotb.MeanShiftSmoothing(INPUT)
mss.write(
{
- "fout": "/tmp/test_ext_fn_fout.tif?&nodata=1",
- "foutpos": "/tmp/test_ext_fn_foutpos.tif?&nodata=2",
+ "fout": "/dev/shm/test_ext_fn_fout.tif?&nodata=1",
+ "foutpos": "/dev/shm/test_ext_fn_foutpos.tif?&nodata=2",
},
ext_fname={"nodata": 0, "gdal:co:COMPRESS": "DEFLATE"},
)
@@ -196,144 +197,89 @@ def test_ext_fname():
mss["foutpos"].filepath.unlink()
-def test_frozen_app_write():
- app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True)
- assert app.write("/tmp/test_frozen_app_write.tif")
- app["out"].filepath.unlink()
-
- app = pyotb.BandMath(
- INPUT, exp="im1b1", out="/tmp/test_frozen_app_write.tif", frozen=True
- )
- assert app.write()
- app["out"].filepath.unlink()
-
-
-def test_output_write():
- assert INPUT["out"].write("/tmp/test_output_write.tif")
+def test_output():
+ assert INPUT["out"].write("/dev/shm/test_output_write.tif")
INPUT["out"].filepath.unlink()
-
-
-def test_frozen_output_write():
- app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True)
- assert app["out"].write("/tmp/test_frozen_app_write.tif")
- app["out"].filepath.unlink()
-
-
-def test_output_in_arg():
- info = pyotb.ReadImageInfo(INPUT["out"])
- assert info.data
+ frozen_app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True)
+ assert frozen_app["out"].write("/dev/shm/test_frozen_app_write.tif")
+ frozen_app["out"].filepath.unlink()
+ info_from_output_obj = pyotb.ReadImageInfo(INPUT["out"])
+ assert info_from_output_obj.data
# Slicer
-def test_slicer_shape():
- extract = INPUT[:50, :60, :3]
- assert extract.shape == (50, 60, 3)
- assert extract.parameters["cl"] == ["Channel1", "Channel2", "Channel3"]
-
-
-def test_slicer_preserve_dtype():
- extract = INPUT[:50, :60, :3]
- assert extract.dtype == "uint8"
-
-
-def test_slicer_negative_band_index():
- assert INPUT[:50, :60, :-2].shape == (50, 60, 2)
-
-
-def test_slicer_in_output():
- slc = pyotb.BandMath([INPUT], exp="im1b1")["out"][:50, :60, :-2]
- assert isinstance(slc, pyotb.core.Slicer)
-
-
-# Arithmetic
-def test_rational_operators():
- def _test(func, exp):
- meas = func(INPUT)
- ref = pyotb.BandMathX({"il": [SPOT_IMG_URL], "exp": exp})
- for i in range(1, 5):
- compared = pyotb.CompareImages(
- {"ref.in": ref, "meas.in": meas, "ref.channel": i, "meas.channel": i}
- )
- assert (compared["count"], compared["mse"]) == (0, 0)
-
- _test(lambda x: x + x, "im1 + im1")
- _test(lambda x: x - x, "im1 - im1")
- _test(lambda x: x / x, "im1 div im1")
- _test(lambda x: x * x, "im1 mult im1")
- _test(lambda x: x + SPOT_IMG_URL, "im1 + im1")
- _test(lambda x: x - SPOT_IMG_URL, "im1 - im1")
- _test(lambda x: x / SPOT_IMG_URL, "im1 div im1")
- _test(lambda x: x * SPOT_IMG_URL, "im1 mult im1")
- _test(lambda x: SPOT_IMG_URL + x, "im1 + im1")
- _test(lambda x: SPOT_IMG_URL - x, "im1 - im1")
- _test(lambda x: SPOT_IMG_URL / x, "im1 div im1")
- _test(lambda x: SPOT_IMG_URL * x, "im1 mult im1")
- _test(lambda x: x + 2, "im1 + {2;2;2;2}")
- _test(lambda x: x - 2, "im1 - {2;2;2;2}")
- _test(lambda x: x / 2, "0.5 * im1")
- _test(lambda x: x * 2, "im1 * 2")
- _test(lambda x: x + 2.0, "im1 + {2.0;2.0;2.0;2.0}")
- _test(lambda x: x - 2.0, "im1 - {2.0;2.0;2.0;2.0}")
- _test(lambda x: x / 2.0, "0.5 * im1")
- _test(lambda x: x * 2.0, "im1 * 2.0")
- _test(lambda x: 2 + x, "{2;2;2;2} + im1")
- _test(lambda x: 2 - x, "{2;2;2;2} - im1")
- _test(lambda x: 2 / x, "{2;2;2;2} div im1")
- _test(lambda x: 2 * x, "2 * im1")
- _test(lambda x: 2.0 + x, "{2.0;2.0;2.0;2.0} + im1")
- _test(lambda x: 2.0 - x, "{2.0;2.0;2.0;2.0} - im1")
- _test(lambda x: 2.0 / x, "{2.0;2.0;2.0;2.0} div im1")
- _test(lambda x: 2.0 * x, "2.0 * im1")
-
-
-def test_operation():
+def test_slicer():
+ sliced = INPUT[:50, :60, :3]
+ assert sliced.parameters["cl"] == ["Channel1", "Channel2", "Channel3"]
+ assert sliced.shape == (50, 60, 3)
+ assert sliced.dtype == "uint8"
+ sliced_negative_band_idx = INPUT[:50, :60, :-2]
+ assert sliced_negative_band_idx.shape == (50, 60, 2)
+ sliced_from_output = pyotb.BandMath([INPUT], exp="im1b1")["out"][:50, :60, :-2]
+ assert isinstance(sliced_from_output, pyotb.core.Slicer)
+
+
+# Operation and LogicalOperation
+def test_operator_expressions():
op = INPUT / 255 * 128
assert (
op.exp
== "((im1b1 / 255) * 128);((im1b2 / 255) * 128);((im1b3 / 255) * 128);((im1b4 / 255) * 128)"
)
assert op.dtype == "float32"
-
-
-def test_func_abs_expression():
assert abs(INPUT).exp == "(abs(im1b1));(abs(im1b2));(abs(im1b3));(abs(im1b4))"
-
-
-def test_sum_bands():
- summed = sum(INPUT[:, :, b] for b in range(INPUT.shape[-1]))
- assert summed.exp == "((((0 + im1b1) + im1b2) + im1b3) + im1b4)"
-
-
-def test_binary_mask_where():
- # Create binary mask based on several possible values
- values = [1, 2, 3, 4]
- res = pyotb.where(pyotb.any(INPUT[:, :, 0] == value for value in values), 255, 0)
- assert (
- res.exp
- == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)"
- )
-
-
-# Essential apps
-def test_app_readimageinfo():
- info = pyotb.ReadImageInfo(INPUT, quiet=True)
- assert (info["sizex"], info["sizey"]) == (251, 304)
- assert info["numberbands"] == 4
-
-
-def test_app_computeimagestats():
- stats = pyotb.ComputeImagesStatistics([INPUT], quiet=True)
- assert stats["out.min"] == TEST_IMAGE_STATS["out.min"]
-
-
-def test_app_computeimagestats_sliced():
- slicer_stats = pyotb.ComputeImagesStatistics(il=[INPUT[:10, :10, 0]], quiet=True)
- assert slicer_stats["out.min"] == [180]
-
-
-def test_read_values_at_coords():
- assert INPUT[0, 0, 0] == 180
- assert INPUT[10, 20, :] == [207, 192, 172, 255]
+ summed_bands = sum(INPUT[:, :, b] for b in range(INPUT.shape[-1]))
+ assert summed_bands.exp == "((((0 + im1b1) + im1b2) + im1b3) + im1b4)"
+
+
+def operation_test(func, exp):
+ meas = func(INPUT)
+ ref = pyotb.BandMathX({"il": [SPOT_IMG_URL], "exp": exp})
+ for i in range(1, 5):
+ compared = pyotb.CompareImages(
+ {"ref.in": ref, "meas.in": meas, "ref.channel": i, "meas.channel": i}
+ )
+ assert (compared["count"], compared["mse"]) == (0, 0)
+
+
+def test_operation_add():
+ operation_test(lambda x: x + x, "im1 + im1")
+ operation_test(lambda x: x + INPUT, "im1 + im1")
+ operation_test(lambda x: INPUT + x, "im1 + im1")
+ operation_test(lambda x: x + 2, "im1 + {2;2;2;2}")
+ operation_test(lambda x: x + 2.0, "im1 + {2.0;2.0;2.0;2.0}")
+ operation_test(lambda x: 2 + x, "{2;2;2;2} + im1")
+ operation_test(lambda x: 2.0 + x, "{2.0;2.0;2.0;2.0} + im1")
+
+
+def test_operation_sub():
+ operation_test(lambda x: x - x, "im1 - im1")
+ operation_test(lambda x: x - INPUT, "im1 - im1")
+ operation_test(lambda x: INPUT - x, "im1 - im1")
+ operation_test(lambda x: x - 2, "im1 - {2;2;2;2}")
+ operation_test(lambda x: x - 2.0, "im1 - {2.0;2.0;2.0;2.0}")
+ operation_test(lambda x: 2 - x, "{2;2;2;2} - im1")
+ operation_test(lambda x: 2.0 - x, "{2.0;2.0;2.0;2.0} - im1")
+
+
+def test_operation_mult():
+ operation_test(lambda x: x * x, "im1 mult im1")
+ operation_test(lambda x: x * INPUT, "im1 mult im1")
+ operation_test(lambda x: INPUT * x, "im1 mult im1")
+ operation_test(lambda x: x * 2, "im1 * 2")
+ operation_test(lambda x: x * 2.0, "im1 * 2.0")
+ operation_test(lambda x: 2 * x, "2 * im1")
+ operation_test(lambda x: 2.0 * x, "2.0 * im1")
+
+
+def test_operation_div():
+ operation_test(lambda x: x / x, "im1 div im1")
+ operation_test(lambda x: x / INPUT, "im1 div im1")
+ operation_test(lambda x: INPUT / x, "im1 div im1")
+ operation_test(lambda x: x / 2, "im1 * 0.5")
+ operation_test(lambda x: x / 2.0, "im1 * 0.5")
+ operation_test(lambda x: 2 / x, "{2;2;2;2} div im1")
+ operation_test(lambda x: 2.0 / x, "{2.0;2.0;2.0;2.0} div im1")
# BandMath NDVI == RadiometricIndices NDVI ?
@@ -345,11 +291,11 @@ def test_ndvi_comparison():
INPUT, {"list": ["Vegetation:NDVI"], "channels.red": 1, "channels.nir": 4}
)
assert ndvi_bandmath.exp == "((im1b4 - im1b1) / (im1b4 + im1b1))"
- assert ndvi_bandmath.write("/tmp/ndvi_bandmath.tif", "float")
- assert ndvi_indices.write("/tmp/ndvi_indices.tif", "float")
+ assert ndvi_bandmath.write("/dev/shm/ndvi_bandmath.tif", "float")
+ assert ndvi_indices.write("/dev/shm/ndvi_indices.tif", "float")
compared = pyotb.CompareImages(
- {"ref.in": ndvi_indices, "meas.in": "/tmp/ndvi_bandmath.tif"}
+ {"ref.in": ndvi_indices, "meas.in": "/dev/shm/ndvi_bandmath.tif"}
)
assert (compared["count"], compared["mse"]) == (0, 0)
thresholded_indices = pyotb.where(ndvi_indices >= 0.3, 1, 0)
@@ -361,14 +307,43 @@ def test_ndvi_comparison():
)
-def test_summarize_output():
+# Tests for functions.py
+def test_binary_mask_where():
+ # Create binary mask based on several possible values
+ values = [1, 2, 3, 4]
+ res = pyotb.where(pyotb.any(INPUT[:, :, 0] == value for value in values), 255, 0)
+ assert (
+ res.exp
+ == "(((((im1b1 == 1) || (im1b1 == 2)) || (im1b1 == 3)) || (im1b1 == 4)) ? 255 : 0)"
+ )
+
+
+# Tests for summarize()
+def test_summarize_pipeline_simple():
+ app1 = pyotb.OrthoRectification({"io.in": SPOT_IMG_URL})
+ app2 = pyotb.BandMath({"il": [app1], "exp": "im1b1"})
+ app3 = pyotb.ManageNoData({"in": app2})
+ summary = pyotb.summarize(app3)
+ assert SIMPLE_SERIALIZATION == summary
+
+
+def test_summarize_pipeline_diamond():
+ app1 = pyotb.BandMath({"il": [SPOT_IMG_URL], "exp": "im1b1"})
+ app2 = pyotb.OrthoRectification({"io.in": app1})
+ app3 = pyotb.ManageNoData({"in": app2})
+ app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"})
+ summary = pyotb.summarize(app4)
+ assert DIAMOND_SERIALIZATION == summary
+
+
+def test_summarize_output_obj():
assert pyotb.summarize(INPUT["out"])
def test_summarize_strip_output():
in_fn = SPOT_IMG_URL
in_fn_w_ext = SPOT_IMG_URL + "?&skipcarto=1"
- out_fn = "/tmp/output.tif"
+ out_fn = "/dev/shm/output.tif"
out_fn_w_ext = out_fn + "?&box=10:10:10:10"
baseline = [
@@ -399,7 +374,7 @@ def test_summarize_consistency():
lambda inp: pyotb.OrthoRectification({"io.in": inp}),
]
- def _test(app_fn):
+ def operator_test(app_fn):
"""
Here we create 2 summaries:
- summary of the app before write()
@@ -417,75 +392,42 @@ def test_summarize_consistency():
assert summary_wo_wrt == summay_w_wrt
for app_fn in app_fns:
- _test(app_fn)
-
-
-def test_summary_pipeline_simple():
- # BandMath -> OrthoRectification -> ManageNoData
- app1 = pyotb.OrthoRectification({"io.in": SPOT_IMG_URL})
- app2 = pyotb.BandMath({"il": [app1], "exp": "im1b1"})
- app3 = pyotb.ManageNoData({"in": app2})
- summary = pyotb.summarize(app3)
- assert summary == SIMPLE_SERIALIZATION
+ operator_test(app_fn)
-def test_summary_pipeline_diamond():
- # Diamond graph
- app1 = pyotb.BandMath({"il": [SPOT_IMG_URL], "exp": "im1b1"})
- app2 = pyotb.OrthoRectification({"io.in": app1})
- app3 = pyotb.ManageNoData({"in": app2})
- app4 = pyotb.BandMathX({"il": [app2, app3], "exp": "im1+im2"})
- summary = pyotb.summarize(app4)
- assert summary == COMPLEX_SERIALIZATION
-
-
-# Numpy funcs
-def test_export():
+# Numpy tests
+def test_numpy_exports_dic():
INPUT.export()
- array = INPUT.exports_dic[INPUT.output_image_key]["array"]
- assert isinstance(array, np.ndarray)
- assert array.dtype == "uint8"
+ exported_array = INPUT.exports_dic[INPUT.output_image_key]["array"]
+ assert isinstance(exported_array, np.ndarray)
+ assert exported_array.dtype == "uint8"
del INPUT.exports_dic["out"]
-
-
-def test_output_export():
INPUT["out"].export()
assert INPUT["out"].output_image_key in INPUT["out"].exports_dic
-def test_to_numpy():
+def test_numpy_conversions():
array = INPUT.to_numpy()
assert array.dtype == np.uint8
assert array.shape == INPUT.shape
- assert array.min() == 33
- assert array.max() == 255
-
-
-def test_to_numpy_sliced():
+ assert (array.min(), array.max()) == (33, 255)
+ # Sliced img to array
sliced = INPUT[:100, :200, :3]
- array = sliced.to_numpy()
- assert array.dtype == np.uint8
- assert array.shape == (100, 200, 3)
-
-
-def test_convert_to_array():
- array = np.array(INPUT)
- assert isinstance(array, np.ndarray)
- assert INPUT.shape == array.shape
-
-
-def test_pixel_coords_otb_equals_numpy():
+ sliced_array = sliced.to_numpy()
+ assert sliced_array.dtype == np.uint8
+ assert sliced_array.shape == (100, 200, 3)
+ # Test auto convert to numpy
+ assert isinstance(np.array(INPUT), np.ndarray)
+ assert INPUT.shape == np.array(INPUT).shape
assert INPUT[19, 7] == list(INPUT.to_numpy()[19, 7])
-
-
-def test_add_noise_array():
+ # Add noise test from the docs
white_noise = np.random.normal(0, 50, size=INPUT.shape)
noisy_image = INPUT + white_noise
assert isinstance(noisy_image, pyotb.core.App)
assert noisy_image.shape == INPUT.shape
-def test_to_rasterio():
+def test_numpy_to_rasterio():
array, profile = INPUT.to_rasterio()
assert array.dtype == profile["dtype"] == np.uint8
assert array.shape == (4, 304, 251)
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index 0b52ea2..5646674 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -74,12 +74,18 @@ def pipeline2str(pipeline):
a string
"""
- return " > ".join([INPUT.__class__.__name__] + [f"{i}.{app.name.split()[0]}" for i, app in enumerate(pipeline)])
+ return " > ".join(
+ [INPUT.__class__.__name__]
+ + [f"{i}.{app.name.split()[0]}" for i, app in enumerate(pipeline)]
+ )
def make_pipelines_list():
"""Create a list of pipelines using different lengths and blocks"""
- blocks = {SPOT_IMG_URL: OTBAPPS_BLOCKS, INPUT: ALL_BLOCKS} # for filepath, we can't use Slicer or Operation
+ blocks = {
+ SPOT_IMG_URL: OTBAPPS_BLOCKS,
+ INPUT: ALL_BLOCKS,
+ } # for filepath, we can't use Slicer or Operation
pipelines = []
names = []
for inp, blocks in blocks.items():
@@ -96,49 +102,83 @@ def make_pipelines_list():
return pipelines, names
-PIPELINES, NAMES = make_pipelines_list()
-
-
-@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES)
-def test_pipeline_shape(pipe):
+def shape(pipe):
for app in pipe:
- assert bool(app.shape)
+ yield bool(app.shape)
-@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES)
-def test_pipeline_shape_nointermediate(pipe):
+def shape_nointermediate(pipe):
app = [pipe[-1]][0]
- assert bool(app.shape)
+ yield bool(app.shape)
-@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES)
-def test_pipeline_shape_backward(pipe):
+def shape_backward(pipe):
for app in reversed(pipe):
- assert bool(app.shape)
+ yield bool(app.shape)
-@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES)
-def test_pipeline_write(pipe):
+def write(pipe):
for i, app in enumerate(pipe):
out = f"/tmp/out_{i}.tif"
if os.path.isfile(out):
os.remove(out)
- assert app.write(out)
+ yield app.write(out)
-@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES)
-def test_pipeline_write_nointermediate(pipe):
+def write_nointermediate(pipe):
app = [pipe[-1]][0]
out = "/tmp/out_0.tif"
if os.path.isfile(out):
os.remove(out)
- assert app.write(out)
+ yield app.write(out)
-@pytest.mark.parametrize("pipe", PIPELINES, ids=NAMES)
-def test_pipeline_write_backward(pipe):
+def write_backward(pipe):
for i, app in enumerate(reversed(pipe)):
out = f"/tmp/out_{i}.tif"
if os.path.isfile(out):
os.remove(out)
- assert app.write(out)
+ yield app.write(out)
+
+
+funcs = [
+ shape,
+ shape_nointermediate,
+ shape_backward,
+ write,
+ write_nointermediate,
+ write_backward,
+]
+
+PIPELINES, NAMES = make_pipelines_list()
+
+
+@pytest.mark.parametrize("test_func", funcs)
+def test(test_func):
+ fname = test_func.__name__
+ successes, failures = 0, 0
+ total_successes = []
+ for pipeline, blocks in zip(PIPELINES, NAMES):
+ err = None
+ try:
+ # Here we count non-empty shapes or write results, no errors during exec
+ bool_tests = list(test_func(pipeline))
+ overall_success = all(bool_tests)
+ except Exception as e:
+ # Unexpected exception in the pipeline, e.g. a RuntimeError we missed
+ bool_tests = []
+ overall_success = False
+ err = e
+ if overall_success:
+ print(f"\033[92m{fname}: success with [{blocks}]\033[0m\n")
+ successes += 1
+ else:
+ print(f"\033[91m{fname}: failure with [{blocks}]\033[0m {bool_tests}\n")
+ if err:
+ print(f"exception thrown: {err}")
+ failures += 1
+ total_successes.append(overall_success)
+ print(f"\nEnded test {fname} with {successes} successes, {failures} failures")
+ if err:
+ raise err
+ assert all(total_successes)
diff --git a/tests/tests_data.py b/tests/tests_data.py
index 983b190..2e40c95 100644
--- a/tests/tests_data.py
+++ b/tests/tests_data.py
@@ -5,17 +5,18 @@ import pyotb
SPOT_IMG_URL = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
PLEIADES_IMG_URL = "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Baseline/OTB/Images/prTvOrthoRectification_pleiades-1_noDEM.tif"
-
INPUT = pyotb.Input(SPOT_IMG_URL)
+
+
TEST_IMAGE_STATS = {
- 'out.mean': [79.5505, 109.225, 115.456, 249.349],
- 'out.min': [33, 64, 91, 47],
- 'out.max': [255, 255, 230, 255],
- 'out.std': [51.0754, 35.3152, 23.4514, 20.3827]
+ "out.mean": [79.5505, 109.225, 115.456, 249.349],
+ "out.min": [33, 64, 91, 47],
+ "out.max": [255, 255, 230, 255],
+ "out.std": [51.0754, 35.3152, 23.4514, 20.3827],
}
json_file = Path(__file__).parent / "pipeline_summary.json"
with json_file.open("r", encoding="utf-8") as js:
data = json.load(js)
SIMPLE_SERIALIZATION = data["SIMPLE"]
-COMPLEX_SERIALIZATION = data["COMPLEX"]
+DIAMOND_SERIALIZATION = data["DIAMOND"]
--
GitLab
From 79faeb63a1ba8028517b965726d75b0bb68d7d27 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 5 Aug 2023 17:40:38 +0200
Subject: [PATCH 281/399] STYLE: summarize type hints #111
---
pyotb/core.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 71a032c..a39e0cf 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1712,10 +1712,10 @@ def get_out_images_param_keys(app: OTBObject) -> list[str]:
def summarize(
- obj: App | Output | Any,
- strip_input_paths: bool = False,
- strip_output_paths: bool = False,
-) -> dict[str, str | dict[str, Any]]:
+ obj: App | Output | str | float | list,
+ strip_inpath: bool = False,
+ strip_outpath: bool = False,
+) -> dict[str, dict | Any] | str | float | list:
"""Recursively summarize parameters of an App or Output object and its parents.
Args:
--
GitLab
From 2919258acb2062e1f81607376b5c5aa8c9fdd3b3 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 5 Aug 2023 17:41:06 +0200
Subject: [PATCH 282/399] DOC: better summarize() docstring
---
pyotb/core.py | 18 ++++++++----------
1 file changed, 8 insertions(+), 10 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index a39e0cf..acf72a1 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1718,25 +1718,23 @@ def summarize(
) -> dict[str, dict | Any] | str | float | list:
"""Recursively summarize parameters of an App or Output object and its parents.
+ At the deepest recursion level, this function just return any parameter value,
+ path stripped if needed, or app summarized in case of a pipeline.
+
Args:
- obj: input object to summarize
- strip_input_paths: strip all input paths: If enabled, paths related to
+ obj: input object / parameter value to summarize
+ strip_inpath: strip all input paths: If enabled, paths related to
inputs are truncated after the first "?" character. Can be useful
to remove URLs tokens (e.g. SAS or S3 credentials).
- strip_output_paths: strip all output paths: If enabled, paths related
+ strip_outpath: strip all output paths: If enabled, paths related
to outputs are truncated after the first "?" character. Can be
useful to remove extended filenames.
Returns:
- nested dictionary with serialized App(s) containing name and parameters of an app and its parents
+ nested dictionary containing name and parameters of an app and its parents
"""
-
- def strip_path(param: str | Any):
- if not isinstance(param, str):
- return summarize(param)
- return param.split("?")[0]
-
+ # This is the deepest recursion level
if isinstance(obj, list):
return [summarize(o) for o in obj]
if isinstance(obj, Output):
--
GitLab
From 39057f6ec3fff8d926d2ca19ed32b754658f1f79 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 5 Aug 2023 17:42:43 +0200
Subject: [PATCH 283/399] ENH: docstrings and readability of summarize(),
shorter param names
---
pyotb/core.py | 1 +
tests/test_core.py | 4 ++--
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index acf72a1..54017d1 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1742,6 +1742,7 @@ def summarize(
if not isinstance(obj, App):
return obj
+ if not isinstance(param, str):
parameters = {}
for key, param in obj.parameters.items():
if (
diff --git a/tests/test_core.py b/tests/test_core.py
index 82173cd..d03f31f 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -348,9 +348,9 @@ def test_summarize_strip_output():
baseline = [
(in_fn, out_fn_w_ext, "out", {}, out_fn_w_ext),
- (in_fn, out_fn_w_ext, "out", {"strip_output_paths": True}, out_fn),
+ (in_fn, out_fn_w_ext, "out", {"strip_outpath": True}, out_fn),
(in_fn_w_ext, out_fn, "in", {}, in_fn_w_ext),
- (in_fn_w_ext, out_fn, "in", {"strip_input_paths": True}, in_fn),
+ (in_fn_w_ext, out_fn, "in", {"strip_inpath": True}, in_fn),
]
for inp, out, key, extra_args, expected in baseline:
--
GitLab
From 3753779ccdc74b0d0c8555d022bcafa7e3a8c747 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 5 Aug 2023 18:06:22 +0200
Subject: [PATCH 284/399] FIX: incomplete commit
---
pyotb/core.py | 21 ++++++++++-----------
1 file changed, 10 insertions(+), 11 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 54017d1..ae6c13b 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1742,20 +1742,19 @@ def summarize(
if not isinstance(obj, App):
return obj
+ # Call / top level of recursion : obj is an App
+ def strip_path(param: str | Any):
+ if isinstance(param, list):
+ return [strip_path(p) for p in param]
if not isinstance(param, str):
+ return summarize(param)
+ return param.split("?")[0]
+
+ # We need to return parameters values, summarized if param is an App
parameters = {}
for key, param in obj.parameters.items():
- if (
- strip_input_paths
- and obj.is_input(key)
- or strip_output_paths
- and obj.is_output(key)
- ):
- parameters[key] = (
- [strip_path(p) for p in param]
- if isinstance(param, list)
- else strip_path(param)
- )
+ if strip_inpath and obj.is_input(key) or strip_outpath and obj.is_output(key):
+ parameters[key] = strip_path(param)
else:
parameters[key] = summarize(param)
return {"name": obj.app.GetName(), "parameters": parameters}
--
GitLab
From c67f1093ce7a1250c0107edc4c69ada78ddd1e4e Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 5 Aug 2023 18:10:15 +0200
Subject: [PATCH 285/399] STYLE: move comment
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index ae6c13b..76eef90 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1742,7 +1742,6 @@ def summarize(
if not isinstance(obj, App):
return obj
- # Call / top level of recursion : obj is an App
def strip_path(param: str | Any):
if isinstance(param, list):
return [strip_path(p) for p in param]
@@ -1750,6 +1749,7 @@ def summarize(
return summarize(param)
return param.split("?")[0]
+ # Call / top level of recursion : obj is an App
# We need to return parameters values, summarized if param is an App
parameters = {}
for key, param in obj.parameters.items():
--
GitLab
From a76171ee95584b700fc965f253677770d604daec Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 7 Aug 2023 11:19:16 +0000
Subject: [PATCH 286/399] ENH: Automatically add vsi prefix for sub path in
archived source files
---
pyotb/core.py | 81 +++++++++++++++++++++++++---------------------
tests/test_core.py | 16 +++++----
2 files changed, 55 insertions(+), 42 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 71a032c..0047dd5 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -2,6 +2,7 @@
"""This module is the core of pyotb."""
from __future__ import annotations
+import re
from abc import ABC, abstractmethod
from ast import literal_eval
from pathlib import Path
@@ -731,10 +732,7 @@ class App(OTBObject):
)
try:
if self.is_input(key):
- if self.is_key_images_list(key):
- self.__set_param(key, [add_vsi_prefix(p) for p in obj])
- else:
- self.__set_param(key, add_vsi_prefix(obj))
+ self.__set_param(key, self.__check_input_param(obj))
else:
self.__set_param(key, obj)
except (RuntimeError, TypeError, ValueError, KeyError) as e:
@@ -969,6 +967,49 @@ class App(OTBObject):
kwargs.update({self.input_key: arg})
return kwargs
+ def __check_input_param(
+ self, obj: list | OTBObject | str | Path
+ ) -> list | OTBObject | str:
+ """Check the type and value of an input param.
+
+ Args:
+ obj: input parameter value
+
+ Returns:
+ object, string with new /vsi prefix(es) if needed
+
+ """
+ if isinstance(obj, list):
+ return [self.__check_input_param(o) for o in obj]
+ # May be we could add some checks here
+ if isinstance(obj, OTBObject):
+ return obj
+ if isinstance(obj, Path):
+ obj = str(obj)
+ if isinstance(obj, str):
+ if not obj.startswith("/vsi"):
+ # Remote file. TODO: add support for S3 / GS / AZ
+ if obj.startswith(("https://", "http://", "ftp://")):
+ obj = "/vsicurl/" + obj
+ # Compressed file
+ prefixes = {
+ ".tar": "vsitar",
+ ".tar.gz": "vsitar",
+ ".tgz": "vsitar",
+ ".gz": "vsigzip",
+ ".7z": "vsi7z",
+ ".zip": "vsizip",
+ ".rar": "vsirar",
+ }
+ expr = r"(.*?)(\.7z|\.zip|\.rar|\.tar\.gz|\.tgz|\.tar|\.gz)(.*)"
+ parts = re.match(expr, obj)
+ if parts:
+ file, ext = parts.group(1), parts.group(2)
+ if not Path(file + ext).is_dir():
+ obj = f"/{prefixes[ext]}/{obj}"
+ return obj
+ raise TypeError(f"{self.name}: wrong input parameter type ({type(obj)})")
+
def __set_param(
self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]
):
@@ -1580,38 +1621,6 @@ class Output(OTBObject):
return str(self.filepath)
-def add_vsi_prefix(filepath: str | Path) -> str:
- """Append vsi prefixes to file URL or path if needed.
-
- Args:
- filepath: file path or URL
-
- Returns:
- string with new /vsi prefix(es)
-
- """
- if isinstance(filepath, Path):
- filepath = str(filepath)
- if isinstance(filepath, str) and not filepath.startswith("/vsi"):
- # Remote file. TODO: add support for S3 / GS / AZ
- if filepath.startswith(("https://", "http://", "ftp://")):
- filepath = "/vsicurl/" + filepath
- # Compressed file
- prefixes = {
- ".tar": "vsitar",
- ".tgz": "vsitar",
- ".gz": "vsigzip",
- ".7z": "vsi7z",
- ".zip": "vsizip",
- ".rar": "vsirar",
- }
- basename = filepath.split("?")[0]
- ext = Path(basename).suffix
- if ext in prefixes:
- filepath = f"/{prefixes[ext]}/{filepath}"
- return filepath
-
-
def get_nbchannels(inp: str | Path | OTBObject) -> int:
"""Get the nb of bands of input image.
diff --git a/tests/test_core.py b/tests/test_core.py
index 82173cd..3bfc5dd 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -49,17 +49,23 @@ def test_app_properties():
def test_app_input_vsi():
+ # Ensure old way is still working: ExtractROI will raise RuntimeError if a path is malformed
+ pyotb.Input("/vsicurl/" + SPOT_IMG_URL)
# Simple remote file
info = pyotb.ReadImageInfo("https://fake.com/image.tif", frozen=True)
assert info.app.GetParameterValue("in") == "/vsicurl/https://fake.com/image.tif"
assert info.parameters["in"] == "https://fake.com/image.tif"
- # Compressed remote file
- info = pyotb.ReadImageInfo("https://fake.com/image.tif.zip", frozen=True)
+ # Compressed single file archive
+ info = pyotb.ReadImageInfo("image.tif.zip", frozen=True)
+ assert info.app.GetParameterValue("in") == "/vsizip/image.tif.zip"
+ assert info.parameters["in"] == "image.tif.zip"
+ # File within compressed remote archive
+ info = pyotb.ReadImageInfo("https://fake.com/archive.tar.gz/image.tif", frozen=True)
assert (
info.app.GetParameterValue("in")
- == "/vsizip//vsicurl/https://fake.com/image.tif.zip"
+ == "/vsitar//vsicurl/https://fake.com/archive.tar.gz/image.tif"
)
- assert info.parameters["in"] == "https://fake.com/image.tif.zip"
+ assert info.parameters["in"] == "https://fake.com/archive.tar.gz/image.tif"
# Piped curl --> zip --> tiff
ziped_tif_urls = (
"https://github.com/OSGeo/gdal/raw/master"
@@ -70,8 +76,6 @@ def test_app_input_vsi():
for ziped_tif_url in ziped_tif_urls:
info = pyotb.ReadImageInfo(ziped_tif_url)
assert info["sizex"] == 20
- # Ensure old way is still working: ExtractROI will raise RuntimeError if a path is malformed
- pyotb.Input("/vsicurl/" + SPOT_IMG_URL)
def test_img_properties():
--
GitLab
From e36e4a14ec34d3f2514c4d94e8b09c6ee1582b44 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 18:11:13 +0200
Subject: [PATCH 287/399] FIX: latest OTB version should be used when scanning
user dir
---
pyotb/helpers.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 5363937..aed1bd6 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -274,7 +274,7 @@ def __find_otb_root(scan_userdir: bool = False):
prefix = path.parent.absolute()
# If possible, use OTB found in user's HOME tree (this may take some time)
if scan_userdir:
- for path in Path.home().glob("**/OTB-*/lib"):
+ for path in sorted(Path.home().glob("**/OTB-*/lib/")):
logger.info("Found %s", path.parent)
prefix = path.parent.absolute()
# Return latest found prefix (and version), see precedence in function def find_otb()
--
GitLab
From 3e9ec90763b4b38f3bf685c3dbf291e01f00ae63 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 18:11:55 +0200
Subject: [PATCH 288/399] STYLE: apply black
---
pyotb/__init__.py | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index bf55392..507a408 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -3,8 +3,16 @@
__version__ = "2.0.0.dev4"
from .helpers import logger, set_logger_level
+from .core import (
+ OTBObject,
+ App,
+ Input,
+ Output,
+ get_nbchannels,
+ get_pixel_type,
+ summarize,
+)
from .apps import *
-from .core import App, Input, Output, get_nbchannels, get_pixel_type, summarize, OTBObject
from .functions import ( # pylint: disable=redefined-builtin
all,
--
GitLab
From 4ca2fc0d26ff6fd1d0ea181cae1d3ce249064663 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 18:22:20 +0200
Subject: [PATCH 289/399] ADD: function to install OTB (in interactive mode
only)
---
pyotb/helpers.py | 118 +++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 113 insertions(+), 5 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index aed1bd6..825dd9b 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -1,13 +1,19 @@
# -*- coding: utf-8 -*-
"""This module helps to ensure we properly initialize pyotb: only in case OTB is found and apps are available."""
+import json
import logging
import os
+import re
+import subprocess
import sys
+import tempfile
+import urllib.request
from pathlib import Path
from shutil import which
# Allow user to switch between OTB directories without setting every env variable
OTB_ROOT = os.environ.get("OTB_ROOT")
+DOCS_URL = "https://www.orfeo-toolbox.org/CookBook/Installation.html"
# Logging
# User can also get logger with `logging.getLogger("pyOTB")`
@@ -90,6 +96,22 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru
# Else search system
logger.info("Failed to import OTB. Searching for it...")
prefix = __find_otb_root(scan_userdir)
+ if not prefix:
+ if hasattr(sys, "ps1"):
+ if input("OTB is missing. Do you want to install it ? (y/n): ") == "y":
+ version = input(
+ "Choose a version number to install (default is latest): "
+ )
+ path = input(
+ "Provide a path for installation "
+ "(default is <user_dir>/Applications/OTB-<version>): "
+ )
+ return find_otb(install_otb(version, path))
+ if not prefix:
+ raise SystemExit(
+ "OTB not found on disk. "
+ "To install it, open an interactive python shell and type 'import pyotb'"
+ )
# Try to import one last time before raising error
try:
set_environment(prefix)
@@ -98,9 +120,6 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru
return otb
except EnvironmentError as e:
raise SystemExit("Auto setup for OTB env failed. Exiting.") from e
- # Unknown error
- except ModuleNotFoundError as e:
- raise SystemExit("Can't run without OTB installed. Exiting.") from e
# Help user to fix this
except ImportError as e:
__suggest_fix_import(str(e), prefix)
@@ -166,6 +185,96 @@ def set_environment(prefix: str):
os.environ["PROJ_LIB"] = proj_lib
+def otb_latest_release_tag():
+ """Use gitlab API to find latest release tag name, but skip pre-releases."""
+ api_endpoint = "https://gitlab.orfeo-toolbox.org/api/v4/projects/53/repository/tags"
+ vers_regex = re.compile(r"^\d\.\d\.\d$")
+ with urllib.request.urlopen(api_endpoint) as stream:
+ data = json.loads(stream.read())
+ releases = sorted(
+ [tag["name"] for tag in data if vers_regex.match(tag["name"])],
+ )
+ return releases[-1]
+
+
+def install_otb(version: str = "latest", path: str = ""):
+ """Install pre-compiled OTB binaries in path, use latest release by default.
+
+ Args:
+ version: OTB version tag, e.g. '8.1.2'
+ path: installation directory
+
+ Returns:
+ full path of the new installation
+ """
+ major = sys.version_info.major
+ if major == 2:
+ raise SystemExit(
+ "You need to use python3 in order to import OTB python bindings."
+ )
+ minor = sys.version_info.minor
+ name_corresp = {"linux": "Linux64", "darwnin": "Darwin64", "win32": "Win64"}
+ sysname = name_corresp[sys.platform]
+ if sysname == "Win64":
+ cmd = which("cmd.exe")
+ ext = "zip"
+ if minor != 7:
+ raise SystemExit(
+ "Python version 3.7 is required to import python bindings on Windows."
+ )
+ else:
+ cmd = which("zsh") or which("bash") or which("sh")
+ ext = "run"
+
+ # Fetch archive and run installer
+ if not version or version == "latest":
+ version = otb_latest_release_tag()
+ filename = f"OTB-{version}-{sysname}.{ext}"
+ url = f"https://www.orfeo-toolbox.org/packages/archives/OTB/{filename}"
+ tmpdir = tempfile.gettempdir()
+ tmpfile = Path(tmpdir) / filename
+ print(f"Downloading {url}")
+ if not tmpfile.exists():
+ urllib.request.urlretrieve(url, tmpfile)
+ if path:
+ path = Path(path)
+ else:
+ path = Path.home() / "Applications" / tmpfile.stem
+ install_cmd = f"{cmd} {tmpfile} --target {path} --accept"
+ print(f"Executing '{install_cmd}'\n")
+ subprocess.run(f"{cmd} {tmpfile} --target {path} --accept", shell=True, check=True)
+ tmpfile.unlink()
+
+ # Add env variable to profile
+ if sysname != "Win64":
+ with open(Path.home() / ".profile", "a", encoding="utf-8") as buf:
+ buf.write(f'\n. "{path}/otbenv.profile"\n')
+ else:
+ print(
+ "In order to speed-up pyotb import, remember to call 'otbenv.bat' "
+ "before importing pyotb, or add 'OTB_ROOT=\"{path}\"' to your env variables."
+ )
+ if (
+ sysname == "Win64"
+ or (sysname == "Linux64" and minor == 8)
+ or (sysname == "Darwin64" and minor == 7)
+ ):
+ return str(path)
+ # Recompile bindings : this may fail because of OpenGL...
+ if sys.executable and which("ctest") and which("python3-config"):
+ print("\nRecompiling python bindings...")
+ ctest_cmd = (
+ ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV"
+ )
+ subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True)
+ return str(path)
+ print(
+ "\nYou'll need to install 'cmake', 'python3-dev' and 'libgl1-mesa-dev'"
+ " in order to recompile python bindings. "
+ )
+ raise SystemExit
+
+
def __find_lib(prefix: str = None, otb_module=None):
"""Try to find OTB external libraries directory.
@@ -330,9 +439,8 @@ def __suggest_fix_import(error_message: str, prefix: str):
"It seems that your env variables aren't properly set,"
" first use 'call otbenv.bat' then try to import pyotb once again"
)
- docs_link = "https://www.orfeo-toolbox.org/CookBook/Installation.html"
logger.critical(
- "You can verify installation requirements for your OS at %s", docs_link
+ "You can verify installation requirements for your OS at %s", DOCS_URL
)
--
GitLab
From 147aa1132c26bd8562317c2d09c6bdc5453ae430 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 18:24:00 +0200
Subject: [PATCH 290/399] ADD: env variable to allow auto OTB install for non
interactive mode
---
pyotb/helpers.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 825dd9b..942511c 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -445,4 +445,7 @@ def __suggest_fix_import(error_message: str, prefix: str):
# Since helpers is the first module to be inititialized, this will prevent pyotb to run if OTB is not found
-find_otb()
+if os.environ.get("OTB_AUTO_INSTALL") in ("1", "true", "ON", "YES"):
+ find_otb(install_otb())
+else:
+ find_otb()
--
GitLab
From 1cc011b718748af13dfd85942acc98737051da67 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 18:29:15 +0200
Subject: [PATCH 291/399] ENH: inform user that .profile has been modified
---
pyotb/helpers.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 942511c..49746b0 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -247,7 +247,9 @@ def install_otb(version: str = "latest", path: str = ""):
# Add env variable to profile
if sysname != "Win64":
- with open(Path.home() / ".profile", "a", encoding="utf-8") as buf:
+ profile = Path.home() / ".profile"
+ print(f"Adding new env variables to {profile}")
+ with open(profile, "a", encoding="utf-8") as buf:
buf.write(f'\n. "{path}/otbenv.profile"\n')
else:
print(
--
GitLab
From 4512c08c0213d50018959a63752e6ed1dbd7650f Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 21:12:39 +0200
Subject: [PATCH 292/399] FIX: typos and add comments
---
pyotb/helpers.py | 31 +++++++++++++------------------
1 file changed, 13 insertions(+), 18 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 49746b0..5d7c1a8 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -188,7 +188,7 @@ def set_environment(prefix: str):
def otb_latest_release_tag():
"""Use gitlab API to find latest release tag name, but skip pre-releases."""
api_endpoint = "https://gitlab.orfeo-toolbox.org/api/v4/projects/53/repository/tags"
- vers_regex = re.compile(r"^\d\.\d\.\d$")
+ vers_regex = re.compile(r"^\d\.\d\.\d$") # we ignore rc-* or alpha-*
with urllib.request.urlopen(api_endpoint) as stream:
data = json.loads(stream.read())
releases = sorted(
@@ -209,19 +209,17 @@ def install_otb(version: str = "latest", path: str = ""):
"""
major = sys.version_info.major
if major == 2:
- raise SystemExit(
- "You need to use python3 in order to import OTB python bindings."
- )
+ raise SystemExit("Python 3 is required for OTB bindings.")
minor = sys.version_info.minor
name_corresp = {"linux": "Linux64", "darwnin": "Darwin64", "win32": "Win64"}
sysname = name_corresp[sys.platform]
if sysname == "Win64":
- cmd = which("cmd.exe")
- ext = "zip"
if minor != 7:
raise SystemExit(
"Python version 3.7 is required to import python bindings on Windows."
)
+ cmd = which("cmd.exe")
+ ext = "zip"
else:
cmd = which("zsh") or which("bash") or which("sh")
ext = "run"
@@ -234,8 +232,7 @@ def install_otb(version: str = "latest", path: str = ""):
tmpdir = tempfile.gettempdir()
tmpfile = Path(tmpdir) / filename
print(f"Downloading {url}")
- if not tmpfile.exists():
- urllib.request.urlretrieve(url, tmpfile)
+ urllib.request.urlretrieve(url, tmpfile)
if path:
path = Path(path)
else:
@@ -243,7 +240,7 @@ def install_otb(version: str = "latest", path: str = ""):
install_cmd = f"{cmd} {tmpfile} --target {path} --accept"
print(f"Executing '{install_cmd}'\n")
subprocess.run(f"{cmd} {tmpfile} --target {path} --accept", shell=True, check=True)
- tmpfile.unlink()
+ tmpfile.unlink() # cleaning
# Add env variable to profile
if sysname != "Win64":
@@ -254,16 +251,14 @@ def install_otb(version: str = "latest", path: str = ""):
else:
print(
"In order to speed-up pyotb import, remember to call 'otbenv.bat' "
- "before importing pyotb, or add 'OTB_ROOT=\"{path}\"' to your env variables."
+ f"before importing pyotb, or add 'OTB_ROOT=\"{path}\"' to your env variables."
)
- if (
- sysname == "Win64"
- or (sysname == "Linux64" and minor == 8)
- or (sysname == "Darwin64" and minor == 7)
- ):
return str(path)
- # Recompile bindings : this may fail because of OpenGL...
- if sys.executable and which("ctest") and which("python3-config"):
+ # Linux requires 3.8, macOS requires 3.7
+ if (sysname == "Linux64" and minor == 8) or (sysname == "Darwin64" and minor == 7):
+ return str(path)
+ # Else recompile bindings : this may fail because of OpenGL
+ if which("ctest") and which("python3-config"):
print("\nRecompiling python bindings...")
ctest_cmd = (
". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV"
@@ -271,7 +266,7 @@ def install_otb(version: str = "latest", path: str = ""):
subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True)
return str(path)
print(
- "\nYou'll need to install 'cmake', 'python3-dev' and 'libgl1-mesa-dev'"
+ "\nYou need to install 'cmake', 'python3-dev' and 'libgl1-mesa-dev'"
" in order to recompile python bindings. "
)
raise SystemExit
--
GitLab
From e8e1672af3be1047229e041aa6704a7afd441c57 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 21:33:49 +0200
Subject: [PATCH 293/399] ENH: removed full user dir scan, only check for
Applications dir
---
pyotb/helpers.py | 42 +++++++++++++++++-------------------------
1 file changed, 17 insertions(+), 25 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 5d7c1a8..6319167 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -45,19 +45,19 @@ def set_logger_level(level: str):
logger_handler.setLevel(getattr(logging, level))
-def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = True):
+def find_otb(prefix: str = OTB_ROOT, scan: bool = True):
"""Try to load OTB bindings or scan system, help user in case of failure, set env variables.
- Path precedence : OTB_ROOT > python bindings directory
- OR search for releases installations : HOME
- OR (for Linux) : /opt/otbtf > /opt/otb > /usr/local > /usr
- OR (for MacOS) : ~/Applications
- OR (for Windows) : C:/Program Files
+ The OTB_ROOT variable allow one to override default OTB version, with auto env setting.
+ Path precedence : $OTB_ROOT > location of python bindings location
+ Then, if OTB is not found:
+ search for releases installations: $HOME/Applications
+ OR (for Linux): /opt/otbtf > /opt/otb > /usr/local > /usr
+ OR (for Windows): C:/Program Files
Args:
prefix: prefix to search OTB in (Default value = OTB_ROOT)
scan: find otb in system known locations (Default value = True)
- scan_userdir: search for OTB release in user's home directory (Default value = True)
Returns:
otbApplication module
@@ -95,7 +95,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True, scan_userdir: bool = Tru
) from e
# Else search system
logger.info("Failed to import OTB. Searching for it...")
- prefix = __find_otb_root(scan_userdir)
+ prefix = __find_otb_root()
if not prefix:
if hasattr(sys, "ps1"):
if input("OTB is missing. Do you want to install it ? (y/n): ") == "y":
@@ -341,12 +341,9 @@ def __find_apps_path(lib_dir: Path):
return ""
-def __find_otb_root(scan_userdir: bool = False):
+def __find_otb_root():
"""Search for OTB root directory in well known locations.
- Args:
- scan_userdir: search with glob in $HOME directory
-
Returns:
str path of the OTB directory
@@ -369,22 +366,17 @@ def __find_otb_root(scan_userdir: bool = False):
prefix = path.parent.parent.parent
else:
prefix = path.parent.parent
- prefix = prefix.absolute()
elif sys.platform == "win32":
- for path in Path("c:/Program Files").glob("**/OTB-*/lib"):
- logger.info("Found %s", path.parent)
- prefix = path.parent.absolute()
- elif sys.platform == "darwin":
- for path in (Path.home() / "Applications").glob("**/OTB-*/lib"):
- logger.info("Found %s", path.parent)
- prefix = path.parent.absolute()
- # If possible, use OTB found in user's HOME tree (this may take some time)
- if scan_userdir:
- for path in sorted(Path.home().glob("**/OTB-*/lib/")):
+ for path in sorted(Path("c:/Program Files").glob("**/OTB-*/lib")):
logger.info("Found %s", path.parent)
- prefix = path.parent.absolute()
+ prefix = path.parent
+ # Search for pyotb OTB install, or default on macOS
+ apps = Path.home() / "Applications"
+ for path in sorted(apps.glob("OTB-*/lib/")):
+ logger.info("Found %s", path.parent)
+ prefix = path.parent
# Return latest found prefix (and version), see precedence in function def find_otb()
- return prefix
+ return prefix.absolute()
def __suggest_fix_import(error_message: str, prefix: str):
--
GitLab
From c3f8976301541055d48e00f32809837e9cc4e199 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 21:34:04 +0200
Subject: [PATCH 294/399] ENH: comment about pyotb init mechanism
---
pyotb/helpers.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 6319167..1a64fa0 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -432,9 +432,10 @@ def __suggest_fix_import(error_message: str, prefix: str):
"You can verify installation requirements for your OS at %s", DOCS_URL
)
-
-# Since helpers is the first module to be inititialized, this will prevent pyotb to run if OTB is not found
+# This part of pyotb is the first imported during __init__ and checks if OTB is found
+# User may trigger auto installation with
if os.environ.get("OTB_AUTO_INSTALL") in ("1", "true", "ON", "YES"):
find_otb(install_otb())
+# If OTB is not found, a SystemExit is raised, to prevent execution of the core module
else:
find_otb()
--
GitLab
From 36681d77311ec6cd38d4d1a5a85347a7d7cc2a38 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 21:34:15 +0200
Subject: [PATCH 295/399] CI: bump version
---
pyotb/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index 507a408..4e89789 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
"""This module provides convenient python wrapping of otbApplications."""
-__version__ = "2.0.0.dev4"
+__version__ = "2.0.0.dev5"
from .helpers import logger, set_logger_level
from .core import (
--
GitLab
From e4cf1b08751250b7dfd07bbd5eeb72059ccad2c8 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 22:06:11 +0200
Subject: [PATCH 296/399] Remove OTB_AUTO_INSTALL
---
pyotb/helpers.py | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 1a64fa0..ea791be 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -433,9 +433,5 @@ def __suggest_fix_import(error_message: str, prefix: str):
)
# This part of pyotb is the first imported during __init__ and checks if OTB is found
-# User may trigger auto installation with
-if os.environ.get("OTB_AUTO_INSTALL") in ("1", "true", "ON", "YES"):
- find_otb(install_otb())
# If OTB is not found, a SystemExit is raised, to prevent execution of the core module
-else:
- find_otb()
+find_otb()
--
GitLab
From 619dc50cdc87bf9909d9c342089f68d651f14309 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 22:09:08 +0200
Subject: [PATCH 297/399] FIX: window install just unzip files
---
pyotb/helpers.py | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index ea791be..53d9f9a 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -7,6 +7,7 @@ import re
import subprocess
import sys
import tempfile
+import zipfile
import urllib.request
from pathlib import Path
from shutil import which
@@ -237,9 +238,13 @@ def install_otb(version: str = "latest", path: str = ""):
path = Path(path)
else:
path = Path.home() / "Applications" / tmpfile.stem
- install_cmd = f"{cmd} {tmpfile} --target {path} --accept"
- print(f"Executing '{install_cmd}'\n")
- subprocess.run(f"{cmd} {tmpfile} --target {path} --accept", shell=True, check=True)
+ if sysname == "Win64":
+ with zipfile.ZipFile(tmpfile) as zipf:
+ zipf.extractall(path)
+ else:
+ install_cmd = f"{cmd} {tmpfile} --target {path} --accept"
+ print(f"Executing '{install_cmd}'\n")
+ subprocess.run(f"{cmd} {tmpfile} --target {path} --accept", shell=True, check=True)
tmpfile.unlink() # cleaning
# Add env variable to profile
--
GitLab
From 65cf76ebe173ca7469ccb58d1826a7c9874ca007 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 22:19:39 +0200
Subject: [PATCH 298/399] FIX: bug if OTB is not installed
---
pyotb/helpers.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 53d9f9a..9944dd3 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -98,6 +98,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True):
logger.info("Failed to import OTB. Searching for it...")
prefix = __find_otb_root()
if not prefix:
+ # Python is interactive
if hasattr(sys, "ps1"):
if input("OTB is missing. Do you want to install it ? (y/n): ") == "y":
version = input(
@@ -113,7 +114,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True):
"OTB not found on disk. "
"To install it, open an interactive python shell and type 'import pyotb'"
)
- # Try to import one last time before raising error
+ # If OTB is found on disk, set env and try to import one last time
try:
set_environment(prefix)
import otbApplication as otb # pylint: disable=import-outside-toplevel
@@ -381,7 +382,8 @@ def __find_otb_root():
logger.info("Found %s", path.parent)
prefix = path.parent
# Return latest found prefix (and version), see precedence in function def find_otb()
- return prefix.absolute()
+ if isinstance(prefix, Path):
+ return prefix.absolute()
def __suggest_fix_import(error_message: str, prefix: str):
--
GitLab
From 9ca34eecd801147f4c38efc17a8f4200d5d93316 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 22:29:20 +0200
Subject: [PATCH 299/399] FIX: zipfile not extracted in subdirectory
---
pyotb/helpers.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 9944dd3..ed2d96c 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -204,7 +204,7 @@ def install_otb(version: str = "latest", path: str = ""):
Args:
version: OTB version tag, e.g. '8.1.2'
- path: installation directory
+ path: installation directory, default is $HOME/Applications
Returns:
full path of the new installation
@@ -241,11 +241,12 @@ def install_otb(version: str = "latest", path: str = ""):
path = Path.home() / "Applications" / tmpfile.stem
if sysname == "Win64":
with zipfile.ZipFile(tmpfile) as zipf:
- zipf.extractall(path)
+ print("Extracting zip file...")
+ zipf.extractall(path.parent)
else:
install_cmd = f"{cmd} {tmpfile} --target {path} --accept"
print(f"Executing '{install_cmd}'\n")
- subprocess.run(f"{cmd} {tmpfile} --target {path} --accept", shell=True, check=True)
+ subprocess.run(install_cmd, shell=True, check=True)
tmpfile.unlink() # cleaning
# Add env variable to profile
--
GitLab
From 502fe83c7b7479a131f610358ef90eac487b20c4 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 22:35:49 +0200
Subject: [PATCH 300/399] STYLE: pylint
---
pyotb/helpers.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index ed2d96c..ac69729 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -385,6 +385,7 @@ def __find_otb_root():
# Return latest found prefix (and version), see precedence in function def find_otb()
if isinstance(prefix, Path):
return prefix.absolute()
+ return None
def __suggest_fix_import(error_message: str, prefix: str):
@@ -440,6 +441,7 @@ def __suggest_fix_import(error_message: str, prefix: str):
"You can verify installation requirements for your OS at %s", DOCS_URL
)
+
# This part of pyotb is the first imported during __init__ and checks if OTB is found
# If OTB is not found, a SystemExit is raised, to prevent execution of the core module
find_otb()
--
GitLab
From bf0030a35921f1b35696fc094abf1c5594c4bbb8 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 22:43:52 +0200
Subject: [PATCH 301/399] STYLE: add print headers for steps in install_otb
---
pyotb/helpers.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index ac69729..c99bbb8 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -233,7 +233,7 @@ def install_otb(version: str = "latest", path: str = ""):
url = f"https://www.orfeo-toolbox.org/packages/archives/OTB/{filename}"
tmpdir = tempfile.gettempdir()
tmpfile = Path(tmpdir) / filename
- print(f"Downloading {url}")
+ print(f"##### Downloading {url}")
urllib.request.urlretrieve(url, tmpfile)
if path:
path = Path(path)
@@ -241,18 +241,18 @@ def install_otb(version: str = "latest", path: str = ""):
path = Path.home() / "Applications" / tmpfile.stem
if sysname == "Win64":
with zipfile.ZipFile(tmpfile) as zipf:
- print("Extracting zip file...")
+ print("##### Extracting zip file...")
zipf.extractall(path.parent)
else:
install_cmd = f"{cmd} {tmpfile} --target {path} --accept"
- print(f"Executing '{install_cmd}'\n")
+ print(f"##### Executing '{install_cmd}'\n")
subprocess.run(install_cmd, shell=True, check=True)
tmpfile.unlink() # cleaning
# Add env variable to profile
if sysname != "Win64":
profile = Path.home() / ".profile"
- print(f"Adding new env variables to {profile}")
+ print(f"##### Adding new env variables to {profile}")
with open(profile, "a", encoding="utf-8") as buf:
buf.write(f'\n. "{path}/otbenv.profile"\n')
else:
@@ -266,7 +266,7 @@ def install_otb(version: str = "latest", path: str = ""):
return str(path)
# Else recompile bindings : this may fail because of OpenGL
if which("ctest") and which("python3-config"):
- print("\nRecompiling python bindings...")
+ print("\n##### Recompiling python bindings...")
ctest_cmd = (
". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV"
)
--
GitLab
From 7d24c797b0c10b0b9027ab0bcc3585d8c489f158 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 22:46:20 +0200
Subject: [PATCH 302/399] FIX: raise system exit with message
---
pyotb/helpers.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index c99bbb8..120b859 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -272,11 +272,11 @@ def install_otb(version: str = "latest", path: str = ""):
)
subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True)
return str(path)
- print(
+ msg = (
"\nYou need to install 'cmake', 'python3-dev' and 'libgl1-mesa-dev'"
" in order to recompile python bindings. "
)
- raise SystemExit
+ raise SystemExit(msg)
def __find_lib(prefix: str = None, otb_module=None):
--
GitLab
From 116cba62bb956c7072f510ee73babb0270c1d903 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 23:06:10 +0200
Subject: [PATCH 303/399] ENH: ask before editing user env .profile
---
pyotb/helpers.py | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 120b859..9739e62 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -108,7 +108,10 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True):
"Provide a path for installation "
"(default is <user_dir>/Applications/OTB-<version>): "
)
- return find_otb(install_otb(version, path))
+ edit_env = input(
+ "Enable user environment variables for this installation ? (y/n): "
+ )
+ return find_otb(install_otb(version, path, edit_env))
if not prefix:
raise SystemExit(
"OTB not found on disk. "
@@ -199,7 +202,7 @@ def otb_latest_release_tag():
return releases[-1]
-def install_otb(version: str = "latest", path: str = ""):
+def install_otb(version: str = "latest", path: str = "", edit_env: bool = False):
"""Install pre-compiled OTB binaries in path, use latest release by default.
Args:
@@ -250,7 +253,7 @@ def install_otb(version: str = "latest", path: str = ""):
tmpfile.unlink() # cleaning
# Add env variable to profile
- if sysname != "Win64":
+ if sysname != "Win64" and edit_env:
profile = Path.home() / ".profile"
print(f"##### Adding new env variables to {profile}")
with open(profile, "a", encoding="utf-8") as buf:
--
GitLab
From 95a41bd6bc73f1510128e164da4d4a2e0ff57bbd Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Thu, 10 Aug 2023 23:07:55 +0200
Subject: [PATCH 304/399] FIX: check user input before editing profile
---
pyotb/helpers.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 9739e62..c4414b7 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -109,8 +109,8 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True):
"(default is <user_dir>/Applications/OTB-<version>): "
)
edit_env = input(
- "Enable user environment variables for this installation ? (y/n): "
- )
+ "Modify user environment variables for this installation ? (y/n): "
+ ) == "y"
return find_otb(install_otb(version, path, edit_env))
if not prefix:
raise SystemExit(
--
GitLab
From 0d486b76ceb4df4ebbb379ff55a45429861389f9 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 15:19:37 +0200
Subject: [PATCH 305/399] ADD: new module pyotb.install, clean helpers.py
---
pyotb/__init__.py | 1 +
pyotb/helpers.py | 127 +++--------------------------------
pyotb/install.py | 165 ++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 176 insertions(+), 117 deletions(-)
create mode 100644 pyotb/install.py
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index 4e89789..c134e58 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -2,6 +2,7 @@
"""This module provides convenient python wrapping of otbApplications."""
__version__ = "2.0.0.dev5"
+from .install import install_otb
from .helpers import logger, set_logger_level
from .core import (
OTBObject,
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index c4414b7..ea14c55 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -1,17 +1,13 @@
# -*- coding: utf-8 -*-
"""This module helps to ensure we properly initialize pyotb: only in case OTB is found and apps are available."""
-import json
import logging
import os
-import re
-import subprocess
import sys
-import tempfile
-import zipfile
-import urllib.request
from pathlib import Path
from shutil import which
+from .install import install_otb, interactive_config
+
# Allow user to switch between OTB directories without setting every env variable
OTB_ROOT = os.environ.get("OTB_ROOT")
DOCS_URL = "https://www.orfeo-toolbox.org/CookBook/Installation.html"
@@ -97,27 +93,16 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True):
# Else search system
logger.info("Failed to import OTB. Searching for it...")
prefix = __find_otb_root()
- if not prefix:
- # Python is interactive
- if hasattr(sys, "ps1"):
- if input("OTB is missing. Do you want to install it ? (y/n): ") == "y":
- version = input(
- "Choose a version number to install (default is latest): "
- )
- path = input(
- "Provide a path for installation "
- "(default is <user_dir>/Applications/OTB-<version>): "
- )
- edit_env = input(
- "Modify user environment variables for this installation ? (y/n): "
- ) == "y"
- return find_otb(install_otb(version, path, edit_env))
- if not prefix:
+ # Try auto install if shell is interactive
+ if not prefix and hasattr(sys, "ps1"):
+ if input("OTB is missing. Do you want to install it ? (y/n): ") == "y":
+ return find_otb(install_otb(*interactive_config()))
+ elif not prefix:
raise SystemExit(
- "OTB not found on disk. "
- "To install it, open an interactive python shell and type 'import pyotb'"
+ "OTB libraries not found on disk. "
+ "To install it, open an interactive python shell and 'import pyotb'"
)
- # If OTB is found on disk, set env and try to import one last time
+ # If OTB was found on disk, set env and try to import one last time
try:
set_environment(prefix)
import otbApplication as otb # pylint: disable=import-outside-toplevel
@@ -190,98 +175,6 @@ def set_environment(prefix: str):
os.environ["PROJ_LIB"] = proj_lib
-def otb_latest_release_tag():
- """Use gitlab API to find latest release tag name, but skip pre-releases."""
- api_endpoint = "https://gitlab.orfeo-toolbox.org/api/v4/projects/53/repository/tags"
- vers_regex = re.compile(r"^\d\.\d\.\d$") # we ignore rc-* or alpha-*
- with urllib.request.urlopen(api_endpoint) as stream:
- data = json.loads(stream.read())
- releases = sorted(
- [tag["name"] for tag in data if vers_regex.match(tag["name"])],
- )
- return releases[-1]
-
-
-def install_otb(version: str = "latest", path: str = "", edit_env: bool = False):
- """Install pre-compiled OTB binaries in path, use latest release by default.
-
- Args:
- version: OTB version tag, e.g. '8.1.2'
- path: installation directory, default is $HOME/Applications
-
- Returns:
- full path of the new installation
- """
- major = sys.version_info.major
- if major == 2:
- raise SystemExit("Python 3 is required for OTB bindings.")
- minor = sys.version_info.minor
- name_corresp = {"linux": "Linux64", "darwnin": "Darwin64", "win32": "Win64"}
- sysname = name_corresp[sys.platform]
- if sysname == "Win64":
- if minor != 7:
- raise SystemExit(
- "Python version 3.7 is required to import python bindings on Windows."
- )
- cmd = which("cmd.exe")
- ext = "zip"
- else:
- cmd = which("zsh") or which("bash") or which("sh")
- ext = "run"
-
- # Fetch archive and run installer
- if not version or version == "latest":
- version = otb_latest_release_tag()
- filename = f"OTB-{version}-{sysname}.{ext}"
- url = f"https://www.orfeo-toolbox.org/packages/archives/OTB/{filename}"
- tmpdir = tempfile.gettempdir()
- tmpfile = Path(tmpdir) / filename
- print(f"##### Downloading {url}")
- urllib.request.urlretrieve(url, tmpfile)
- if path:
- path = Path(path)
- else:
- path = Path.home() / "Applications" / tmpfile.stem
- if sysname == "Win64":
- with zipfile.ZipFile(tmpfile) as zipf:
- print("##### Extracting zip file...")
- zipf.extractall(path.parent)
- else:
- install_cmd = f"{cmd} {tmpfile} --target {path} --accept"
- print(f"##### Executing '{install_cmd}'\n")
- subprocess.run(install_cmd, shell=True, check=True)
- tmpfile.unlink() # cleaning
-
- # Add env variable to profile
- if sysname != "Win64" and edit_env:
- profile = Path.home() / ".profile"
- print(f"##### Adding new env variables to {profile}")
- with open(profile, "a", encoding="utf-8") as buf:
- buf.write(f'\n. "{path}/otbenv.profile"\n')
- else:
- print(
- "In order to speed-up pyotb import, remember to call 'otbenv.bat' "
- f"before importing pyotb, or add 'OTB_ROOT=\"{path}\"' to your env variables."
- )
- return str(path)
- # Linux requires 3.8, macOS requires 3.7
- if (sysname == "Linux64" and minor == 8) or (sysname == "Darwin64" and minor == 7):
- return str(path)
- # Else recompile bindings : this may fail because of OpenGL
- if which("ctest") and which("python3-config"):
- print("\n##### Recompiling python bindings...")
- ctest_cmd = (
- ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV"
- )
- subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True)
- return str(path)
- msg = (
- "\nYou need to install 'cmake', 'python3-dev' and 'libgl1-mesa-dev'"
- " in order to recompile python bindings. "
- )
- raise SystemExit(msg)
-
-
def __find_lib(prefix: str = None, otb_module=None):
"""Try to find OTB external libraries directory.
diff --git a/pyotb/install.py b/pyotb/install.py
new file mode 100644
index 0000000..c4e1c9d
--- /dev/null
+++ b/pyotb/install.py
@@ -0,0 +1,165 @@
+"""This module contains functions for interactive auto installation of OTB."""
+import json
+import sys
+import re
+import subprocess
+import tempfile
+import zipfile
+import urllib.request
+from pathlib import Path
+from shutil import which
+
+
+def otb_latest_release_tag():
+ """Use gitlab API to find latest release tag name, but skip pre-releases."""
+ api_endpoint = "https://gitlab.orfeo-toolbox.org/api/v4/projects/53/repository/tags"
+ vers_regex = re.compile(r"^\d\.\d\.\d$") # we ignore rc-* or alpha-*
+ with urllib.request.urlopen(api_endpoint) as stream:
+ data = json.loads(stream.read())
+ releases = sorted(
+ [tag["name"] for tag in data if vers_regex.match(tag["name"])],
+ )
+ return releases[-1]
+
+
+def check_versions(sysname: str, python_minor: int, otb_major: int):
+ """Verify if python version is compatible with OTB version.
+
+ Args:
+ sysname: OTB's system name convention (Linux64, Darwin64, Win64)
+ python_minor: minor version of python
+ otb_major: major version of OTB to be installed
+ Returns:
+ True if requirements are satisfied
+ """
+ if sysname == "Win64":
+ expected = 5 if otb_major in (6, 7) else 7
+ if python_minor == expected:
+ return True, 0
+ elif sysname == "Darwin64":
+ expected = 7, 0
+ if python_minor == expected:
+ return True, 0
+ elif sysname == "Linux64":
+ expected = 5 if otb_major in (6, 7) else 8
+ if python_minor == expected:
+ return True, 0
+ return False, expected
+
+
+def install_otb(version: str = "latest", path: str = "", edit_env: bool = False):
+ """Install pre-compiled OTB binaries in path, use latest release by default.
+
+ Args:
+ version: OTB version tag, e.g. '8.1.2'
+ path: installation directory, default is $HOME/Applications
+
+ Returns:
+ full path of the new installation
+ """
+ # Read env config
+ if sys.version_info.major == 2:
+ raise SystemExit("Python 3 is required for OTB bindings.")
+ python_minor = sys.version_info.minor
+ if not version or version == "latest":
+ version = otb_latest_release_tag()
+ name_corresp = {"linux": "Linux64", "darwnin": "Darwin64", "win32": "Win64"}
+ sysname = name_corresp[sys.platform]
+ ext = "zip" if sysname == "Win64" else "run"
+ cmd = which("zsh") or which("bash") or which("sh")
+ otb_major = int(version[0])
+ check, expected = check_versions(sysname, python_minor, otb_major)
+ if sysname == "Win64" and not check:
+ print(f"Python 3.{expected} is required to import bindings on Windows.")
+ return
+
+ # Fetch archive and run installer
+ filename = f"OTB-{version}-{sysname}.{ext}"
+ url = f"https://www.orfeo-toolbox.org/packages/archives/OTB/{filename}"
+ tmpdir = tempfile.gettempdir()
+ tmpfile = Path(tmpdir) / filename
+ print(f"##### Downloading {url}")
+ urllib.request.urlretrieve(url, tmpfile)
+ if path:
+ default_path = False
+ path = Path(path)
+ else:
+ default_path = True
+ path = Path.home() / "Applications" / tmpfile.stem
+ if sysname == "Win64":
+ with zipfile.ZipFile(tmpfile) as zipf:
+ print("##### Extracting zip file...")
+ zipf.extractall(path.parent)
+ else:
+ install_cmd = f"{cmd} {tmpfile} --target {path} --accept"
+ print(f"##### Executing '{install_cmd}'\n")
+ subprocess.run(install_cmd, shell=True, check=True)
+ tmpfile.unlink() # cleaning
+
+ # Add env variable to profile
+ if edit_env:
+ if sysname == "Win64":
+ # TODO: import winreg
+ return str(path)
+ else:
+ profile = Path.home() / ".profile"
+ print(f"##### Adding new env variables to {profile}")
+ with open(profile, "a", encoding="utf-8") as buf:
+ buf.write(f'\n. "{path}/otbenv.profile"\n')
+ elif not default_path:
+ ext = "bat" if sysname == "Win64" else "profile"
+ print(
+ f"Remember to call 'otbenv.{ext}' before importing pyotb, "
+ f"or add 'OTB_ROOT=\"{path}\"' to your env variables."
+ )
+ # No recompilation or symlink required
+ if check:
+ return str(path)
+
+ # Else recompile bindings : this may fail because of OpenGL
+ target_lib = f"{path}/lib/libpython3.{expected}.so.rh-python3{expected}-1.0"
+ if which("ctest") and which("python3-config"):
+ print("\n##### Recompiling python bindings...")
+ ctest_cmd = (
+ ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV"
+ )
+ subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True)
+ return str(path)
+ # Use dirty cross python version symlink
+ elif sys.executable.startswith("/usr/bin"):
+ lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so"
+ if Path(lib).exists():
+ ln_cmd = f'ln -s "{lib}" "{target_lib}"'
+ try:
+ subprocess.run(ln_cmd, executable=cmd, shell=True, check=True)
+ return str(path)
+ except subprocess.CalledProcessError as err:
+ raise SystemExit(
+ "Unable to recompile python bindings, "
+ "some dependencies may require manuel installation"
+ ) from err
+ else:
+ print(
+ f"Unable to automatically locate library for executable {sys.executable}"
+ f"You'll need to manually symlink that one file to {target_lib}"
+ )
+ # TODO: support for auto build deps install using brew, apt, pacman/yay, yum...
+ msg = (
+ "\nYou need to install 'cmake', 'python3-dev' and 'libgl1-mesa-dev'"
+ " in order to recompile python bindings. "
+ )
+ raise SystemExit(msg)
+
+
+def interactive_config():
+ """Prompt user to configure installation variables."""
+ version = input("Choose a version number to install (default is latest): ")
+ path = input(
+ "Provide a path for installation "
+ "(default is <user_dir>/Applications/OTB-<version>): "
+ )
+ edit_env = (
+ input("Modify user environment variables for this installation ? (y/n): ")
+ == "y"
+ )
+ return version, path, edit_env
--
GitLab
From cdd87a628824199f8f7a1fd7fe81196b5eed6ac6 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 15:27:41 +0200
Subject: [PATCH 306/399] STYLE: enforce max line length=88
---
pyotb/helpers.py | 40 ++++++++++++++++++++++------------------
1 file changed, 22 insertions(+), 18 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index ea14c55..30c3e4a 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-"""This module helps to ensure we properly initialize pyotb: only in case OTB is found and apps are available."""
+"""This module ensure we properly initialize pyotb, or raise SystemExit in case of broken install."""
import logging
import os
import sys
@@ -43,8 +43,9 @@ def set_logger_level(level: str):
def find_otb(prefix: str = OTB_ROOT, scan: bool = True):
- """Try to load OTB bindings or scan system, help user in case of failure, set env variables.
+ """Try to load OTB bindings or scan system, help user in case of failure, set env.
+ If in interactive prompt, user will be asked if he wants to install OTB.
The OTB_ROOT variable allow one to override default OTB version, with auto env setting.
Path precedence : $OTB_ROOT > location of python bindings location
Then, if OTB is not found:
@@ -75,7 +76,8 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True):
raise SystemExit("Failed to import OTB. Exiting.") from e
# Else try import from actual Python path
try:
- # Here, we can't properly set env variables before OTB import. We assume user did this before running python
+ # Here, we can't properly set env variables before OTB import.
+ # We assume user did this before running python
# For LD_LIBRARY_PATH problems, use OTB_ROOT instead of PYTHONPATH
import otbApplication as otb # pylint: disable=import-outside-toplevel
@@ -145,9 +147,9 @@ def set_environment(prefix: str):
raise EnvironmentError("Can't find OTB Python API")
if otb_api not in sys.path:
sys.path.insert(0, otb_api)
- # Add /bin first in PATH, in order to avoid conflicts with another GDAL install when using os.system()
+ # Add /bin first in PATH, in order to avoid conflicts with another GDAL install
os.environ["PATH"] = f"{prefix / 'bin'}{os.pathsep}{os.environ['PATH']}"
- # Applications path (this can be tricky since OTB import will succeed even without apps)
+ # Ensure APPLICATION_PATH is set
apps_path = __find_apps_path(lib_dir)
if Path(apps_path).exists():
os.environ["OTB_APPLICATION_PATH"] = apps_path
@@ -180,7 +182,7 @@ def __find_lib(prefix: str = None, otb_module=None):
Args:
prefix: try with OTB root directory
- otb_module: try with OTB python module (otbApplication) library path if found, else None
+ otb_module: try with otbApplication library path if found, else None
Returns:
lib path
@@ -278,7 +280,7 @@ def __find_otb_root():
for path in sorted(apps.glob("OTB-*/lib/")):
logger.info("Found %s", path.parent)
prefix = path.parent
- # Return latest found prefix (and version), see precedence in function def find_otb()
+ # Return latest found prefix (and version), see precedence in find_otb() docstrings
if isinstance(prefix, Path):
return prefix.absolute()
return None
@@ -297,25 +299,27 @@ def __suggest_fix_import(error_message: str, prefix: str):
lib = (
f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so"
)
- if which("ctest"):
+ if which("ctest") and which("python3-config"):
logger.critical(
- "To recompile python bindings, use 'cd %s ; source otbenv.profile ; "
+ "To recompile python bindings, use "
+ "'cd %s ; source otbenv.profile ; "
"ctest -S share/otb/swig/build_wrapping.cmake -VV'",
prefix,
)
elif Path(lib).exists():
- expect_minor = int(error_message[11])
- if expect_minor != sys.version_info.minor:
+ expected = int(error_message[11])
+ if expected != sys.version_info.minor:
logger.critical(
- "Python library version mismatch (OTB was expecting 3.%s) : "
- "a simple symlink may not work, depending on your python version",
- expect_minor,
+ "Python library version mismatch (OTB expected 3.%s) : "
+ "a symlink may not work, depending on your python version",
+ expected,
)
- target_lib = f"{prefix}/lib/libpython3.{expect_minor}.so.rh-python3{expect_minor}-1.0"
- logger.critical("Use 'ln -s %s %s'", lib, target_lib)
+ target = f"{prefix}/lib/libpython3.{expected}.so.1.0"
+ logger.critical("Use 'ln -s %s %s'", lib, target)
else:
logger.critical(
- "You may need to install cmake in order to recompile python bindings"
+ "You may need to install cmake, python3-dev and mesa's libgl"
+ " in order to recompile python bindings"
)
else:
logger.critical(
@@ -326,7 +330,7 @@ def __suggest_fix_import(error_message: str, prefix: str):
if error_message.startswith("DLL load failed"):
if sys.version_info.minor != 7:
logger.critical(
- "You need Python 3.5 (OTB releases 6.4 to 7.4) or Python 3.7 (since OTB 8)"
+ "You need Python 3.5 (OTB 6.4 to 7.4) or Python 3.7 (since OTB 8)"
)
else:
logger.critical(
--
GitLab
From dca3ea5f75f9314b4faf78bc423cc25e88e3d145 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 15:34:22 +0200
Subject: [PATCH 307/399] FIX: raise error when recompilation failed
---
pyotb/install.py | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index c4e1c9d..8c1e797 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -123,21 +123,21 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
ctest_cmd = (
". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV"
)
- subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True)
- return str(path)
+ try:
+ subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True)
+ return str(path)
+ except subprocess.CalledProcessError as err:
+ raise SystemExit(
+ "Unable to recompile python bindings, "
+ "some dependencies may require manuel installation."
+ ) from err
# Use dirty cross python version symlink
elif sys.executable.startswith("/usr/bin"):
lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so"
if Path(lib).exists():
ln_cmd = f'ln -s "{lib}" "{target_lib}"'
- try:
- subprocess.run(ln_cmd, executable=cmd, shell=True, check=True)
- return str(path)
- except subprocess.CalledProcessError as err:
- raise SystemExit(
- "Unable to recompile python bindings, "
- "some dependencies may require manuel installation"
- ) from err
+ subprocess.run(ln_cmd, executable=cmd, shell=True, check=True)
+ return str(path)
else:
print(
f"Unable to automatically locate library for executable {sys.executable}"
--
GitLab
From 37827e778050edf8d51671b338c33e104f724bbe Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 15:35:41 +0200
Subject: [PATCH 308/399] ENH: talk about libgl1 in error message
---
pyotb/install.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 8c1e797..f5edeab 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -129,7 +129,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
except subprocess.CalledProcessError as err:
raise SystemExit(
"Unable to recompile python bindings, "
- "some dependencies may require manuel installation."
+ "some dependencies (libgl1) may require manual installation."
) from err
# Use dirty cross python version symlink
elif sys.executable.startswith("/usr/bin"):
--
GitLab
From b0a907d7003f3a48807ef66da1efe6f9b6bc5098 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 15:37:39 +0200
Subject: [PATCH 309/399] ENH: print message before creating symlinks
---
pyotb/install.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index f5edeab..6a7c66d 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -117,7 +117,6 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
return str(path)
# Else recompile bindings : this may fail because of OpenGL
- target_lib = f"{path}/lib/libpython3.{expected}.so.rh-python3{expected}-1.0"
if which("ctest") and which("python3-config"):
print("\n##### Recompiling python bindings...")
ctest_cmd = (
@@ -133,9 +132,11 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
) from err
# Use dirty cross python version symlink
elif sys.executable.startswith("/usr/bin"):
+ target_lib = f"{path}/lib/libpython3.{expected}.so.rh-python3{expected}-1.0"
lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so"
if Path(lib).exists():
- ln_cmd = f'ln -s "{lib}" "{target_lib}"'
+ print(f"Creating symbolic links: {lib} -> {target_lib}")
+ ln_cmd = f'ln -sf "{lib}" "{target_lib}"'
subprocess.run(ln_cmd, executable=cmd, shell=True, check=True)
return str(path)
else:
--
GitLab
From 7ecd4a4536b3b63ddbb426aa215abb6ac46f07ce Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 15:45:13 +0200
Subject: [PATCH 310/399] FIX: wrong python lib name for symlink with OTB 8
---
pyotb/install.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 6a7c66d..4f26afe 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -130,9 +130,10 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
"Unable to recompile python bindings, "
"some dependencies (libgl1) may require manual installation."
) from err
- # Use dirty cross python version symlink
+ # Use dirty cross python version symlink (only tested on Ubuntu)
elif sys.executable.startswith("/usr/bin"):
- target_lib = f"{path}/lib/libpython3.{expected}.so.rh-python3{expected}-1.0"
+ suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else ".so.1.0"
+ target_lib = f"{path}/lib/libpython3.{expected}.{suffix}"
lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so"
if Path(lib).exists():
print(f"Creating symbolic links: {lib} -> {target_lib}")
--
GitLab
From f7697d0d52341f27399a2a8b858ddc1d862345bd Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 15:46:53 +0200
Subject: [PATCH 311/399] ENH: log message about python lib symlinks
---
pyotb/helpers.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 30c3e4a..812b72e 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -315,7 +315,7 @@ def __suggest_fix_import(error_message: str, prefix: str):
expected,
)
target = f"{prefix}/lib/libpython3.{expected}.so.1.0"
- logger.critical("Use 'ln -s %s %s'", lib, target)
+ logger.critical("If using OTB>=8.0, use 'ln -s %s %s'", lib, target)
else:
logger.critical(
"You may need to install cmake, python3-dev and mesa's libgl"
--
GitLab
From f593595d6e27f58453421c12bd61140f2d998380 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 15:49:53 +0200
Subject: [PATCH 312/399] FIX: case python exe is not default system python
---
pyotb/install.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 4f26afe..32a78cb 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -117,6 +117,8 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
return str(path)
# Else recompile bindings : this may fail because of OpenGL
+ suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else ".so.1.0"
+ target_lib = f"{path}/lib/libpython3.{expected}.{suffix}"
if which("ctest") and which("python3-config"):
print("\n##### Recompiling python bindings...")
ctest_cmd = (
@@ -132,8 +134,6 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
) from err
# Use dirty cross python version symlink (only tested on Ubuntu)
elif sys.executable.startswith("/usr/bin"):
- suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else ".so.1.0"
- target_lib = f"{path}/lib/libpython3.{expected}.{suffix}"
lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so"
if Path(lib).exists():
print(f"Creating symbolic links: {lib} -> {target_lib}")
--
GitLab
From 1de178d4a249f306d65d8e928cf4d9bfb459c773 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 15:51:11 +0200
Subject: [PATCH 313/399] FIX: typo in python lib path
---
pyotb/install.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 32a78cb..a02cf9d 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -117,7 +117,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
return str(path)
# Else recompile bindings : this may fail because of OpenGL
- suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else ".so.1.0"
+ suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else "so.1.0"
target_lib = f"{path}/lib/libpython3.{expected}.{suffix}"
if which("ctest") and which("python3-config"):
print("\n##### Recompiling python bindings...")
--
GitLab
From 1458e253250f4be1bd759dfb909a1b390e9b6bb9 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 15:57:37 +0200
Subject: [PATCH 314/399] FIX: do not try to recompile when is colab env
---
pyotb/install.py | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index a02cf9d..2729f22 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -1,6 +1,7 @@
"""This module contains functions for interactive auto installation of OTB."""
import json
import sys
+import os
import re
import subprocess
import tempfile
@@ -119,13 +120,20 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
# Else recompile bindings : this may fail because of OpenGL
suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else "so.1.0"
target_lib = f"{path}/lib/libpython3.{expected}.{suffix}"
- if which("ctest") and which("python3-config"):
+ if (
+ which("ctest")
+ and which("python3-config")
+ # Google Colab ships with cmake and python3-dev, but not libgl1-mesa-dev
+ and "COLAB_RELEASE_TAG" not in os.environ
+ ):
print("\n##### Recompiling python bindings...")
ctest_cmd = (
". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV"
)
try:
- subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True)
+ subprocess.run(
+ ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True
+ )
return str(path)
except subprocess.CalledProcessError as err:
raise SystemExit(
--
GitLab
From c18843f560e21622f2180ca63efb2aa7db560090 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr>
Date: Fri, 11 Aug 2023 14:01:21 +0000
Subject: [PATCH 315/399] Apply 1 suggestion(s) to 1 file(s)
---
pyotb/install.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 2729f22..abf0499 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -64,7 +64,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
python_minor = sys.version_info.minor
if not version or version == "latest":
version = otb_latest_release_tag()
- name_corresp = {"linux": "Linux64", "darwnin": "Darwin64", "win32": "Win64"}
+ name_corresp = {"linux": "Linux64", "darwin": "Darwin64", "win32": "Win64"}
sysname = name_corresp[sys.platform]
ext = "zip" if sysname == "Win64" else "run"
cmd = which("zsh") or which("bash") or which("sh")
--
GitLab
From ebd7cb34fe481b247e511d7e6f25149a35da9ad7 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 16:14:07 +0200
Subject: [PATCH 316/399] STYLE: run pylint and isort
---
pyotb/install.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index abf0499..76f1feb 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -1,12 +1,12 @@
"""This module contains functions for interactive auto installation of OTB."""
import json
-import sys
import os
import re
import subprocess
+import sys
import tempfile
-import zipfile
import urllib.request
+import zipfile
from pathlib import Path
from shutil import which
@@ -72,7 +72,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
check, expected = check_versions(sysname, python_minor, otb_major)
if sysname == "Win64" and not check:
print(f"Python 3.{expected} is required to import bindings on Windows.")
- return
+ return ""
# Fetch archive and run installer
filename = f"OTB-{version}-{sysname}.{ext}"
@@ -101,7 +101,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
if edit_env:
if sysname == "Win64":
# TODO: import winreg
- return str(path)
+ ...
else:
profile = Path.home() / ".profile"
print(f"##### Adding new env variables to {profile}")
--
GitLab
From e85ea8281db2ce285248a89529c3781872207a5c Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 17:08:50 +0200
Subject: [PATCH 317/399] ADD: env setting using registry on windows
---
pyotb/install.py | 46 ++++++++++++++++++++++++++++++++++++++--------
1 file changed, 38 insertions(+), 8 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 76f1feb..35b57a2 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -48,6 +48,39 @@ def check_versions(sysname: str, python_minor: int, otb_major: int):
return False, expected
+def update_unix_env(otb_path: Path):
+ """Update env profile for current user with new otb_env.profile call.
+
+ Args:
+ otb_path: the path of the new OTB installation
+
+ """
+ profile = Path.home() / ".profile"
+ with open(profile, "a", encoding="utf-8") as buf:
+ buf.write(f'\n. "{otb_path}/otbenv.profile"\n')
+ print(f"##### Added new environment variables to {profile}")
+
+
+def update_windows_env(otb_path: Path):
+ """Update registry hive for current user with new OTB_ROOT env variable.
+
+ Args:
+ otb_path: path of the new OTB installation
+
+ """
+ import winreg # pylint: disable=import-error,import-outside-toplevel
+
+ with winreg.OpenKeyEx(
+ winreg.HKEY_CURRENT_USER, "Environment", 0, winreg.KEY_SET_VALUE
+ ) as reg_key:
+ winreg.SetValueEx(reg_key, "OTB_ROOT", 0, winreg.REG_EXPAND_SZ, str(otb_path))
+ print(
+ "##### Environment variable 'OTB_ROOT' added successfully to user's hive."
+ )
+ key = "HKEY_CURRENT_USER\\Environment\\OTB_ROOT"
+ print(f"To undo this permanent setting, use 'reg.exe delete \"{key}\"'")
+
+
def install_otb(version: str = "latest", path: str = "", edit_env: bool = False):
"""Install pre-compiled OTB binaries in path, use latest release by default.
@@ -57,6 +90,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
Returns:
full path of the new installation
+
"""
# Read env config
if sys.version_info.major == 2:
@@ -89,7 +123,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
path = Path.home() / "Applications" / tmpfile.stem
if sysname == "Win64":
with zipfile.ZipFile(tmpfile) as zipf:
- print("##### Extracting zip file...")
+ print("##### Extracting zip file...\n")
zipf.extractall(path.parent)
else:
install_cmd = f"{cmd} {tmpfile} --target {path} --accept"
@@ -100,17 +134,13 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
# Add env variable to profile
if edit_env:
if sysname == "Win64":
- # TODO: import winreg
- ...
+ update_windows_env(path)
else:
- profile = Path.home() / ".profile"
- print(f"##### Adding new env variables to {profile}")
- with open(profile, "a", encoding="utf-8") as buf:
- buf.write(f'\n. "{path}/otbenv.profile"\n')
+ update_unix_env(path)
elif not default_path:
ext = "bat" if sysname == "Win64" else "profile"
print(
- f"Remember to call 'otbenv.{ext}' before importing pyotb, "
+ f"Remember to call '{path}{os.sep}otbenv.{ext}' before importing pyotb, "
f"or add 'OTB_ROOT=\"{path}\"' to your env variables."
)
# No recompilation or symlink required
--
GitLab
From 2e7b43e5de020a097a2e64ac26ffaa8c06e890bf Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 17:18:47 +0200
Subject: [PATCH 318/399] ENH: install_otb smaller functions for env, recompile
and symlinks steps
---
pyotb/install.py | 97 ++++++++++++++++++++++++++++--------------------
1 file changed, 56 insertions(+), 41 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 35b57a2..8f6c281 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -11,6 +11,20 @@ from pathlib import Path
from shutil import which
+def interactive_config():
+ """Prompt user to configure installation variables."""
+ version = input("Choose a version number to install (default is latest): ")
+ path = input(
+ "Provide a path for installation "
+ "(default is <user_dir>/Applications/OTB-<version>): "
+ )
+ edit_env = (
+ input("Modify user environment variables for this installation ? (y/n): ")
+ == "y"
+ )
+ return version, path, edit_env
+
+
def otb_latest_release_tag():
"""Use gitlab API to find latest release tag name, but skip pre-releases."""
api_endpoint = "https://gitlab.orfeo-toolbox.org/api/v4/projects/53/repository/tags"
@@ -81,6 +95,40 @@ def update_windows_env(otb_path: Path):
print(f"To undo this permanent setting, use 'reg.exe delete \"{key}\"'")
+def recompile_python_bindings(path: str, cmd: str):
+ """Run subprocess command to recompile python bindings.
+
+ Args:
+ path: path of the new OTB installation
+ cmd: path of the default system shell command
+
+ """
+ print("\n##### Recompiling python bindings...")
+ ctest_cmd = ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV"
+ try:
+ subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True)
+ except subprocess.CalledProcessError as err:
+ raise SystemExit(
+ "Unable to recompile python bindings, "
+ "some dependencies (libgl1) may require manual installation."
+ ) from err
+
+
+def symlink_python_library(target_lib: str, cmd: str):
+ """Run subprocess command to recompile python bindings.
+
+ Args:
+ path: path of the new OTB installation
+ cmd: path of the default system shell command
+
+ """
+ lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so"
+ if Path(lib).exists():
+ print(f"##### Creating symbolic links: {lib} -> {target_lib}")
+ ln_cmd = f'ln -sf "{lib}" "{target_lib}"'
+ subprocess.run(ln_cmd, executable=cmd, shell=True, check=True)
+
+
def install_otb(version: str = "latest", path: str = "", edit_env: bool = False):
"""Install pre-compiled OTB binaries in path, use latest release by default.
@@ -150,34 +198,15 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
# Else recompile bindings : this may fail because of OpenGL
suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else "so.1.0"
target_lib = f"{path}/lib/libpython3.{expected}.{suffix}"
- if (
- which("ctest")
- and which("python3-config")
- # Google Colab ships with cmake and python3-dev, but not libgl1-mesa-dev
- and "COLAB_RELEASE_TAG" not in os.environ
- ):
- print("\n##### Recompiling python bindings...")
- ctest_cmd = (
- ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV"
- )
- try:
- subprocess.run(
- ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True
- )
- return str(path)
- except subprocess.CalledProcessError as err:
- raise SystemExit(
- "Unable to recompile python bindings, "
- "some dependencies (libgl1) may require manual installation."
- ) from err
- # Use dirty cross python version symlink (only tested on Ubuntu)
+ can_compile = which("ctest") and which("python3-config")
+ # Google Colab ships with cmake and python3-dev, but not libgl1-mesa-dev
+ if can_compile and "COLAB_RELEASE_TAG" not in os.environ:
+ recompile_python_bindings(path, cmd)
+ return str(path)
+ # Or use dirty cross version python symlink (only tested on Ubuntu)
elif sys.executable.startswith("/usr/bin"):
- lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so"
- if Path(lib).exists():
- print(f"Creating symbolic links: {lib} -> {target_lib}")
- ln_cmd = f'ln -sf "{lib}" "{target_lib}"'
- subprocess.run(ln_cmd, executable=cmd, shell=True, check=True)
- return str(path)
+ symlink_python_library(target_lib, cmd)
+ return str(path)
else:
print(
f"Unable to automatically locate library for executable {sys.executable}"
@@ -189,17 +218,3 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
" in order to recompile python bindings. "
)
raise SystemExit(msg)
-
-
-def interactive_config():
- """Prompt user to configure installation variables."""
- version = input("Choose a version number to install (default is latest): ")
- path = input(
- "Provide a path for installation "
- "(default is <user_dir>/Applications/OTB-<version>): "
- )
- edit_env = (
- input("Modify user environment variables for this installation ? (y/n): ")
- == "y"
- )
- return version, path, edit_env
--
GitLab
From 6a9a53a5c0422f9af0cee17c64a36f5b3606ef08 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 17:20:16 +0200
Subject: [PATCH 319/399] DOC: update docstrings
---
pyotb/install.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 8f6c281..9d3402e 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -95,7 +95,7 @@ def update_windows_env(otb_path: Path):
print(f"To undo this permanent setting, use 'reg.exe delete \"{key}\"'")
-def recompile_python_bindings(path: str, cmd: str):
+def recompile_python_bindings(path: Path, cmd: str):
"""Run subprocess command to recompile python bindings.
Args:
@@ -118,7 +118,7 @@ def symlink_python_library(target_lib: str, cmd: str):
"""Run subprocess command to recompile python bindings.
Args:
- path: path of the new OTB installation
+ target_lib: path of the missing python library
cmd: path of the default system shell command
"""
@@ -135,6 +135,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
Args:
version: OTB version tag, e.g. '8.1.2'
path: installation directory, default is $HOME/Applications
+ edit_env: wether or not to permanently modify user's environment variables
Returns:
full path of the new installation
--
GitLab
From 1c7bb6fd538f739f5a8307ead782970afbcdec06 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 18:11:30 +0200
Subject: [PATCH 320/399] STYLE: codespell and pylint
---
pyotb/install.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 9d3402e..345f43b 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -135,7 +135,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
Args:
version: OTB version tag, e.g. '8.1.2'
path: installation directory, default is $HOME/Applications
- edit_env: wether or not to permanently modify user's environment variables
+ edit_env: whether or not to permanently modify user's environment variables
Returns:
full path of the new installation
@@ -205,7 +205,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
recompile_python_bindings(path, cmd)
return str(path)
# Or use dirty cross version python symlink (only tested on Ubuntu)
- elif sys.executable.startswith("/usr/bin"):
+ if sys.executable.startswith("/usr/bin"):
symlink_python_library(target_lib, cmd)
return str(path)
else:
--
GitLab
From 3099a5bcd7210d7dd7699676e200a398020d60e9 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 18:16:42 +0200
Subject: [PATCH 321/399] FIX: ensure to exit python if version requirement is
not satisfied for Windows
---
pyotb/install.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 345f43b..9a59842 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -154,8 +154,9 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
otb_major = int(version[0])
check, expected = check_versions(sysname, python_minor, otb_major)
if sysname == "Win64" and not check:
- print(f"Python 3.{expected} is required to import bindings on Windows.")
- return ""
+ raise SystemExit(
+ f"Python 3.{expected} is required to import bindings on Windows."
+ )
# Fetch archive and run installer
filename = f"OTB-{version}-{sysname}.{ext}"
--
GitLab
From 29530682207006e1ee2343b6c00019bf9c33afa7 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 18:46:24 +0200
Subject: [PATCH 322/399] Fix Windows install custom path + hints about
registry
---
pyotb/install.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 9a59842..f6fc07c 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -90,9 +90,10 @@ def update_windows_env(otb_path: Path):
winreg.SetValueEx(reg_key, "OTB_ROOT", 0, winreg.REG_EXPAND_SZ, str(otb_path))
print(
"##### Environment variable 'OTB_ROOT' added successfully to user's hive."
+ "You'll need to login / logout to apply this change."
)
- key = "HKEY_CURRENT_USER\\Environment\\OTB_ROOT"
- print(f"To undo this permanent setting, use 'reg.exe delete \"{key}\"'")
+ reg_cmd = "reg.exe delete HKEY_CURRENT_USER\\Environment /v OTB_ROOT /f"
+ print(f"To undo this permanent setting, use '{reg_cmd}'")
def recompile_python_bindings(path: Path, cmd: str):
@@ -174,7 +175,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
if sysname == "Win64":
with zipfile.ZipFile(tmpfile) as zipf:
print("##### Extracting zip file...\n")
- zipf.extractall(path.parent)
+ zipf.extractall(path.parent if default_path else path)
else:
install_cmd = f"{cmd} {tmpfile} --target {path} --accept"
print(f"##### Executing '{install_cmd}'\n")
--
GitLab
From 9609826e8df9318b9250dccdf0ac05117cc13fea Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 18:51:36 +0200
Subject: [PATCH 323/399] FIX: user custom path must be parent dir
---
pyotb/install.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index f6fc07c..7f42bc2 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -15,8 +15,8 @@ def interactive_config():
"""Prompt user to configure installation variables."""
version = input("Choose a version number to install (default is latest): ")
path = input(
- "Provide a path for installation "
- "(default is <user_dir>/Applications/OTB-<version>): "
+ "Provide a parent directory for installation "
+ "(default is <user_dir>/Applications/): "
)
edit_env = (
input("Modify user environment variables for this installation ? (y/n): ")
@@ -175,6 +175,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
if sysname == "Win64":
with zipfile.ZipFile(tmpfile) as zipf:
print("##### Extracting zip file...\n")
+ # Unzip will always create a dir with OTB-version name
zipf.extractall(path.parent if default_path else path)
else:
install_cmd = f"{cmd} {tmpfile} --target {path} --accept"
--
GitLab
From c72358fc77b36b5b0c8b79a6c23f296815b8ab12 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 18:53:12 +0200
Subject: [PATCH 324/399] STYLE: remove print newline
---
pyotb/install.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 7f42bc2..178d39c 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -174,7 +174,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
path = Path.home() / "Applications" / tmpfile.stem
if sysname == "Win64":
with zipfile.ZipFile(tmpfile) as zipf:
- print("##### Extracting zip file...\n")
+ print("##### Extracting zip file...")
# Unzip will always create a dir with OTB-version name
zipf.extractall(path.parent if default_path else path)
else:
--
GitLab
From 2cfeb2ea6af5f05d1877ab74a26031ad34022dd7 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 18:58:19 +0200
Subject: [PATCH 325/399] STYLE: run pylint
---
pyotb/install.py | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 178d39c..cb7746b 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -211,11 +211,10 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
if sys.executable.startswith("/usr/bin"):
symlink_python_library(target_lib, cmd)
return str(path)
- else:
- print(
- f"Unable to automatically locate library for executable {sys.executable}"
- f"You'll need to manually symlink that one file to {target_lib}"
- )
+ print(
+ f"Unable to automatically locate library for executable {sys.executable}"
+ f"You could manually create a symlink from that file to {target_lib}"
+ )
# TODO: support for auto build deps install using brew, apt, pacman/yay, yum...
msg = (
"\nYou need to install 'cmake', 'python3-dev' and 'libgl1-mesa-dev'"
--
GitLab
From 7b781eda246262ae098e025c0245f757b485835f Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 19:43:00 +0200
Subject: [PATCH 326/399] ENH: better messages in cli
---
pyotb/install.py | 16 ++++++----------
1 file changed, 6 insertions(+), 10 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index cb7746b..6ce23c2 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -13,16 +13,12 @@ from shutil import which
def interactive_config():
"""Prompt user to configure installation variables."""
- version = input("Choose a version number to install (default is latest): ")
+ version = input("OTB version to download (default is latest): ")
path = input(
- "Provide a parent directory for installation "
- "(default is <user_dir>/Applications/): "
+ "Parent directory for installation (default is <user_dir>/Applications/): "
)
- edit_env = (
- input("Modify user environment variables for this installation ? (y/n): ")
- == "y"
- )
- return version, path, edit_env
+ env = input("Permanently change user's environment variables ? (y/n): ") == "y"
+ return version, path, env
def otb_latest_release_tag():
@@ -89,11 +85,11 @@ def update_windows_env(otb_path: Path):
) as reg_key:
winreg.SetValueEx(reg_key, "OTB_ROOT", 0, winreg.REG_EXPAND_SZ, str(otb_path))
print(
- "##### Environment variable 'OTB_ROOT' added successfully to user's hive."
+ "##### Environment variable 'OTB_ROOT' added to user's registry."
"You'll need to login / logout to apply this change."
)
reg_cmd = "reg.exe delete HKEY_CURRENT_USER\\Environment /v OTB_ROOT /f"
- print(f"To undo this permanent setting, use '{reg_cmd}'")
+ print(f"To undo this, you may use '{reg_cmd}'")
def recompile_python_bindings(path: Path, cmd: str):
--
GitLab
From cd6662d8b3bf3dfcb1dc6cba16ea47abd88a2e74 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 19:45:21 +0200
Subject: [PATCH 327/399] ENH: prompt messages
---
pyotb/install.py | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 6ce23c2..1e4eb6c 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -13,10 +13,9 @@ from shutil import which
def interactive_config():
"""Prompt user to configure installation variables."""
- version = input("OTB version to download (default is latest): ")
- path = input(
- "Parent directory for installation (default is <user_dir>/Applications/): "
- )
+ version = input("Choose a version to download (default is latest): ")
+ default_dir = f"<user_dir>{os.path.sep}Applications{os.path.sep}"
+ path = input(f"Parent directory for installation (default is {default_dir}): ")
env = input("Permanently change user's environment variables ? (y/n): ") == "y"
return version, path, env
--
GitLab
From 66c308044da1560dc7ffdfa05675443c01d047e4 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 19:46:45 +0200
Subject: [PATCH 328/399] ENH: use pathlib to print absolute path of default
install dir
---
pyotb/install.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 1e4eb6c..ad3cc76 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -14,7 +14,7 @@ from shutil import which
def interactive_config():
"""Prompt user to configure installation variables."""
version = input("Choose a version to download (default is latest): ")
- default_dir = f"<user_dir>{os.path.sep}Applications{os.path.sep}"
+ default_dir = Path.home() / "Applications"
path = input(f"Parent directory for installation (default is {default_dir}): ")
env = input("Permanently change user's environment variables ? (y/n): ") == "y"
return version, path, env
--
GitLab
From ddb0863c47bfe184e353cd980b3a69e572f395d2 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 19:49:24 +0200
Subject: [PATCH 329/399] ENH: function names
---
pyotb/install.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index ad3cc76..5a08592 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -57,7 +57,7 @@ def check_versions(sysname: str, python_minor: int, otb_major: int):
return False, expected
-def update_unix_env(otb_path: Path):
+def env_config_unix(otb_path: Path):
"""Update env profile for current user with new otb_env.profile call.
Args:
@@ -70,7 +70,7 @@ def update_unix_env(otb_path: Path):
print(f"##### Added new environment variables to {profile}")
-def update_windows_env(otb_path: Path):
+def env_config_windows(otb_path: Path):
"""Update registry hive for current user with new OTB_ROOT env variable.
Args:
@@ -84,7 +84,7 @@ def update_windows_env(otb_path: Path):
) as reg_key:
winreg.SetValueEx(reg_key, "OTB_ROOT", 0, winreg.REG_EXPAND_SZ, str(otb_path))
print(
- "##### Environment variable 'OTB_ROOT' added to user's registry."
+ "##### Environment variable 'OTB_ROOT' added to user's registry. "
"You'll need to login / logout to apply this change."
)
reg_cmd = "reg.exe delete HKEY_CURRENT_USER\\Environment /v OTB_ROOT /f"
@@ -181,9 +181,9 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
# Add env variable to profile
if edit_env:
if sysname == "Win64":
- update_windows_env(path)
+ env_config_windows(path)
else:
- update_unix_env(path)
+ env_config_unix(path)
elif not default_path:
ext = "bat" if sysname == "Win64" else "profile"
print(
--
GitLab
From 3ac9f25ec53af5d385333fde2e1139a85cbd8f93 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 21:34:15 +0200
Subject: [PATCH 330/399] FIX: always try the symlink trick
---
pyotb/install.py | 79 ++++++++++++++++--------------------------------
1 file changed, 26 insertions(+), 53 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 5a08592..d8b6c60 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -91,41 +91,7 @@ def env_config_windows(otb_path: Path):
print(f"To undo this, you may use '{reg_cmd}'")
-def recompile_python_bindings(path: Path, cmd: str):
- """Run subprocess command to recompile python bindings.
-
- Args:
- path: path of the new OTB installation
- cmd: path of the default system shell command
-
- """
- print("\n##### Recompiling python bindings...")
- ctest_cmd = ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV"
- try:
- subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), shell=True, check=True)
- except subprocess.CalledProcessError as err:
- raise SystemExit(
- "Unable to recompile python bindings, "
- "some dependencies (libgl1) may require manual installation."
- ) from err
-
-
-def symlink_python_library(target_lib: str, cmd: str):
- """Run subprocess command to recompile python bindings.
-
- Args:
- target_lib: path of the missing python library
- cmd: path of the default system shell command
-
- """
- lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so"
- if Path(lib).exists():
- print(f"##### Creating symbolic links: {lib} -> {target_lib}")
- ln_cmd = f'ln -sf "{lib}" "{target_lib}"'
- subprocess.run(ln_cmd, executable=cmd, shell=True, check=True)
-
-
-def install_otb(version: str = "latest", path: str = "", edit_env: bool = False):
+def install_otb(version: str = "latest", path: str = "", edit_env: bool = True):
"""Install pre-compiled OTB binaries in path, use latest release by default.
Args:
@@ -153,7 +119,6 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
raise SystemExit(
f"Python 3.{expected} is required to import bindings on Windows."
)
-
# Fetch archive and run installer
filename = f"OTB-{version}-{sysname}.{ext}"
url = f"https://www.orfeo-toolbox.org/packages/archives/OTB/{filename}"
@@ -194,25 +159,33 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = False)
if check:
return str(path)
- # Else recompile bindings : this may fail because of OpenGL
+ # Here version check failed, try recompile bindings : can fail because of OpenGL
suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else "so.1.0"
target_lib = f"{path}/lib/libpython3.{expected}.{suffix}"
- can_compile = which("ctest") and which("python3-config")
- # Google Colab ships with cmake and python3-dev, but not libgl1-mesa-dev
- if can_compile and "COLAB_RELEASE_TAG" not in os.environ:
- recompile_python_bindings(path, cmd)
- return str(path)
- # Or use dirty cross version python symlink (only tested on Ubuntu)
- if sys.executable.startswith("/usr/bin"):
- symlink_python_library(target_lib, cmd)
- return str(path)
+ if which("ctest") and which("python3-config"):
+ try:
+ print("\n##### Python version mismatch. Trying to recompile bindings...")
+ ctest_cmd = (
+ ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV"
+ )
+ subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), check=True)
+ return str(path)
+ except subprocess.CalledProcessError:
+ print("\nCompilation failed. ")
print(
- f"Unable to automatically locate library for executable {sys.executable}"
- f"You could manually create a symlink from that file to {target_lib}"
+ "You need cmake, python3-dev and libgl1-mesa-dev installed. "
+ "Trying to symlink libraries instead - this may fail with newest versions."
)
- # TODO: support for auto build deps install using brew, apt, pacman/yay, yum...
- msg = (
- "\nYou need to install 'cmake', 'python3-dev' and 'libgl1-mesa-dev'"
- " in order to recompile python bindings. "
+ # TODO: support for sudo auto build deps install using apt, pacman/yay, brew...
+ # Else use dirty cross version python symlink (only tested on Ubuntu)
+ if sys.executable.startswith("/usr/bin"):
+ lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so"
+ if Path(lib).exists():
+ print(f"##### Creating symbolic links: {lib} -> {target_lib}")
+ ln_cmd = f'ln -sf "{lib}" "{target_lib}"'
+ subprocess.run(ln_cmd, executable=cmd, shell=True, check=True)
+ return str(path)
+ raise SystemExit(
+ f"Unable to automatically locate library for executable '{sys.executable}', "
+ f"you could manually create a symlink from that file to {target_lib}"
)
- raise SystemExit(msg)
--
GitLab
From a4c12ab85249ee95b189d3c4787db66ec861a8ea Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 22:32:52 +0200
Subject: [PATCH 331/399] FIX: shell=True is required for process to complete
---
pyotb/install.py | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index d8b6c60..ed54a25 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -134,7 +134,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True):
path = Path.home() / "Applications" / tmpfile.stem
if sysname == "Win64":
with zipfile.ZipFile(tmpfile) as zipf:
- print("##### Extracting zip file...")
+ print("##### Extracting zip file")
# Unzip will always create a dir with OTB-version name
zipf.extractall(path.parent if default_path else path)
else:
@@ -164,14 +164,15 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True):
target_lib = f"{path}/lib/libpython3.{expected}.{suffix}"
if which("ctest") and which("python3-config"):
try:
- print("\n##### Python version mismatch. Trying to recompile bindings...")
+ print("\n!!!!! Python version mismatch, trying to recompile bindings")
ctest_cmd = (
- ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -VV"
+ ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -V"
)
- subprocess.run(ctest_cmd, executable=cmd, cwd=str(path), check=True)
+ print(f"##### Executing '{ctest_cmd}'")
+ subprocess.run(ctest_cmd, cwd=path, check=True, shell=True)
return str(path)
except subprocess.CalledProcessError:
- print("\nCompilation failed. ")
+ print("\nCompilation failed.")
print(
"You need cmake, python3-dev and libgl1-mesa-dev installed. "
"Trying to symlink libraries instead - this may fail with newest versions."
--
GitLab
From aff854d8395f3c287517ec6da8f8399f29c0a2b5 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 22:34:00 +0200
Subject: [PATCH 332/399] ENH: use SystemError instead of EnvironmentError
---
pyotb/helpers.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 812b72e..124abaa 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -69,7 +69,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True):
import otbApplication as otb # pylint: disable=import-outside-toplevel
return otb
- except EnvironmentError as e:
+ except SystemError as e:
raise SystemExit(f"Failed to import OTB with prefix={prefix}") from e
except ImportError as e:
__suggest_fix_import(str(e), prefix)
@@ -110,7 +110,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True):
import otbApplication as otb # pylint: disable=import-outside-toplevel
return otb
- except EnvironmentError as e:
+ except SystemError as e:
raise SystemExit("Auto setup for OTB env failed. Exiting.") from e
# Help user to fix this
except ImportError as e:
@@ -136,7 +136,7 @@ def set_environment(prefix: str):
# External libraries
lib_dir = __find_lib(prefix)
if not lib_dir:
- raise EnvironmentError("Can't find OTB external libraries")
+ raise SystemError("Can't find OTB external libraries")
# This does not seems to work
if sys.platform == "linux" and built_from_source:
new_ld_path = f"{lib_dir}:{os.environ.get('LD_LIBRARY_PATH') or ''}"
@@ -144,7 +144,7 @@ def set_environment(prefix: str):
# Add python bindings directory first in PYTHONPATH
otb_api = __find_python_api(lib_dir)
if not otb_api:
- raise EnvironmentError("Can't find OTB Python API")
+ raise SystemError("Can't find OTB Python API")
if otb_api not in sys.path:
sys.path.insert(0, otb_api)
# Add /bin first in PATH, in order to avoid conflicts with another GDAL install
@@ -154,7 +154,7 @@ def set_environment(prefix: str):
if Path(apps_path).exists():
os.environ["OTB_APPLICATION_PATH"] = apps_path
else:
- raise EnvironmentError("Can't find OTB applications directory")
+ raise SystemError("Can't find OTB applications directory")
os.environ["LC_NUMERIC"] = "C"
os.environ["GDAL_DRIVER_PATH"] = "disable"
@@ -170,7 +170,7 @@ def set_environment(prefix: str):
gdal_data = str(prefix / "share/data")
proj_lib = str(prefix / "share/proj")
else:
- raise EnvironmentError(
+ raise SystemError(
f"Can't find GDAL location with current OTB prefix '{prefix}' or in /usr"
)
os.environ["GDAL_DATA"] = gdal_data
--
GitLab
From 59a63f396157bca70195adbf772976ed2c920a3e Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 22:34:20 +0200
Subject: [PATCH 333/399] FIX: do not exit python when interactive
---
pyotb/helpers.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 124abaa..3560178 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -99,6 +99,8 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True):
if not prefix and hasattr(sys, "ps1"):
if input("OTB is missing. Do you want to install it ? (y/n): ") == "y":
return find_otb(install_otb(*interactive_config()))
+ else:
+ raise SystemError("OTB libraries not found on disk. ")
elif not prefix:
raise SystemExit(
"OTB libraries not found on disk. "
--
GitLab
From 5cc50cea8105f88dcd3d8859e190a2195db3b463 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 22:45:58 +0200
Subject: [PATCH 334/399] STYLE: run pylint
---
pyotb/helpers.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 3560178..9322041 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -99,8 +99,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True):
if not prefix and hasattr(sys, "ps1"):
if input("OTB is missing. Do you want to install it ? (y/n): ") == "y":
return find_otb(install_otb(*interactive_config()))
- else:
- raise SystemError("OTB libraries not found on disk. ")
+ raise SystemError("OTB libraries not found on disk. ")
elif not prefix:
raise SystemExit(
"OTB libraries not found on disk. "
--
GitLab
From 783d95627abf7077c76a21e05203bd1472ad1894 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 11 Aug 2023 22:49:50 +0200
Subject: [PATCH 335/399] STYLE: remove useless elif
---
pyotb/helpers.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 9322041..907ab88 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -100,7 +100,7 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True):
if input("OTB is missing. Do you want to install it ? (y/n): ") == "y":
return find_otb(install_otb(*interactive_config()))
raise SystemError("OTB libraries not found on disk. ")
- elif not prefix:
+ if not prefix:
raise SystemExit(
"OTB libraries not found on disk. "
"To install it, open an interactive python shell and 'import pyotb'"
--
GitLab
From eef2ccf81751ba6af7b77b065ede257a68f5df1b Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sat, 12 Aug 2023 00:07:49 +0200
Subject: [PATCH 336/399] ENH: show pylint TODO in IDE, but not in CI
---
.gitlab-ci.yml | 2 +-
pyproject.toml | 1 -
2 files changed, 1 insertion(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2c6e521..f0846f0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -47,7 +47,7 @@ pylint:
before_script:
- pip install pylint
script:
- - pylint $PWD/pyotb
+ - pylint $PWD/pyotb --disable=fixme
codespell:
extends: .static_analysis
diff --git a/pyproject.toml b/pyproject.toml
index 30804d4..fdaa0a3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -56,7 +56,6 @@ max-line-length = 88
max-module-lines = 2000
good-names = ["x", "y", "i", "j", "k", "e"]
disable = [
- "fixme",
"line-too-long",
"too-many-locals",
"too-many-branches",
--
GitLab
From 87de0932f8e5c9dcfc878c2152dd8fcf51e4fa66 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 13 Aug 2023 10:51:58 +0200
Subject: [PATCH 337/399] ENH: comments and docstrings
---
pyotb/helpers.py | 3 +--
pyotb/install.py | 16 +++++++---------
2 files changed, 8 insertions(+), 11 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 907ab88..b043f13 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -302,8 +302,7 @@ def __suggest_fix_import(error_message: str, prefix: str):
)
if which("ctest") and which("python3-config"):
logger.critical(
- "To recompile python bindings, use "
- "'cd %s ; source otbenv.profile ; "
+ "To compile, use 'cd %s ; source otbenv.profile ; "
"ctest -S share/otb/swig/build_wrapping.cmake -VV'",
prefix,
)
diff --git a/pyotb/install.py b/pyotb/install.py
index ed54a25..76919e2 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -40,7 +40,7 @@ def check_versions(sysname: str, python_minor: int, otb_major: int):
python_minor: minor version of python
otb_major: major version of OTB to be installed
Returns:
- True if requirements are satisfied
+ (True, 0) or (False, expected_version) if case of version conflict
"""
if sysname == "Win64":
expected = 5 if otb_major in (6, 7) else 7
@@ -65,7 +65,7 @@ def env_config_unix(otb_path: Path):
"""
profile = Path.home() / ".profile"
- with open(profile, "a", encoding="utf-8") as buf:
+ with profile.open("a", encoding="utf-8") as buf:
buf.write(f'\n. "{otb_path}/otbenv.profile"\n')
print(f"##### Added new environment variables to {profile}")
@@ -155,18 +155,16 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True):
f"Remember to call '{path}{os.sep}otbenv.{ext}' before importing pyotb, "
f"or add 'OTB_ROOT=\"{path}\"' to your env variables."
)
- # No recompilation or symlink required
+ # Requirements are met, no recompilation or symlink required
if check:
return str(path)
- # Here version check failed, try recompile bindings : can fail because of OpenGL
- suffix = f"so.rh-python3{expected}-1.0" if otb_major < 8 else "so.1.0"
- target_lib = f"{path}/lib/libpython3.{expected}.{suffix}"
+ # Else try recompile bindings : can fail because of OpenGL
if which("ctest") and which("python3-config"):
try:
print("\n!!!!! Python version mismatch, trying to recompile bindings")
ctest_cmd = (
- ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -V"
+ ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -Vv"
)
print(f"##### Executing '{ctest_cmd}'")
subprocess.run(ctest_cmd, cwd=path, check=True, shell=True)
@@ -174,8 +172,8 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True):
except subprocess.CalledProcessError:
print("\nCompilation failed.")
print(
- "You need cmake, python3-dev and libgl1-mesa-dev installed. "
- "Trying to symlink libraries instead - this may fail with newest versions."
+ "You need cmake, python3-dev and libgl1-mesa-dev installed."
+ "\nTrying to symlink libraries instead - this may fail with newest versions."
)
# TODO: support for sudo auto build deps install using apt, pacman/yay, brew...
# Else use dirty cross version python symlink (only tested on Ubuntu)
--
GitLab
From 128794573474a0ce6f4a925b656492c1185729b4 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 13 Aug 2023 10:52:15 +0200
Subject: [PATCH 338/399] FIX: try symlink python libs for executable in
/usr/local/bin
---
pyotb/__init__.py | 2 +-
pyotb/install.py | 11 ++++++++---
2 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index c134e58..aff3736 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
"""This module provides convenient python wrapping of otbApplications."""
-__version__ = "2.0.0.dev5"
+__version__ = "2.0.0.dev6"
from .install import install_otb
from .helpers import logger, set_logger_level
diff --git a/pyotb/install.py b/pyotb/install.py
index 76919e2..07922c1 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -177,10 +177,15 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True):
)
# TODO: support for sudo auto build deps install using apt, pacman/yay, brew...
# Else use dirty cross version python symlink (only tested on Ubuntu)
- if sys.executable.startswith("/usr/bin"):
- lib = f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so"
+ suffix = "so.1.0" if otb_major >= 8 else f"so.rh-python3{expected}-1.0"
+ target_lib = f"{path}/lib/libpython3.{expected}.{suffix}"
+ for prefix in ("/usr", "/usr/local"):
+ if not sys.executable.startswith(f"{prefix}/bin"):
+ continue
+ lib_dir = f"{prefix}/lib" + ("/x86_64-linux-gnu" if prefix == "/usr" else "")
+ lib = f"{lib_dir}/libpython3.{sys.version_info.minor}.so"
if Path(lib).exists():
- print(f"##### Creating symbolic links: {lib} -> {target_lib}")
+ print(f"##### Creating symbolic link: {lib} -> {target_lib}")
ln_cmd = f'ln -sf "{lib}" "{target_lib}"'
subprocess.run(ln_cmd, executable=cmd, shell=True, check=True)
return str(path)
--
GitLab
From 67cd415347c9da6b1100018d7829080e0498e6c3 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 13 Aug 2023 11:42:09 +0200
Subject: [PATCH 339/399] ENH: use sysconfig.get_config_var to locate python
library
---
pyotb/helpers.py | 69 +++++++++++++++++++++++-------------------------
pyotb/install.py | 18 ++++++-------
2 files changed, 41 insertions(+), 46 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index b043f13..3fc93b3 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -3,6 +3,7 @@
import logging
import os
import sys
+import sysconfig
from pathlib import Path
from shutil import which
@@ -291,42 +292,7 @@ def __suggest_fix_import(error_message: str, prefix: str):
"""Help user to fix the OTB installation with appropriate log messages."""
logger.critical("An error occurred while importing OTB Python API")
logger.critical("OTB error message was '%s'", error_message)
- if sys.platform == "linux":
- if error_message.startswith("libpython3."):
- logger.critical(
- "It seems like you need to symlink or recompile python bindings"
- )
- if sys.executable.startswith("/usr/bin"):
- lib = (
- f"/usr/lib/x86_64-linux-gnu/libpython3.{sys.version_info.minor}.so"
- )
- if which("ctest") and which("python3-config"):
- logger.critical(
- "To compile, use 'cd %s ; source otbenv.profile ; "
- "ctest -S share/otb/swig/build_wrapping.cmake -VV'",
- prefix,
- )
- elif Path(lib).exists():
- expected = int(error_message[11])
- if expected != sys.version_info.minor:
- logger.critical(
- "Python library version mismatch (OTB expected 3.%s) : "
- "a symlink may not work, depending on your python version",
- expected,
- )
- target = f"{prefix}/lib/libpython3.{expected}.so.1.0"
- logger.critical("If using OTB>=8.0, use 'ln -s %s %s'", lib, target)
- else:
- logger.critical(
- "You may need to install cmake, python3-dev and mesa's libgl"
- " in order to recompile python bindings"
- )
- else:
- logger.critical(
- "Unable to automatically locate python dynamic library of %s",
- sys.executable,
- )
- elif sys.platform == "win32":
+ if sys.platform == "win32":
if error_message.startswith("DLL load failed"):
if sys.version_info.minor != 7:
logger.critical(
@@ -337,6 +303,37 @@ def __suggest_fix_import(error_message: str, prefix: str):
"It seems that your env variables aren't properly set,"
" first use 'call otbenv.bat' then try to import pyotb once again"
)
+ elif error_message.startswith("libpython3."):
+ logger.critical(
+ "It seems like you need to symlink or recompile python bindings"
+ )
+ if (
+ sys.executable.startswith("/usr/bin")
+ and which("ctest")
+ and which("python3-config")
+ ):
+ logger.critical(
+ "To compile, use 'cd %s ; source otbenv.profile ; "
+ "ctest -S share/otb/swig/build_wrapping.cmake -VV'",
+ prefix,
+ )
+ return
+ logger.critical(
+ "You may need to install cmake, python3-dev and mesa's libgl"
+ " in order to recompile python bindings"
+ )
+ expected = int(error_message[11])
+ if expected != sys.version_info.minor:
+ logger.critical(
+ "Python library version mismatch (OTB expected 3.%s) : "
+ "a symlink may not work, depending on your python version",
+ expected,
+ )
+ lib_dir = sysconfig.get_config_var("LIBDIR")
+ lib = f"{lib_dir}/libpython3.{sys.version_info.minor}.so"
+ if Path(lib).exists():
+ target = f"{prefix}/lib/libpython3.{expected}.so.1.0"
+ logger.critical("If using OTB>=8.0, try 'ln -sf %s %s'", lib, target)
logger.critical(
"You can verify installation requirements for your OS at %s", DOCS_URL
)
diff --git a/pyotb/install.py b/pyotb/install.py
index 07922c1..9de6ac2 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -4,6 +4,7 @@ import os
import re
import subprocess
import sys
+import sysconfig
import tempfile
import urllib.request
import zipfile
@@ -179,16 +180,13 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True):
# Else use dirty cross version python symlink (only tested on Ubuntu)
suffix = "so.1.0" if otb_major >= 8 else f"so.rh-python3{expected}-1.0"
target_lib = f"{path}/lib/libpython3.{expected}.{suffix}"
- for prefix in ("/usr", "/usr/local"):
- if not sys.executable.startswith(f"{prefix}/bin"):
- continue
- lib_dir = f"{prefix}/lib" + ("/x86_64-linux-gnu" if prefix == "/usr" else "")
- lib = f"{lib_dir}/libpython3.{sys.version_info.minor}.so"
- if Path(lib).exists():
- print(f"##### Creating symbolic link: {lib} -> {target_lib}")
- ln_cmd = f'ln -sf "{lib}" "{target_lib}"'
- subprocess.run(ln_cmd, executable=cmd, shell=True, check=True)
- return str(path)
+ lib_dir = sysconfig.get_config_var("LIBDIR")
+ lib = f"{lib_dir}/libpython3.{sys.version_info.minor}.so"
+ if Path(lib).exists():
+ print(f"##### Creating symbolic link: {lib} -> {target_lib}")
+ ln_cmd = f'ln -sf "{lib}" "{target_lib}"'
+ subprocess.run(ln_cmd, executable=cmd, shell=True, check=True)
+ return str(path)
raise SystemExit(
f"Unable to automatically locate library for executable '{sys.executable}', "
f"you could manually create a symlink from that file to {target_lib}"
--
GitLab
From bf468ceda3b26eae246f14c4edc636e3697d0ae5 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Sun, 13 Aug 2023 13:54:03 +0200
Subject: [PATCH 340/399] DOC: comments
---
pyotb/install.py | 18 ++++++++++++------
1 file changed, 12 insertions(+), 6 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 9de6ac2..fe42fd7 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -34,7 +34,7 @@ def otb_latest_release_tag():
def check_versions(sysname: str, python_minor: int, otb_major: int):
- """Verify if python version is compatible with OTB version.
+ """Verify if python version is compatible with major OTB version.
Args:
sysname: OTB's system name convention (Linux64, Darwin64, Win64)
@@ -72,7 +72,7 @@ def env_config_unix(otb_path: Path):
def env_config_windows(otb_path: Path):
- """Update registry hive for current user with new OTB_ROOT env variable.
+ """Update user's registry hive with new OTB_ROOT env variable.
Args:
otb_path: path of the new OTB installation
@@ -161,23 +161,29 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True):
return str(path)
# Else try recompile bindings : can fail because of OpenGL
- if which("ctest") and which("python3-config"):
+ # Here we check for /usr/bin because CMake's will find_package() only there
+ if (
+ sys.executable.startswith("/usr/bin")
+ and which("ctest")
+ and which("python3-config")
+ ):
try:
print("\n!!!!! Python version mismatch, trying to recompile bindings")
ctest_cmd = (
- ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -Vv"
+ ". ./otbenv.profile && ctest -S share/otb/swig/build_wrapping.cmake -V"
)
print(f"##### Executing '{ctest_cmd}'")
subprocess.run(ctest_cmd, cwd=path, check=True, shell=True)
return str(path)
except subprocess.CalledProcessError:
print("\nCompilation failed.")
+ # TODO: support for sudo auto build deps install using apt, pacman/yay, brew...
print(
"You need cmake, python3-dev and libgl1-mesa-dev installed."
"\nTrying to symlink libraries instead - this may fail with newest versions."
)
- # TODO: support for sudo auto build deps install using apt, pacman/yay, brew...
- # Else use dirty cross version python symlink (only tested on Ubuntu)
+
+ # Finally try with cross version python symlink (only tested on Ubuntu)
suffix = "so.1.0" if otb_major >= 8 else f"so.rh-python3{expected}-1.0"
target_lib = f"{path}/lib/libpython3.{expected}.{suffix}"
lib_dir = sysconfig.get_config_var("LIBDIR")
--
GitLab
From f62ec2ff89d05dd2bf2b180652bda3897ec0a651 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 15 Aug 2023 11:40:32 +0200
Subject: [PATCH 341/399] ENH: do not raise SystemExit in interactive mode
---
pyotb/install.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index fe42fd7..47a6ee7 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -193,7 +193,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True):
ln_cmd = f'ln -sf "{lib}" "{target_lib}"'
subprocess.run(ln_cmd, executable=cmd, shell=True, check=True)
return str(path)
- raise SystemExit(
+ raise SystemError(
f"Unable to automatically locate library for executable '{sys.executable}', "
f"you could manually create a symlink from that file to {target_lib}"
)
--
GitLab
From 2a357eaa5ad383e821eb883411d5e4a149ff8cea Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 15 Aug 2023 11:41:05 +0200
Subject: [PATCH 342/399] CI: shorter test job names
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f0846f0..27cb2d6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -84,7 +84,7 @@ test_install:
before_script:
- pip install pytest pytest-cov
-test_module_core:
+module_core:
extends: .tests
variables:
OTB_LOGGER_LEVEL: INFO
@@ -101,7 +101,7 @@ test_module_core:
- curl -fsLI $PLEIADES_IMG_URL
- python3 -m pytest -vv --junitxml=test-module-core.xml --cov-report xml:coverage.xml tests/test_core.py
-test_pipeline_permutations:
+pipeline_permutations:
extends: .tests
variables:
OTB_LOGGER_LEVEL: WARNING
--
GitLab
From 08b9a567dee508666d026d05841398a4a0a925ff Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 16 Aug 2023 10:19:09 +0200
Subject: [PATCH 343/399] CI: remove unused variables
---
.gitlab-ci.yml | 2 --
1 file changed, 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 27cb2d6..48b6a18 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -79,8 +79,6 @@ test_install:
variables:
OTB_ROOT: /opt/otb
LD_LIBRARY_PATH: /opt/otb/lib
- SPOT_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif
- PLEIADES_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Baseline/OTB/Images/prTvOrthoRectification_pleiades-1_noDEM.tif
before_script:
- pip install pytest pytest-cov
--
GitLab
From b0b59dccd6003d35987719d45eed03813fa2160b Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 16 Aug 2023 10:37:04 +0200
Subject: [PATCH 344/399] Revert "CI: remove unused variables"
This reverts commit 08b9a567dee508666d026d05841398a4a0a925ff.
---
.gitlab-ci.yml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 48b6a18..27cb2d6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -79,6 +79,8 @@ test_install:
variables:
OTB_ROOT: /opt/otb
LD_LIBRARY_PATH: /opt/otb/lib
+ SPOT_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif
+ PLEIADES_IMG_URL: https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Baseline/OTB/Images/prTvOrthoRectification_pleiades-1_noDEM.tif
before_script:
- pip install pytest pytest-cov
--
GitLab
From 2e58ff1a0d9fc455e03b3cf82ec49fe361424e28 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 28 Aug 2023 12:19:22 +0200
Subject: [PATCH 345/399] FIX: #116
---
pyotb/core.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index e4ca2b6..4d0ab82 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -856,7 +856,9 @@ class App(OTBObject):
path,
)
elif isinstance(path, (str, Path)) and self.output_key:
- kwargs.update({self.output_key: str(path)})
+ kwargs[self.output_key] = str(path)
+ elif not path and self.output_image_key in self.parameters:
+ kwargs[self.output_key] = self.parameters[self.output_key]
elif path is not None:
raise TypeError(f"{self.name}: unsupported filepath type ({type(path)})")
if not (kwargs or any(k in self._settings for k in self._out_param_types)):
--
GitLab
From 1dc6f9c60fdf9905f8f90289adbe69635b641187 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 28 Aug 2023 12:19:26 +0200
Subject: [PATCH 346/399] FIX: #117
---
pyotb/core.py | 31 ++++++++++++++++++-------------
1 file changed, 18 insertions(+), 13 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 4d0ab82..045721e 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -732,9 +732,10 @@ class App(OTBObject):
)
try:
if self.is_input(key):
- self.__set_param(key, self.__check_input_param(obj))
- else:
- self.__set_param(key, obj)
+ obj = self.__check_input_param(obj)
+ elif self.is_output(key):
+ obj = self.__check_output_param(obj)
+ self.__set_param(key, obj)
except (RuntimeError, TypeError, ValueError, KeyError) as e:
raise RuntimeError(
f"{self.name}: error before execution, while setting parameter '{key}' to '{obj}': {e})"
@@ -972,15 +973,7 @@ class App(OTBObject):
def __check_input_param(
self, obj: list | OTBObject | str | Path
) -> list | OTBObject | str:
- """Check the type and value of an input param.
-
- Args:
- obj: input parameter value
-
- Returns:
- object, string with new /vsi prefix(es) if needed
-
- """
+ """Check the type and value of an input parameter, add vsi prefixes if needed."""
if isinstance(obj, list):
return [self.__check_input_param(o) for o in obj]
# May be we could add some checks here
@@ -1012,8 +1005,20 @@ class App(OTBObject):
return obj
raise TypeError(f"{self.name}: wrong input parameter type ({type(obj)})")
+ def __check_output_param(
+ self, obj: list | OTBObject | str | Path
+ ) -> list | OTBObject | str:
+ """Check the type and value of an output parameter."""
+ if isinstance(obj, list):
+ return [self.__check_output_param(o) for o in obj]
+ if isinstance(obj, Path):
+ obj = str(obj)
+ if isinstance(obj, str):
+ return obj
+ raise TypeError(f"{self.name}: wrong output parameter type ({type(obj)})")
+
def __set_param(
- self, key: str, obj: list | tuple | OTBObject | otb.Application | list[Any]
+ self, key: str, obj: str | float | list | tuple | OTBObject | otb.Application
):
"""Set one parameter, decide which otb.Application method to use depending on target object."""
if obj is None or (isinstance(obj, (list, tuple)) and not obj):
--
GitLab
From 23845245262c82522e4ea1366b5661ec06ffe3e5 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 28 Aug 2023 12:22:52 +0200
Subject: [PATCH 347/399] TERST: add test for output Path object param
---
tests/test_core.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tests/test_core.py b/tests/test_core.py
index d172739..5a32a6f 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -141,6 +141,8 @@ def test_xy_to_rowcol():
def test_write():
assert INPUT.write("/dev/shm/test_write.tif", ext_fname="nodata=0")
INPUT["out"].filepath.unlink()
+ assert INPUT.write(Path("/dev/shm/test_write.tif"), ext_fname="nodata=0")
+ INPUT["out"].filepath.unlink()
# Frozen
frozen_app = pyotb.BandMath(INPUT, exp="im1b1", frozen=True)
assert frozen_app.write("/dev/shm/test_frozen_app_write.tif")
--
GitLab
From 799fe2e61e36872de2d8651994b4c135abde79c0 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 28 Aug 2023 12:27:09 +0200
Subject: [PATCH 348/399] TEST: check output dtype is set in frozen mode
---
tests/test_core.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/tests/test_core.py b/tests/test_core.py
index 5a32a6f..1de0937 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -150,7 +150,8 @@ def test_write():
frozen_app_init_with_outfile = pyotb.BandMath(
INPUT, exp="im1b1", out="/dev/shm/test_frozen_app_write.tif", frozen=True
)
- assert frozen_app_init_with_outfile.write()
+ assert frozen_app_init_with_outfile.write(pixel_type="uint16")
+ assert frozen_app_init_with_outfile.dtype == "uint16"
frozen_app_init_with_outfile["out"].filepath.unlink()
--
GitLab
From 5afdc751a87638a8f639743e2e631527643860ce Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 28 Aug 2023 13:28:52 +0200
Subject: [PATCH 349/399] STYLE: black and docstrings
---
pyotb/core.py | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 045721e..8bb493b 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1005,9 +1005,7 @@ class App(OTBObject):
return obj
raise TypeError(f"{self.name}: wrong input parameter type ({type(obj)})")
- def __check_output_param(
- self, obj: list | OTBObject | str | Path
- ) -> list | OTBObject | str:
+ def __check_output_param(self, obj: list | str | Path) -> list | str:
"""Check the type and value of an output parameter."""
if isinstance(obj, list):
return [self.__check_output_param(o) for o in obj]
--
GitLab
From 6a113ac845c7fa81072d42859021a3f31294f5a3 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 28 Aug 2023 13:39:44 +0200
Subject: [PATCH 350/399] TEST: update tests for new summarize behaviour
---
tests/pipeline_summary.json | 6 +++---
tests/test_core.py | 21 +++++++++++++--------
2 files changed, 16 insertions(+), 11 deletions(-)
diff --git a/tests/pipeline_summary.json b/tests/pipeline_summary.json
index 56ebf8d..443f9ef 100644
--- a/tests/pipeline_summary.json
+++ b/tests/pipeline_summary.json
@@ -30,7 +30,7 @@
"interpolator.bco.radius": 2,
"opt.rpc": 10,
"opt.gridspacing": 4.0,
- "io.in": "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
+ "io.in": "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
}
}
],
@@ -67,7 +67,7 @@
"name": "BandMath",
"parameters": {
"il": [
- "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
+ "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
],
"exp": "im1b1"
}
@@ -105,7 +105,7 @@
"name": "BandMath",
"parameters": {
"il": [
- "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
+ "/vsicurl/https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/raw/develop/Data/Input/SP67_FR_subset_1.tif"
],
"exp": "im1b1"
}
diff --git a/tests/test_core.py b/tests/test_core.py
index 1de0937..fc4712c 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -7,7 +7,6 @@ from tests_data import *
def test_app_parameters():
# Input / ExtractROI
assert INPUT.parameters
- assert INPUT.parameters["in"] == SPOT_IMG_URL
assert (INPUT.parameters["sizex"], INPUT.parameters["sizey"]) == (251, 304)
# OrthoRectification
app = pyotb.OrthoRectification(INPUT)
@@ -53,19 +52,25 @@ def test_app_input_vsi():
pyotb.Input("/vsicurl/" + SPOT_IMG_URL)
# Simple remote file
info = pyotb.ReadImageInfo("https://fake.com/image.tif", frozen=True)
- assert info.app.GetParameterValue("in") == "/vsicurl/https://fake.com/image.tif"
- assert info.parameters["in"] == "https://fake.com/image.tif"
+ assert (
+ info.app.GetParameterValue("in")
+ == info.parameters["in"]
+ == "/vsicurl/https://fake.com/image.tif"
+ )
# Compressed single file archive
info = pyotb.ReadImageInfo("image.tif.zip", frozen=True)
- assert info.app.GetParameterValue("in") == "/vsizip/image.tif.zip"
- assert info.parameters["in"] == "image.tif.zip"
+ assert (
+ info.app.GetParameterValue("in")
+ == info.parameters["in"]
+ == "/vsizip/image.tif.zip"
+ )
# File within compressed remote archive
info = pyotb.ReadImageInfo("https://fake.com/archive.tar.gz/image.tif", frozen=True)
assert (
info.app.GetParameterValue("in")
+ == info.parameters["in"]
== "/vsitar//vsicurl/https://fake.com/archive.tar.gz/image.tif"
)
- assert info.parameters["in"] == "https://fake.com/archive.tar.gz/image.tif"
# Piped curl --> zip --> tiff
ziped_tif_urls = (
"https://github.com/OSGeo/gdal/raw/master"
@@ -348,8 +353,8 @@ def test_summarize_output_obj():
def test_summarize_strip_output():
- in_fn = SPOT_IMG_URL
- in_fn_w_ext = SPOT_IMG_URL + "?&skipcarto=1"
+ in_fn = "/vsicurl/" + SPOT_IMG_URL
+ in_fn_w_ext = "/vsicurl/" + SPOT_IMG_URL + "?&skipcarto=1"
out_fn = "/dev/shm/output.tif"
out_fn_w_ext = out_fn + "?&box=10:10:10:10"
--
GitLab
From d2043b98148b7ef310afe0d2f651ea0dad053456 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 11 Oct 2023 11:50:32 +0200
Subject: [PATCH 351/399] ENH: ensure Input object init is quiet
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 8bb493b..db05bed 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1521,7 +1521,7 @@ class Input(App):
filepath: Anything supported by GDAL (local file on the filesystem, remote resource e.g. /vsicurl/.., etc.)
"""
- super().__init__("ExtractROI", {"in": filepath}, frozen=True)
+ super().__init__("ExtractROI", {"in": filepath}, quiet=True, frozen=True)
self._name = f"Input from {filepath}"
if not filepath.startswith(("/vsi", "http://", "https://", "ftp://")):
filepath = Path(filepath)
--
GitLab
From aaf8b9e1d300f319f14013158127ded94c79a456 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 11 Oct 2023 11:55:59 +0200
Subject: [PATCH 352/399] ENH: use attr output_image_key in write()
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index db05bed..15d021c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -859,7 +859,7 @@ class App(OTBObject):
elif isinstance(path, (str, Path)) and self.output_key:
kwargs[self.output_key] = str(path)
elif not path and self.output_image_key in self.parameters:
- kwargs[self.output_key] = self.parameters[self.output_key]
+ kwargs[self.output_key] = self.parameters[self.output_image_key]
elif path is not None:
raise TypeError(f"{self.name}: unsupported filepath type ({type(path)})")
if not (kwargs or any(k in self._settings for k in self._out_param_types)):
--
GitLab
From def8449334668c26c25df74041a22b67ff9308e8 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 11 Oct 2023 11:57:22 +0200
Subject: [PATCH 353/399] ENH: use output_key to make it work with non image
output
---
pyotb/core.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 15d021c..aa5f52e 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -858,8 +858,8 @@ class App(OTBObject):
)
elif isinstance(path, (str, Path)) and self.output_key:
kwargs[self.output_key] = str(path)
- elif not path and self.output_image_key in self.parameters:
- kwargs[self.output_key] = self.parameters[self.output_image_key]
+ elif not path and self.output_key in self.parameters:
+ kwargs[self.output_key] = self.parameters[self.output_key]
elif path is not None:
raise TypeError(f"{self.name}: unsupported filepath type ({type(path)})")
if not (kwargs or any(k in self._settings for k in self._out_param_types)):
--
GitLab
From 0404eeb53c8226797b409f3874f15ea2b9f2d02d Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 11 Oct 2023 12:00:41 +0200
Subject: [PATCH 354/399] STYLE: typo
---
pyotb/helpers.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 5363937..6b53c2b 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -336,5 +336,5 @@ def __suggest_fix_import(error_message: str, prefix: str):
)
-# Since helpers is the first module to be inititialized, this will prevent pyotb to run if OTB is not found
+# Since helpers is the first module to be initialized, this will prevent pyotb to run if OTB is not found
find_otb()
--
GitLab
From b4b282dd5e2c28172ea6186bf48dcd5146b88d58 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Wed, 11 Oct 2023 12:18:13 +0200
Subject: [PATCH 355/399] CI: bump dev version
---
pyotb/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index aff3736..40e6434 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
"""This module provides convenient python wrapping of otbApplications."""
-__version__ = "2.0.0.dev6"
+__version__ = "2.0.0.dev7"
from .install import install_otb
from .helpers import logger, set_logger_level
--
GitLab
From 291c89648c5204e65e1567e99c522ec848a0fc63 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Tue, 24 Oct 2023 14:56:26 +0000
Subject: [PATCH 356/399] Remove duplicated execution in case of failure, but
ensure execution in case of multiple outputs
---
pyotb/__init__.py | 2 +-
pyotb/core.py | 22 +++++++++-------------
tests/test_core.py | 44 ++++++++++++++++++++++++++++++++++++--------
3 files changed, 46 insertions(+), 22 deletions(-)
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index 40e6434..5e74f37 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
"""This module provides convenient python wrapping of otbApplications."""
-__version__ = "2.0.0.dev7"
+__version__ = "2.0.0.dev8"
from .install import install_otb
from .helpers import logger, set_logger_level
diff --git a/pyotb/core.py b/pyotb/core.py
index aa5f52e..d05c7d5 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -626,6 +626,10 @@ class App(OTBObject):
raise KeyError(f"key {key} not found in the application parameters types")
return self._all_param_types[key] in param_types
+ def __is_multi_output(self):
+ """Check if app has multiple outputs to ensure execution during write()."""
+ return len(self.outputs) > 1
+
def is_input(self, key: str) -> bool:
"""Returns True if the key is an input.
@@ -803,18 +807,8 @@ class App(OTBObject):
def flush(self):
"""Flush data to disk, this is when WriteOutput is actually called."""
- try:
- logger.debug("%s: flushing data to disk", self.name)
- self.app.WriteOutput()
- except RuntimeError:
- logger.debug(
- "%s: failed with WriteOutput, executing once again with ExecuteAndWriteOutput",
- self.name,
- )
- self._time_start = perf_counter()
- self.app.ExecuteAndWriteOutput()
- self.__sync_parameters()
- self.frozen = False
+ logger.debug("%s: flushing data to disk", self.name)
+ self.app.WriteOutput()
self._time_end = perf_counter()
@deprecated_alias(filename_extension="ext_fname")
@@ -927,11 +921,13 @@ class App(OTBObject):
if key in data_types:
self.propagate_dtype(key, data_types[key])
self.set_parameters({key: filepath})
- if self.frozen:
+ # TODO: drop multioutput special case when fixed on the OTB side. See discussion in MR !102
+ if self.frozen or self.__is_multi_output():
self.execute()
self.flush()
if not parameters:
return True
+
# Search and log missing files
files, missing = [], []
for key, filepath in parameters.items():
diff --git a/tests/test_core.py b/tests/test_core.py
index fc4712c..8b3f8fa 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -160,6 +160,31 @@ def test_write():
frozen_app_init_with_outfile["out"].filepath.unlink()
+def test_write_multi_output():
+ mss = pyotb.MeanShiftSmoothing(
+ SPOT_IMG_URL,
+ fout="/dev/shm/test_ext_fn_fout.tif",
+ foutpos="/dev/shm/test_ext_fn_foutpos.tif",
+ )
+
+ mss = pyotb.MeanShiftSmoothing(SPOT_IMG_URL)
+ assert mss.write(
+ {
+ "fout": "/dev/shm/test_ext_fn_fout.tif",
+ "foutpos": "/dev/shm/test_ext_fn_foutpos.tif",
+ },
+ ext_fname={"nodata": 0, "gdal:co:COMPRESS": "DEFLATE"},
+ )
+
+ dr = pyotb.DimensionalityReduction(
+ SPOT_IMG_URL, out="/dev/shm/1.tif", outinv="/dev/shm/2.tif"
+ )
+ dr = pyotb.DimensionalityReduction(SPOT_IMG_URL)
+ assert dr.write(
+ {"out": "/dev/shm/1.tif", "outinv": "/dev/shm/2.tif"}
+ )
+
+
def test_write_ext_fname():
def _check(expected: str, key: str = "out", app=INPUT.app):
fn = app.GetParameterString(key)
@@ -195,18 +220,21 @@ def test_write_ext_fname():
_check("nodata=0&gdal:co:COMPRESS=DEFLATE&box=0:0:10:10")
INPUT["out"].filepath.unlink()
- mss = pyotb.MeanShiftSmoothing(INPUT)
- mss.write(
+ mmsd = pyotb.MorphologicalMultiScaleDecomposition(INPUT)
+ mmsd.write(
{
- "fout": "/dev/shm/test_ext_fn_fout.tif?&nodata=1",
- "foutpos": "/dev/shm/test_ext_fn_foutpos.tif?&nodata=2",
+ "outconvex": "/dev/shm/outconvex.tif?&nodata=1",
+ "outconcave": "/dev/shm/outconcave.tif?&nodata=2",
+ "outleveling": "/dev/shm/outleveling.tif?&nodata=3",
},
ext_fname={"nodata": 0, "gdal:co:COMPRESS": "DEFLATE"},
)
- _check("nodata=1&gdal:co:COMPRESS=DEFLATE", key="fout", app=mss.app)
- _check("nodata=2&gdal:co:COMPRESS=DEFLATE", key="foutpos", app=mss.app)
- mss["fout"].filepath.unlink()
- mss["foutpos"].filepath.unlink()
+ _check("nodata=1&gdal:co:COMPRESS=DEFLATE", key="outconvex", app=mmsd.app)
+ _check("nodata=2&gdal:co:COMPRESS=DEFLATE", key="outconcave", app=mmsd.app)
+ _check("nodata=3&gdal:co:COMPRESS=DEFLATE", key="outleveling", app=mmsd.app)
+ mmsd["outconvex"].filepath.unlink()
+ mmsd["outconcave"].filepath.unlink()
+ mmsd["outleveling"].filepath.unlink()
def test_output():
--
GitLab
From dfe07485adf004d421eb5ee507ea7bd0bf5b5772 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 24 Oct 2023 18:47:58 +0200
Subject: [PATCH 357/399] DOC: update release notes
---
RELEASE_NOTES.txt | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index 3086078..746a2a6 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -1,3 +1,25 @@
+---------------------------------------------------------------------
+2.00 (Oct XX, 2023) - Changes since version 1.5.4
+
+- Major refactoring (see troubleshooting/migration)
+- Pythonic extended filenames (can use dict, etc)
+- Easy access to image metadata
+- CI improvements (tests, coverage, doc, etc)
+- Documentation improvement
+- Code format
+- Allow OTB dotted parameters in kwargs
+- Easy access to pixel coordinates
+- Add function to transform x,y coordinates into row, col
+- Native support of vsicurl inputs
+- Fixes in `summarize()`
+- Fixes in `shape`
+- Add typing to function defs to enhance documentation
+
+---------------------------------------------------------------------
+1.5.4 (Oct 01, 2022) - Changes since version 1.5.3
+
+- Fix slicer wrong end of slicing
+
---------------------------------------------------------------------
1.5.3 (Sep 29, 2022) - Changes since version 1.5.2
--
GitLab
From 74473e8440f37c04ad061a585ddf55fc7c9c0e2c Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Nov 2023 16:57:27 +0100
Subject: [PATCH 358/399] ENH: remove useless encoding declaration
---
pyotb/apps.py | 1 -
pyotb/core.py | 1 -
pyotb/functions.py | 1 -
pyotb/helpers.py | 1 -
4 files changed, 4 deletions(-)
diff --git a/pyotb/apps.py b/pyotb/apps.py
index cd0c696..e5523a2 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
"""Search for OTB (set env if necessary), subclass core.App for each available application."""
from __future__ import annotations
diff --git a/pyotb/core.py b/pyotb/core.py
index d05c7d5..6f8c24c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
"""This module is the core of pyotb."""
from __future__ import annotations
diff --git a/pyotb/functions.py b/pyotb/functions.py
index 96431cd..31e366f 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
"""This module provides a set of functions for pyotb."""
from __future__ import annotations
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index 3fc93b3..b6c09b3 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
"""This module ensure we properly initialize pyotb, or raise SystemExit in case of broken install."""
import logging
import os
--
GitLab
From 50a51fd20e9d05b14816c6d520520a2ea58f4983 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Nov 2023 16:58:46 +0100
Subject: [PATCH 359/399] BUG: remove corner case for OTB < 7.4
---
pyotb/functions.py | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git a/pyotb/functions.py b/pyotb/functions.py
index 31e366f..5eff2d4 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -457,12 +457,10 @@ def define_processing_area(
# It should replace any 'outside' pixel by some NoData -> add `fillvalue` argument in the function
# Applying this bounding box to all inputs
+ bounds = (ulx, uly, lrx, lry)
logger.info(
"Cropping all images to extent Upper Left (%s, %s), Lower Right (%s, %s)",
- ulx,
- uly,
- lrx,
- lry,
+ *bounds,
)
new_inputs = []
for inp in inputs:
@@ -472,11 +470,11 @@ def define_processing_area(
"mode": "extent",
"mode.extent.unit": "phy",
"mode.extent.ulx": ulx,
- "mode.extent.uly": lry, # bug in OTB <= 7.3 :
+ "mode.extent.uly": uly,
"mode.extent.lrx": lrx,
- "mode.extent.lry": uly, # ULY/LRY are inverted
+ "mode.extent.lry": lry,
}
- new_input = App("ExtractROI", params)
+ new_input = App("ExtractROI", params, quiet=True)
# TODO: OTB 7.4 fixes this bug, how to handle different versions of OTB?
new_inputs.append(new_input)
# Potentially update the reference inputs for later resampling
--
GitLab
From e670786a81614e0596ba9b722a31f68bd3606b70 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Nov 2023 16:59:25 +0100
Subject: [PATCH 360/399] ENH: update functions.py docstrings and light refac
---
pyotb/functions.py | 151 ++++++++++++++++++++-------------------------
1 file changed, 66 insertions(+), 85 deletions(-)
diff --git a/pyotb/functions.py b/pyotb/functions.py
index 5eff2d4..ad2f24c 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -8,25 +8,28 @@ import sys
import textwrap
import uuid
from collections import Counter
+from pathlib import Path
from .core import App, Input, LogicalOperation, Operation, get_nbchannels
from .helpers import logger
-def where(
- cond: App | str, x: App | str | int | float, y: App | str | int | float
-) -> Operation:
+def where(cond: App | str, x: App | str | float, y: App | str | float) -> Operation:
"""Functionally similar to numpy.where. Where cond is True (!=0), returns x. Else returns y.
+ If cond is monoband whereas x or y are multiband, cond channels are expanded to match x & y ones.
+
Args:
- cond: condition, must be a raster (filepath, App, Operation...). If cond is monoband whereas x or y are
- multiband, cond channels are expanded to match x & y ones.
+ cond: condition, must be a raster (filepath, App, Operation...).
x: value if cond is True. Can be: float, int, App, filepath, Operation...
y: value if cond is False. Can be: float, int, App, filepath, Operation...
Returns:
an output where pixels are x if cond is True, else y
+ Raises:
+ ValueError: if x and y have different number of bands
+
"""
# Checking the number of bands of rasters. Several cases :
# - if cond is monoband, x and y can be multibands. Then cond will adapt to match x and y nb of bands
@@ -61,19 +64,13 @@ def where(
" of channels of condition to match the number of channels of X/Y",
x_or_y_nb_channels,
)
-
# Get the number of bands of the result
- if x_or_y_nb_channels: # if X or Y is a raster
- out_nb_channels = x_or_y_nb_channels
- else: # if only cond is a raster
- out_nb_channels = cond_nb_channels
+ out_nb_channels = x_or_y_nb_channels or cond_nb_channels
return Operation("?", cond, x, y, nb_bands=out_nb_channels)
-def clip(
- image: App | str, v_min: App | str | int | float, v_max: App | str | int | float
-):
+def clip(image: App | str, v_min: App | str | float, v_max: App | str | float):
"""Clip values of image in a range of values.
Args:
@@ -85,19 +82,18 @@ def clip(
raster whose values are clipped in the range
"""
- if isinstance(image, str):
+ if isinstance(image, (str, Path)):
image = Input(image)
- res = where(image <= v_min, v_min, where(image >= v_max, v_max, image))
- return res
+ return where(image <= v_min, v_min, where(image >= v_max, v_max, image))
def all(*inputs): # pylint: disable=redefined-builtin
"""Check if value is different than 0 everywhere along the band axis.
- For only one image, this function checks that all bands of the image are True (i.e. !=0) and outputs
- a singleband boolean raster
+ For only one image, this function checks that all bands of the image are True (i.e. !=0)
+ and outputs a singleband boolean raster
For several images, this function checks that all images are True (i.e. !=0) and outputs
- a boolean raster, with as many bands as the inputs
+ a boolean raster, with as many bands as the inputs
Args:
inputs: inputs can be 1) a single image or 2) several images, either passed as separate arguments
@@ -132,18 +128,18 @@ def all(*inputs): # pylint: disable=redefined-builtin
res = res & inp[:, :, band]
else:
res = res & (inp[:, :, band] != 0)
+ return res
+
# Checking that all images are True
+ if isinstance(inputs[0], LogicalOperation):
+ res = inputs[0]
else:
- if isinstance(inputs[0], LogicalOperation):
- res = inputs[0]
+ res = inputs[0] != 0
+ for inp in inputs[1:]:
+ if isinstance(inp, LogicalOperation):
+ res = res & inp
else:
- res = inputs[0] != 0
- for inp in inputs[1:]:
- if isinstance(inp, LogicalOperation):
- res = res & inp
- else:
- res = res & (inp != 0)
-
+ res = res & (inp != 0)
return res
@@ -158,6 +154,7 @@ def any(*inputs): # pylint: disable=redefined-builtin
Args:
inputs: inputs can be 1) a single image or 2) several images, either passed as separate arguments
or inside a list
+
Returns:
OR intersection
@@ -188,19 +185,18 @@ def any(*inputs): # pylint: disable=redefined-builtin
res = res | inp[:, :, band]
else:
res = res | (inp[:, :, band] != 0)
+ return res
# Checking that at least one image is True
+ if isinstance(inputs[0], LogicalOperation):
+ res = inputs[0]
else:
- if isinstance(inputs[0], LogicalOperation):
- res = inputs[0]
+ res = inputs[0] != 0
+ for inp in inputs[1:]:
+ if isinstance(inp, LogicalOperation):
+ res = res | inp
else:
- res = inputs[0] != 0
- for inp in inputs[1:]:
- if isinstance(inp, LogicalOperation):
- res = res | inp
- else:
- res = res | (inp != 0)
-
+ res = res | (inp != 0)
return res
@@ -224,17 +220,19 @@ def run_tf_function(func):
Returns:
wrapper: a function that returns a pyotb object
+ Raises:
+ SystemError: if OTBTF apps are missing
+
"""
try:
from .apps import ( # pylint: disable=import-outside-toplevel
TensorflowModelServe,
)
- except ImportError:
- logger.error(
+ except ImportError as err:
+ raise SystemError(
"Could not run Tensorflow function: failed to import TensorflowModelServe."
"Check that you have OTBTF configured (https://github.com/remicres/otbtf#how-to-install)"
- )
- raise
+ ) from err
def get_tf_pycmd(output_dir, channels, scalar_inputs):
"""Create a string containing all python instructions necessary to create and save the Keras model.
@@ -301,13 +299,13 @@ def run_tf_function(func):
raster_inputs = []
for inp in inputs:
try:
- # this is for raster input
+ # This is for raster input
channel = get_nbchannels(inp)
channels.append(channel)
scalar_inputs.append(None)
raster_inputs.append(inp)
except TypeError:
- # this is for other inputs (float, int)
+ # This is for other inputs (float, int)
channels.append(None)
scalar_inputs.append(inp)
@@ -316,6 +314,7 @@ def run_tf_function(func):
out_savedmodel = os.path.join(tmp_dir, f"tmp_otbtf_model_{uuid.uuid4()}")
pycmd = get_tf_pycmd(out_savedmodel, channels, scalar_inputs)
cmd_args = [sys.executable, "-c", pycmd]
+ # TODO: remove subprocess execution since this issues has been fixed with OTBTF 4.0
try:
subprocess.run(
cmd_args,
@@ -407,41 +406,25 @@ def define_processing_area(
# TODO: there seems to have a bug, ImageMetaData is not updated when running an app,
# cf https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2234. Should we use ImageOrigin instead?
if not all(
- metadata["UpperLeftCorner"] == any_metadata["UpperLeftCorner"]
- and metadata["LowerRightCorner"] == any_metadata["LowerRightCorner"]
- for metadata in metadatas.values()
+ md["UpperLeftCorner"] == any_metadata["UpperLeftCorner"]
+ and md["LowerRightCorner"] == any_metadata["LowerRightCorner"]
+ for md in metadatas.values()
):
# Retrieving the bounding box that will be common for all inputs
if window_rule == "intersection":
# The coordinates depend on the orientation of the axis of projection
if any_metadata["GeoTransform"][1] >= 0:
- ulx = max(
- metadata["UpperLeftCorner"][0] for metadata in metadatas.values()
- )
- lrx = min(
- metadata["LowerRightCorner"][0] for metadata in metadatas.values()
- )
+ ulx = max(md["UpperLeftCorner"][0] for md in metadatas.values())
+ lrx = min(md["LowerRightCorner"][0] for md in metadatas.values())
else:
- ulx = min(
- metadata["UpperLeftCorner"][0] for metadata in metadatas.values()
- )
- lrx = max(
- metadata["LowerRightCorner"][0] for metadata in metadatas.values()
- )
+ ulx = min(md["UpperLeftCorner"][0] for md in metadatas.values())
+ lrx = max(md["LowerRightCorner"][0] for md in metadatas.values())
if any_metadata["GeoTransform"][-1] >= 0:
- lry = min(
- metadata["LowerRightCorner"][1] for metadata in metadatas.values()
- )
- uly = max(
- metadata["UpperLeftCorner"][1] for metadata in metadatas.values()
- )
+ lry = min(md["LowerRightCorner"][1] for md in metadatas.values())
+ uly = max(md["UpperLeftCorner"][1] for md in metadatas.values())
else:
- lry = max(
- metadata["LowerRightCorner"][1] for metadata in metadatas.values()
- )
- uly = min(
- metadata["UpperLeftCorner"][1] for metadata in metadatas.values()
- )
+ lry = max(md["LowerRightCorner"][1] for md in metadatas.values())
+ uly = min(md["UpperLeftCorner"][1] for md in metadatas.values())
elif window_rule == "same_as_input":
ulx = metadatas[reference_window_input]["UpperLeftCorner"][0]
@@ -449,12 +432,12 @@ def define_processing_area(
lry = metadatas[reference_window_input]["LowerRightCorner"][1]
uly = metadatas[reference_window_input]["UpperLeftCorner"][1]
elif window_rule == "specify":
- pass
- # TODO : it is when the user explicitly specifies the bounding box -> add some arguments in the function
+ # TODO : when the user explicitly specifies the bounding box -> add some arguments in the function
+ ...
elif window_rule == "union":
- pass
- # TODO : it is when the user wants the final bounding box to be the union of all bounding box
+ # TODO : when the user wants the final bounding box to be the union of all bounding box
# It should replace any 'outside' pixel by some NoData -> add `fillvalue` argument in the function
+ ...
# Applying this bounding box to all inputs
bounds = (ulx, uly, lrx, lry)
@@ -478,16 +461,14 @@ def define_processing_area(
# TODO: OTB 7.4 fixes this bug, how to handle different versions of OTB?
new_inputs.append(new_input)
# Potentially update the reference inputs for later resampling
- if str(inp) == str(
- reference_pixel_size_input
- ): # we use comparison of string because calling '=='
+ if str(inp) == str(reference_pixel_size_input):
+ # We use comparison of string because calling '=='
# on pyotb objects implicitly calls BandMathX application, which is not desirable
reference_pixel_size_input = new_input
- except RuntimeError as e:
- logger.error(
- "Cannot define the processing area for input %s: %s", inp, e
- )
- raise
+ except RuntimeError as err:
+ raise ValueError(
+ f"Cannot define the processing area for input {inp}"
+ ) from err
inputs = new_inputs
# Update metadatas
metadatas = {input: input.app.GetImageMetaData("out") for input in inputs}
@@ -496,9 +477,9 @@ def define_processing_area(
any_metadata = next(iter(metadatas.values()))
# Handling different pixel sizes
if not all(
- metadata["GeoTransform"][1] == any_metadata["GeoTransform"][1]
- and metadata["GeoTransform"][5] == any_metadata["GeoTransform"][5]
- for metadata in metadatas.values()
+ md["GeoTransform"][1] == any_metadata["GeoTransform"][1]
+ and md["GeoTransform"][5] == any_metadata["GeoTransform"][5]
+ for md in metadatas.values()
):
# Retrieving the pixel size that will be common for all inputs
if pixel_size_rule == "minimal":
--
GitLab
From e277304176fc09824667ceec2620651bbde51f0d Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Nov 2023 16:59:54 +0100
Subject: [PATCH 361/399] ENH: rename pyOTB logger to just pyotb
---
pyotb/helpers.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index b6c09b3..e0feb63 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -15,13 +15,13 @@ DOCS_URL = "https://www.orfeo-toolbox.org/CookBook/Installation.html"
# Logging
# User can also get logger with `logging.getLogger("pyOTB")`
# then use pyotb.set_logger_level() to adjust logger verbosity
-logger = logging.getLogger("pyOTB")
+logger = logging.getLogger("pyotb")
logger_handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter(
- fmt="%(asctime)s (%(levelname)-4s) [pyOTB] %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
+ fmt="%(asctime)s (%(levelname)-4s) [pyotb] %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
)
logger_handler.setFormatter(formatter)
-# Search for PYOTB_LOGGER_LEVEL, else use OTB_LOGGER_LEVEL as pyOTB level, or fallback to INFO
+# Search for PYOTB_LOGGER_LEVEL, else use OTB_LOGGER_LEVEL as pyotb level, or fallback to INFO
LOG_LEVEL = (
os.environ.get("PYOTB_LOGGER_LEVEL") or os.environ.get("OTB_LOGGER_LEVEL") or "INFO"
)
--
GitLab
From a21aa4dfbcb7212184c3226652c36d5dc1e2abb7 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Nov 2023 17:01:34 +0100
Subject: [PATCH 362/399] ENH: refactor duplicated function in Operation
---
pyotb/core.py | 69 ++++++++++++++++++++++-----------------------------
1 file changed, 30 insertions(+), 39 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 6f8c24c..d56e9e9 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1266,6 +1266,32 @@ class Operation(App):
appname, il=self.unique_inputs, exp=self.exp, quiet=True, name=name
)
+ def get_nb_bands(self, inputs: list[OTBObject | str | int | float]) -> int:
+ """Guess the number of bands of the output image, from the inputs.
+
+ Args:
+ inputs: the Operation operands
+
+ Raises:
+ ValueError: if all inputs don't have the same number of bands
+
+ """
+ if any(
+ isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced")
+ for inp in inputs
+ ):
+ return 1
+ # Check that all inputs have the same band count
+ nb_bands_list = [
+ get_nbchannels(inp)
+ for inp in inputs
+ if not isinstance(inp, (float, int))
+ ]
+ all_same = all(x == nb_bands_list[0] for x in nb_bands_list)
+ if len(nb_bands_list) > 1 and not all_same:
+ raise ValueError("All images do not have the same number of bands")
+ return nb_bands_list[0]
+
def build_fake_expressions(
self,
operator: str,
@@ -1285,29 +1311,12 @@ class Operation(App):
self.inputs.clear()
self.nb_channels.clear()
logger.debug("%s, %s", operator, inputs)
- # This is when we use the ternary operator with `pyotb.where` function. The output nb of bands is already known
+ # When we use the ternary operator with `pyotb.where` function, the output nb of bands is already known
if operator == "?" and nb_bands:
pass
# For any other operations, the output number of bands is the same as inputs
else:
- if any(
- isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced")
- for inp in inputs
- ):
- nb_bands = 1
- else:
- nb_bands_list = [
- get_nbchannels(inp)
- for inp in inputs
- if not isinstance(inp, (float, int))
- ]
- # check that all inputs have the same nb of bands
- if len(nb_bands_list) > 1 and not all(
- x == nb_bands_list[0] for x in nb_bands_list
- ):
- raise ValueError("All images do not have the same number of bands")
- nb_bands = nb_bands_list[0]
-
+ nb_bands = self.get_nb_bands(inputs)
# Create a list of fake expressions, each item of the list corresponding to one band
self.fake_exp_bands.clear()
for i, band in enumerate(range(1, nb_bands + 1)):
@@ -1463,29 +1472,11 @@ class LogicalOperation(Operation):
Args:
operator: str (one of >, <, >=, <=, ==, !=, &, |)
inputs: Can be OTBObject, filepath, int or float
- nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
+ nb_bands: optionnaly specify the output nb of bands - used only internally by pyotb.where
"""
- # For any other operations, the output number of bands is the same as inputs
- if any(
- isinstance(inp, Slicer) and hasattr(inp, "one_band_sliced")
- for inp in inputs
- ):
- nb_bands = 1
- else:
- nb_bands_list = [
- get_nbchannels(inp)
- for inp in inputs
- if not isinstance(inp, (float, int))
- ]
- # check that all inputs have the same nb of bands
- if len(nb_bands_list) > 1 and not all(
- x == nb_bands_list[0] for x in nb_bands_list
- ):
- raise ValueError("All images do not have the same number of bands")
- nb_bands = nb_bands_list[0]
# Create a list of fake exp, each item of the list corresponding to one band
- for i, band in enumerate(range(1, nb_bands + 1)):
+ for i, band in enumerate(range(1, self.get_nb_bands(inputs) + 1)):
expressions = []
for inp in inputs:
fake_exp, corresp_inputs, nb_channels = super().make_fake_exp(
--
GitLab
From 04d3c20d8441c637d6cbe5801a139b5f52670926 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Nov 2023 17:02:55 +0100
Subject: [PATCH 363/399] ENH: add more exceptions to prevent errors during
__set_param
---
pyotb/core.py | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index d56e9e9..2a385ca 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1043,9 +1043,18 @@ class App(OTBObject):
else: # here `input` should be an image filepath
# Append `input` to the list, do not overwrite any previously set element of the image list
self.app.AddParameterStringList(key, inp)
+ else:
+ raise TypeError(
+ f"{self.name}: wrong input parameter type ({type(inp)})"
+ f" found in '{key}' list: {inp}"
+ )
# List of any other types (str, int...)
- else:
+ elif self.is_key_list(key):
self.app.SetParameterValue(key, obj)
+ else:
+ raise TypeError(
+ f"{self.name}: wrong input parameter type ({type(obj)}) for '{key}'"
+ )
def __sync_parameters(self):
"""Save app parameters in _auto_parameters or data dict.
--
GitLab
From 9bd6c3e5f19d55f10a78ebb2d34146bcfbbad4f3 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Nov 2023 17:11:05 +0100
Subject: [PATCH 364/399] DOC: update or enh comments and docstrings, add
"Raises", fix type hints
---
pyotb/apps.py | 18 +--
pyotb/core.py | 301 ++++++++++++++++++++++--------------------
pyotb/depreciation.py | 10 +-
pyotb/functions.py | 17 +--
pyotb/helpers.py | 22 ++-
pyotb/install.py | 9 +-
6 files changed, 202 insertions(+), 175 deletions(-)
diff --git a/pyotb/apps.py b/pyotb/apps.py
index e5523a2..45e49c4 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -12,12 +12,12 @@ from .helpers import logger
def get_available_applications() -> tuple[str]:
"""Find available OTB applications.
- Args:
- as_subprocess: indicate if function should list available applications using subprocess call
-
Returns:
tuple of available applications
+ Raises:
+ SystemExit: if no application is found
+
"""
app_list = otb.Registry.GetAvailableApplications()
if app_list:
@@ -29,7 +29,7 @@ def get_available_applications() -> tuple[str]:
class OTBTFApp(App):
- """Helper for OTBTF."""
+ """Helper for OTBTF to ensure the nb_sources variable is set."""
@staticmethod
def set_nb_sources(*args, n_sources: int = None):
@@ -59,10 +59,8 @@ class OTBTFApp(App):
Args:
name: name of the OTBTF app
- *args: arguments (dict). NB: we don't need kwargs because it cannot contain source#.il
n_sources: number of sources. Default is None (resolves the number of sources based on the
content of the dict passed in args, where some 'source' str is found)
- **kwargs: kwargs
"""
self.set_nb_sources(*args, n_sources=n_sources)
@@ -71,16 +69,12 @@ class OTBTFApp(App):
AVAILABLE_APPLICATIONS = get_available_applications()
-# This is to enable aliases of Apps, i.e. using apps like `pyotb.AppName(...)` instead of `pyotb.App("AppName", ...)`
-_CODE_TEMPLATE = (
- """
+# This is to enable aliases of Apps, i.e. `pyotb.AppName(...)` instead of `pyotb.App("AppName", ...)`
+_CODE_TEMPLATE = """
class {name}(App):
- """
- """
def __init__(self, *args, **kwargs):
super().__init__('{name}', *args, **kwargs)
"""
-)
for _app in AVAILABLE_APPLICATIONS:
# Customize the behavior for some OTBTF applications. `OTB_TF_NSOURCES` is now handled by pyotb
diff --git a/pyotb/core.py b/pyotb/core.py
index 2a385ca..0f27547 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -21,12 +21,12 @@ class OTBObject(ABC):
@property
@abstractmethod
def name(self) -> str:
- """By default, should return the application name, but a custom name may be passed during init."""
+ """Application name by default, but a custom name may be passed during init."""
@property
@abstractmethod
def app(self) -> otb.Application:
- """Reference to the main (or last in pipeline) otb.Application instance linked to this object."""
+ """Reference to the otb.Application instance linked to this object."""
@property
@abstractmethod
@@ -41,11 +41,11 @@ class OTBObject(ABC):
@property
@abstractmethod
def exports_dic(self) -> dict[str, dict]:
- """Return an internal dict object containing np.array exports, to avoid duplicated ExportImage() calls."""
+ """Ref to an internal dict of np.array exports, to avoid duplicated ExportImage()."""
@property
def metadata(self) -> dict[str, (str, float, list[float])]:
- """Return metadata.
+ """Return image metadata as dictionary.
The returned dict results from the concatenation of the first output
image metadata dictionary and the metadata dictionary.
@@ -61,10 +61,7 @@ class OTBObject(ABC):
if getattr(otb_imd, "has")(key)
}
- # Metadata dictionary
- # Replace items like {"metadata_1": "TIFFTAG_SOFTWARE=CSinG - 13
- # SEPTEMBRE 2012"} with {"TIFFTAG_SOFTWARE": "CSinG - 13 SEPTEMBRE
- # 2012"}
+ # Other metadata dictionary: key-value pairs parsing is required
mdd = dict(self.app.GetMetadataDictionary(self.output_image_key))
new_mdd = {}
for key, val in mdd.items():
@@ -104,7 +101,9 @@ class OTBObject(ABC):
@property
def transform(self) -> tuple[int]:
- """Get image affine transform, rasterio style (see https://www.perrygeo.com/python-affine-transforms.html).
+ """Get image affine transform, rasterio style.
+
+ See https://www.perrygeo.com/python-affine-transforms.html
Returns:
transform: (X spacing, X offset, X origin, Y offset, Y spacing, Y origin)
@@ -116,7 +115,7 @@ class OTBObject(ABC):
return spacing_x, 0.0, origin_x, 0.0, spacing_y, origin_y
def summarize(self, *args, **kwargs):
- """Recursively summarize parameters and parents.
+ """Recursively summarize an app parameters and its parents.
Args:
*args: args for `pyotb.summarize()`
@@ -138,7 +137,7 @@ class OTBObject(ABC):
def get_values_at_coords(
self, row: int, col: int, bands: int = None
- ) -> list[int | float] | int | float:
+ ) -> list[float] | float:
"""Get pixel value(s) at a given YX coordinates.
Args:
@@ -149,6 +148,9 @@ class OTBObject(ABC):
Returns:
single numerical value or a list of values for each band
+ Raises:
+ TypeError: if bands is not a slice or list
+
"""
channels = []
app = App("PixelValue", self, coordx=col, coordy=row, frozen=True, quiet=True)
@@ -170,8 +172,19 @@ class OTBObject(ABC):
data = literal_eval(app.app.GetParameterString("value"))
return data[0] if len(channels) == 1 else data
- def channels_list_from_slice(self, bands: int) -> list[int]:
- """Get list of channels to read values at, from a slice."""
+ def channels_list_from_slice(self, bands: slice) -> list[int]:
+ """Get list of channels to read values at, from a slice.
+
+ Args:
+ bands: slice obtained when using app[:]
+
+ Returns:
+ list of channels to select
+
+ Raises:
+ ValueError: if the slice is malformed
+
+ """
nb_channels = self.shape[2]
start, stop, step = bands.start, bands.stop, bands.step
start = nb_channels + start if isinstance(start, int) and start < 0 else start
@@ -226,7 +239,7 @@ class OTBObject(ABC):
required to True if preserve_dtype is False and the source app reference is lost
Returns:
- a numpy array
+ a numpy array that may already have been cached in self.exports_dic
"""
data = self.export(key, preserve_dtype)
@@ -257,6 +270,7 @@ class OTBObject(ABC):
Returns:
pixel index: (row, col)
+
"""
spacing_x, _, origin_x, _, spacing_y, origin_y = self.transform
row, col = (origin_y - y) / spacing_y, (x - origin_x) / spacing_x
@@ -272,8 +286,8 @@ class OTBObject(ABC):
x: first element
y: second element
- Return:
- operator
+ Returns:
+ an Operation object instance
"""
if isinstance(y, (np.ndarray, np.generic)):
@@ -420,7 +434,6 @@ class OTBObject(ABC):
Args:
item: attribute name
-
"""
note = (
"Since pyotb 2.0.0, OTBObject instances have stopped to forward "
@@ -430,8 +443,6 @@ class OTBObject(ABC):
hint = None
if item in dir(self.app):
- # Because otbApplication instances methods names start with an
- # upper case
hint = f"Maybe try `pyotb_app.app.{item}` instead of `pyotb_app.{item}`? "
if item.startswith("GetParameter"):
hint += (
@@ -439,10 +450,8 @@ class OTBObject(ABC):
"shorten with `pyotb_app['paramname']` to access parameters "
"values."
)
-
elif item in self.parameters_keys:
- # Because in pyotb 1.5.4, applications outputs were added as
- # attributes of the instance
+ # Because in pyotb 1.5.4, app outputs were added as instance attributes
hint = (
"Note: `pyotb_app.paramname` is no longer supported. Starting "
"from pyotb 2.0.0, `pyotb_app['paramname']` can be used to "
@@ -469,6 +478,9 @@ class OTBObject(ABC):
Returns:
list of pixel values if vector image, or pixel value, or Slicer
+ Raises:
+ ValueError: if key is not a valid pixel index or slice
+
"""
# Accessing pixel value(s) using Y/X coordinates
if isinstance(key, tuple) and len(key) >= 2:
@@ -488,12 +500,15 @@ class OTBObject(ABC):
f'"{key}" cannot be interpreted as valid slicing. Slicing should be 2D or 3D.'
)
if isinstance(key, tuple) and len(key) == 2:
- # Adding a 3rd dimension
- key = key + (slice(None, None, None),)
+ key = key + (slice(None, None, None),) # adding 3rd dimension
return Slicer(self, *key)
def __repr__(self) -> str:
- """Return a string representation with object id, this is a key used to store image ref in Operation dicts."""
+ """Return a string representation with object id.
+
+ This is used as key to store image ref in Operation dicts.
+
+ """
return f"<pyotb.{self.__class__.__name__} object, id {id(self)}>"
@@ -510,24 +525,20 @@ class App(OTBObject):
otb.ParameterType_InputFilename,
otb.ParameterType_InputFilenameList,
]
-
OUTPUT_IMAGE_TYPES = [otb.ParameterType_OutputImage]
OUTPUT_PARAM_TYPES = OUTPUT_IMAGE_TYPES + [
otb.ParameterType_OutputVectorData,
otb.ParameterType_OutputFilename,
]
-
- INPUT_LIST_TYPES = [
- otb.ParameterType_InputImageList,
- otb.ParameterType_StringList,
- otb.ParameterType_InputFilenameList,
- otb.ParameterType_ListView,
- otb.ParameterType_InputVectorDataList,
- ]
INPUT_IMAGES_LIST_TYPES = [
otb.ParameterType_InputImageList,
otb.ParameterType_InputFilenameList,
]
+ INPUT_LIST_TYPES = [
+ otb.ParameterType_StringList,
+ otb.ParameterType_ListView,
+ otb.ParameterType_InputVectorDataList,
+ ] + INPUT_IMAGES_LIST_TYPES
def __init__(
self,
@@ -550,7 +561,6 @@ class App(OTBObject):
frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___
quiet: whether to print logs of the OTB app
name: custom name that will show up in logs, appname will be used if not provided
-
**kwargs: used for passing application parameters.
e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif'
@@ -606,7 +616,7 @@ class App(OTBObject):
@property
def app(self) -> otb.Application:
- """Property to return an internal _app instance."""
+ """Reference to this app otb.Application instance."""
return self._app
@property
@@ -616,41 +626,25 @@ class App(OTBObject):
@property
def exports_dic(self) -> dict[str, dict]:
- """Returns internal _exports_dic object that contains numpy array exports."""
+ """Reference to an internal dict object that contains numpy array exports."""
return self._exports_dic
def __is_one_of_types(self, key: str, param_types: list[int]) -> bool:
- """Helper to factor is_input and is_output."""
+ """Helper to check the type of a parameter."""
if key not in self._all_param_types:
raise KeyError(f"key {key} not found in the application parameters types")
return self._all_param_types[key] in param_types
def __is_multi_output(self):
- """Check if app has multiple outputs to ensure execution during write()."""
+ """Check if app has multiple outputs to ensure re-execution during write()."""
return len(self.outputs) > 1
def is_input(self, key: str) -> bool:
- """Returns True if the key is an input.
-
- Args:
- key: parameter key
-
- Returns:
- True if the parameter is an input, else False
-
- """
+ """Returns True if the parameter key is an input."""
return self.__is_one_of_types(key=key, param_types=self.INPUT_PARAM_TYPES)
def is_output(self, key: str) -> bool:
- """Returns True if the key is an output.
-
- Args:
- key: parameter key
-
- Returns:
- True if the parameter is an output, else False
-
- """
+ """Returns True if the parameter key is an output."""
return self.__is_one_of_types(key=key, param_types=self.OUTPUT_PARAM_TYPES)
def is_key_list(self, key: str) -> bool:
@@ -670,7 +664,7 @@ class App(OTBObject):
if value == param_type:
return key
raise TypeError(
- f"{self.name}: could not find any parameter key matching the provided types"
+ f"{self.name}: could not find any key matching the provided types"
)
@property
@@ -705,14 +699,14 @@ class App(OTBObject):
instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths
Args:
- *args: Can be : - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
- (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
+ *args: Can be : - dictionary containing key-arguments enumeration. Useful when a keyword is reserved (e.g. "in")
- string or OTBObject, useful when the user implicitly wants to set the param "in"
- list, useful when the user implicitly wants to set the param "il"
**kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif'
Raises:
- Exception: when the setting of a parameter failed
+ KeyError: when the parameter name wasn't recognized
+ RuntimeError: failed to set parameter valie
"""
parameters = kwargs
@@ -723,7 +717,8 @@ class App(OTBObject):
key = key.replace("_", ".")
if key not in self.parameters_keys:
raise KeyError(
- f"{self.name}: parameter '{key}' was not recognized. Available keys are {self.parameters_keys}"
+ f"{self.name}: parameter '{key}' was not recognized."
+ f" Available keys are {self.parameters_keys}"
)
# When the parameter expects a list, if needed, change the value to list
if self.is_key_list(key) and not isinstance(obj, (list, tuple)):
@@ -741,7 +736,8 @@ class App(OTBObject):
self.__set_param(key, obj)
except (RuntimeError, TypeError, ValueError, KeyError) as e:
raise RuntimeError(
- f"{self.name}: error before execution, while setting parameter '{key}' to '{obj}': {e})"
+ f"{self.name}: error before execution,"
+ f" while setting '{key}' to '{obj}': {e})"
) from e
# Save / update setting value and update the Output object initialized in __init__ without a filepath
self._settings[key] = obj
@@ -873,7 +869,6 @@ class App(OTBObject):
if isinstance(ext_fname, str):
ext_fname = _str2dict(ext_fname)
-
logger.debug("%s: extended filename for all outputs:", self.name)
for key, ext in ext_fname.items():
logger.debug("%s: %s", key, ext)
@@ -881,14 +876,12 @@ class App(OTBObject):
for key, filepath in kwargs.items():
if self._out_param_types[key] == otb.ParameterType_OutputImage:
new_ext_fname = ext_fname.copy()
-
- # grab already set extended filename key/values
+ # Grab already set extended filename key/values
if "?&" in filepath:
filepath, already_set_ext = filepath.split("?&", 1)
- # extensions in filepath prevail over `new_ext_fname`
+ # Extensions in filepath prevail over `new_ext_fname`
new_ext_fname.update(_str2dict(already_set_ext))
-
- # transform dict to str
+ # tyransform dict to str
ext_fname_str = "&".join(
[f"{key}={value}" for key, value in new_ext_fname.items()]
)
@@ -911,7 +904,7 @@ class App(OTBObject):
key: parse_pixel_type(dtype) for key, dtype in pixel_type.items()
}
elif preserve_dtype:
- self.propagate_dtype() # all outputs will have the same type as the main input raster
+ self.propagate_dtype()
# Set parameters and flush to disk
for key, filepath in parameters.items():
@@ -942,12 +935,11 @@ class App(OTBObject):
)
return bool(files) and not missing
- # Private functions
def __parse_args(self, args: list[str | OTBObject | dict | list]) -> dict[str, Any]:
"""Gather all input arguments in kwargs dict.
Args:
- args: the list of arguments passed to set_parameters()
+ args: the list of arguments passed to set_parameters (__init__ *args)
Returns:
a dictionary with the right keyword depending on the object
@@ -966,12 +958,11 @@ class App(OTBObject):
return kwargs
def __check_input_param(
- self, obj: list | OTBObject | str | Path
+ self, obj: list | tuple | OTBObject | str | Path
) -> list | OTBObject | str:
"""Check the type and value of an input parameter, add vsi prefixes if needed."""
- if isinstance(obj, list):
+ if isinstance(obj, (list, tuple)):
return [self.__check_input_param(o) for o in obj]
- # May be we could add some checks here
if isinstance(obj, OTBObject):
return obj
if isinstance(obj, Path):
@@ -981,7 +972,6 @@ class App(OTBObject):
# Remote file. TODO: add support for S3 / GS / AZ
if obj.startswith(("https://", "http://", "ftp://")):
obj = "/vsicurl/" + obj
- # Compressed file
prefixes = {
".tar": "vsitar",
".tar.gz": "vsitar",
@@ -1000,9 +990,9 @@ class App(OTBObject):
return obj
raise TypeError(f"{self.name}: wrong input parameter type ({type(obj)})")
- def __check_output_param(self, obj: list | str | Path) -> list | str:
+ def __check_output_param(self, obj: list | tuple | str | Path) -> list | str:
"""Check the type and value of an output parameter."""
- if isinstance(obj, list):
+ if isinstance(obj, (list, tuple)):
return [self.__check_output_param(o) for o in obj]
if isinstance(obj, Path):
obj = str(obj)
@@ -1020,27 +1010,22 @@ class App(OTBObject):
# Single-parameter cases
if isinstance(obj, OTBObject):
self.app.ConnectImage(key, obj.app, obj.output_image_key)
- elif isinstance(
- obj, otb.Application
- ): # this is for backward comp with plain OTB
+ elif isinstance(obj, otb.Application):
self.app.ConnectImage(key, obj, get_out_images_param_keys(obj)[0])
- elif (
- key == "ram"
- ): # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200
+ elif key == "ram":
+ # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf OTB issue 2200
self.app.SetParameterInt("ram", int(obj))
- elif not isinstance(obj, list): # any other parameters (str, int...)
+ # Any other parameters (str, int...)
+ elif not isinstance(obj, (list, tuple)):
self.app.SetParameterValue(key, obj)
# Images list
elif self.is_key_images_list(key):
- # To enable possible in-memory connections, we go through the list and set the parameters one by one
for inp in obj:
if isinstance(inp, OTBObject):
self.app.ConnectImage(key, inp.app, inp.output_image_key)
- elif isinstance(
- inp, otb.Application
- ): # this is for backward comp with plain OTB
+ elif isinstance(inp, otb.Application):
self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0])
- else: # here `input` should be an image filepath
+ elif isinstance(inp, (str, Path)):
# Append `input` to the list, do not overwrite any previously set element of the image list
self.app.AddParameterStringList(key, inp)
else:
@@ -1101,7 +1086,9 @@ class App(OTBObject):
self.data[key] = value
# Special functions
- def __getitem__(self, key: str) -> Any | list[int | float] | int | float | Slicer:
+ def __getitem__(
+ self, key: str | tuple
+ ) -> Any | list[int | float] | int | float | Slicer:
"""This function is called when we use App()[...].
We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer
@@ -1141,7 +1128,10 @@ class Slicer(App):
obj: input
rows: slice along Y / Latitude axis
cols: slice along X / Longitude axis
- channels: channels, can be slicing, list or int
+ channels: bands to extract. can be slicing, list or int
+
+ Raises:
+ TypeError: if channels param isn't slice, list or int
"""
super().__init__(
@@ -1157,20 +1147,18 @@ class Slicer(App):
# Channel slicing
if channels != slice(None, None, None):
- # Trigger source app execution if needed
nb_channels = get_nbchannels(obj)
self.app.Execute() # this is needed by ExtractROI for setting the `cl` parameter
- # if needed, converting int to list
if isinstance(channels, int):
channels = [channels]
- # if needed, converting slice to list
elif isinstance(channels, slice):
channels = self.channels_list_from_slice(channels)
elif isinstance(channels, tuple):
channels = list(channels)
elif not isinstance(channels, list):
- raise ValueError(
- f"Invalid type for channels, should be int, slice or list of bands. : {channels}"
+ raise TypeError(
+ f"Invalid type for channels ({type(channels)})."
+ f" Should be int, slice or list of bands."
)
# Change the potential negative index values to reverse index
channels = [c if c >= 0 else nb_channels + c for c in channels]
@@ -1178,28 +1166,24 @@ class Slicer(App):
# Spatial slicing
spatial_slicing = False
- # TODO: handle the step value in the slice so that NN undersampling is possible ? e.g. raster[::2, ::2]
if rows.start is not None:
parameters.update({"mode.extent.uly": rows.start})
spatial_slicing = True
if rows.stop is not None and rows.stop != -1:
- parameters.update(
- {"mode.extent.lry": rows.stop - 1}
- ) # subtract 1 to respect python convention
+ # Subtract 1 to respect python convention
+ parameters.update({"mode.extent.lry": rows.stop - 1})
spatial_slicing = True
if cols.start is not None:
parameters.update({"mode.extent.ulx": cols.start})
spatial_slicing = True
if cols.stop is not None and cols.stop != -1:
- parameters.update(
- {"mode.extent.lrx": cols.stop - 1}
- ) # subtract 1 to respect python convention
+ # Subtract 1 to respect python convention
+ parameters.update({"mode.extent.lrx": cols.stop - 1})
spatial_slicing = True
- # These are some attributes when the user simply wants to extract *one* band to be used in an Operation
+ # When the user simply wants to extract *one* band to be used in an Operation
if not spatial_slicing and isinstance(channels, list) and len(channels) == 1:
- self.one_band_sliced = (
- channels[0] + 1
- ) # OTB convention: channels start at 1
+ # OTB convention: channels start at 1
+ self.one_band_sliced = channels[0] + 1
self.input = obj
# Execute app
@@ -1230,33 +1214,32 @@ class Operation(App):
"""
def __init__(self, operator: str, *inputs, nb_bands: int = None, name: str = None):
- """Given some inputs and an operator, this function enables to transform this into an OTB application.
+ """Given some inputs and an operator, this object enables to python operator to a BandMath operation.
Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator.
It can have 3 inputs for the ternary operator `cond ? x : y`.
Args:
operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
- *inputs: inputs. Can be OTBObject, filepath, int or float
- nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
- name: override the Operation name
+ *inputs: operands, can be OTBObject, filepath, int or float
+ nb_bands: optionally specify the output nb of bands - used only internally by pyotb.where
+ name: override the default Operation name
"""
self.operator = operator
- # We first create a 'fake' expression. E.g for the operation `input1 + input2` , we create a fake expression
- # that is like "str(input1) + str(input2)"
+ # We first create a 'fake' expression. E.g for the operation `input1 + input2`
+ # we create a fake expression like "str(input1) + str(input2)"
self.inputs = []
self.nb_channels = {}
self.fake_exp_bands = []
self.build_fake_expressions(operator, inputs, nb_bands=nb_bands)
# Transforming images to the adequate im#, e.g. `input1` to "im1"
- # creating a dictionary that is like {str(input1): 'im1', 'image2.tif': 'im2', ...}.
+ # using a dictionary : {str(input1): 'im1', 'image2.tif': 'im2', ...}.
# NB: the keys of the dictionary are strings-only, instead of 'complex' objects, to enable easy serialization
self.im_dic = {}
self.im_count = 1
- map_repr_to_input = (
- {}
- ) # to be able to retrieve the real python object from its string representation
+ # To be able to retrieve the real python object from its string representation
+ map_repr_to_input = {}
for inp in self.inputs:
if not isinstance(inp, (int, float)):
if str(inp) not in self.im_dic:
@@ -1316,6 +1299,9 @@ class Operation(App):
inputs: inputs. Can be OTBObject, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
+ Raises:
+ ValueError: if all inputs don't have the same number of bands
+
"""
self.inputs.clear()
self.nb_channels.clear()
@@ -1336,30 +1322,30 @@ class Operation(App):
if len(inputs) == 3 and k == 0:
# When cond is monoband whereas the result is multiband, we expand the cond to multiband
cond_band = 1 if nb_bands != inp.shape[2] else band
- fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp(
+ fake_exp, corresp_inputs, nb_channels = self.make_fake_exp(
inp, cond_band, keep_logical=True
)
else:
# Any other input
- fake_exp, corresponding_inputs, nb_channels = self.make_fake_exp(
+ fake_exp, corresp_inputs, nb_channels = self.make_fake_exp(
inp, band, keep_logical=False
)
expressions.append(fake_exp)
# Reference the inputs and nb of channels (only on first pass in the loop to avoid duplicates)
- if i == 0 and corresponding_inputs and nb_channels:
- self.inputs.extend(corresponding_inputs)
+ if i == 0 and corresp_inputs and nb_channels:
+ self.inputs.extend(corresp_inputs)
self.nb_channels.update(nb_channels)
# Generating the fake expression of the whole operation
- if len(inputs) == 1: # this is only for 'abs'
+ if len(inputs) == 1:
+ # This is only for 'abs()'
fake_exp = f"({operator}({expressions[0]}))"
elif len(inputs) == 2:
# We create here the "fake" expression. For example, for a BandMathX expression such as '2 * im1 + im2',
# the false expression stores the expression 2 * str(input1) + str(input2)
fake_exp = f"({expressions[0]} {operator} {expressions[1]})"
- elif (
- len(inputs) == 3 and operator == "?"
- ): # this is only for ternary expression
+ elif len(inputs) == 3 and operator == "?":
+ # This is only for ternary expression
fake_exp = f"({expressions[0]} ? {expressions[1]} : {expressions[2]})"
self.fake_exp_bands.append(fake_exp)
@@ -1445,10 +1431,12 @@ class Operation(App):
class LogicalOperation(Operation):
- """A specialization of Operation class for boolean logical operations i.e. >, <, >=, <=, ==, !=, `&` and `|`.
+ """A specialization of Operation class for boolean logical operations.
- The only difference is that not only the BandMath expression is saved (e.g. "im1b1 > 0 ? 1 : 0"), but also the
- logical expression (e.g. "im1b1 > 0")
+ Supported operators are >, <, >=, <=, ==, !=, `&` and `|`.
+
+ The only difference is that not only the BandMath expression is saved (e.g. "im1b1 > 0 ? 1 : 0"),
+ but also the logical expression (e.g. "im1b1 > 0")
"""
@@ -1458,7 +1446,7 @@ class LogicalOperation(Operation):
Args:
operator: string operator (one of >, <, >=, <=, ==, !=, &, |)
*inputs: inputs
- nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
+ nb_bands: optionally specify the output nb of bands - used only by pyotb.where
"""
self.logical_fake_exp_bands = []
@@ -1481,7 +1469,7 @@ class LogicalOperation(Operation):
Args:
operator: str (one of >, <, >=, <=, ==, !=, &, |)
inputs: Can be OTBObject, filepath, int or float
- nb_bands: optionnaly specify the output nb of bands - used only internally by pyotb.where
+ nb_bands: optionally specify the output nb of bands - used only internally by pyotb.where
"""
# Create a list of fake exp, each item of the list corresponding to one band
@@ -1530,7 +1518,7 @@ class Input(App):
class Output(OTBObject):
- """Object that behave like a pointer to a specific application output file."""
+ """Object that behave like a pointer to a specific application in-memory output or file."""
_filepath: str | Path = None
@@ -1547,7 +1535,7 @@ class Output(OTBObject):
Args:
pyotb_app: The pyotb App to store reference from
param_key: Output parameter key of the target app
- filepath: path of the output file (if not in memory)
+ filepath: path of the output file (if not memory)
mkdir: create missing parent directories
"""
@@ -1574,7 +1562,7 @@ class Output(OTBObject):
@property
def exports_dic(self) -> dict[str, dict]:
- """Returns internal _exports_dic object that contains numpy array exports."""
+ """Reference to parent _exports_dic object that contains np array exports."""
return self.parent_pyotb_app.exports_dic
@property
@@ -1597,19 +1585,34 @@ class Output(OTBObject):
self._filepath = path
def exists(self) -> bool:
- """Check file exist."""
+ """Check if the output file exist on disk.
+
+ Raises:
+ ValueError: if filepath is not set or is remote URL
+
+ """
if not isinstance(self.filepath, Path):
raise ValueError("Filepath is not set or points to a remote URL")
return self.filepath.exists()
def make_parent_dirs(self):
- """Create missing parent directories."""
+ """Create missing parent directories.
+
+ Raises:
+ ValueError: if filepath is not set or is remote URL
+
+ """
if not isinstance(self.filepath, Path):
raise ValueError("Filepath is not set or points to a remote URL")
self.filepath.parent.mkdir(parents=True, exist_ok=True)
def write(self, filepath: None | str | Path = None, **kwargs) -> bool:
- """Write output to disk, filepath is not required if it was provided to parent App during init."""
+ """Write output to disk, filepath is not required if it was provided to parent App during init.
+
+ Args:
+ filepath: path of the output file, can be None if a value was passed during app init
+
+ """
if filepath is None:
return self.parent_pyotb_app.write(
{self.output_image_key: self.filepath}, **kwargs
@@ -1630,6 +1633,9 @@ def get_nbchannels(inp: str | Path | OTBObject) -> int:
Returns:
number of bands in image
+ Raises:
+ TypeError: if inp band count cannot be retrieved
+
"""
if isinstance(inp, OTBObject):
return inp.shape[-1]
@@ -1657,6 +1663,9 @@ def get_pixel_type(inp: str | Path | OTBObject) -> str:
pixel_type: OTB enum e.g. `otbApplication.ImagePixelType_uint8', which actually is an int.
For an OTBObject with several outputs, only the pixel type of the first output is returned
+ Raises:
+ TypeError: if inp pixel type cannot be retrieved
+
"""
if isinstance(inp, OTBObject):
return inp.app.GetParameterOutputImagePixelType(inp.output_image_key)
@@ -1679,10 +1688,14 @@ def parse_pixel_type(pixel_type: str | int) -> int:
"""Convert one str pixel type to OTB integer enum if necessary.
Args:
- pixel_type: pixel type. can be str, int or dict
+ pixel_type: pixel type. can be int or str (either OTB or numpy convention)
Returns:
- pixel_type integer value
+ pixel_type OTB enum integer value
+
+ Raises:
+ KeyError: if pixel_type name is unknown
+ TypeError: if type(pixel_type) isn't int or str
"""
if isinstance(pixel_type, int): # normal OTB int enum
@@ -1704,19 +1717,19 @@ def parse_pixel_type(pixel_type: str | int) -> int:
if pixel_type in datatype_to_pixeltype:
return getattr(otb, f"ImagePixelType_{datatype_to_pixeltype[pixel_type]}")
raise KeyError(
- f"Unknown data type `{pixel_type}`. Available ones: {datatype_to_pixeltype}"
+ f"Unknown dtype `{pixel_type}`. Available ones: {datatype_to_pixeltype}"
)
raise TypeError(
f"Bad pixel type specification ({pixel_type} of type {type(pixel_type)})"
)
-def get_out_images_param_keys(app: OTBObject) -> list[str]:
- """Return every output parameter keys of an OTB app."""
+def get_out_images_param_keys(otb_app: otb.Application) -> list[str]:
+ """Return every output parameter keys of a bare OTB app."""
return [
key
- for key in app.GetParametersKeys()
- if app.GetParameterType(key) == otb.ParameterType_OutputImage
+ for key in otb_app.GetParametersKeys()
+ if otb_app.GetParameterType(key) == otb.ParameterType_OutputImage
]
diff --git a/pyotb/depreciation.py b/pyotb/depreciation.py
index 687e6ca..6794373 100644
--- a/pyotb/depreciation.py
+++ b/pyotb/depreciation.py
@@ -1,7 +1,6 @@
"""Helps with deprecated classes and methods.
-Taken from https://stackoverflow.com/questions/49802412/how-to-implement-
-deprecation-in-python-with-argument-alias
+Taken from https://stackoverflow.com/questions/49802412/how-to-implement-deprecation-in-python-with-argument-alias
"""
from typing import Callable, Dict, Any
import functools
@@ -17,7 +16,7 @@ def depreciation_warning(message: str):
"""Shows a warning message.
Args:
- message: message
+ message: message to log
"""
warnings.warn(
@@ -63,11 +62,14 @@ def rename_kwargs(func_name: str, kwargs: Dict[str, Any], aliases: Dict[str, str
kwargs: keyword args
aliases: aliases
+ Raises:
+ ValueError: if both old and new arguments are provided
+
"""
for alias, new in aliases.items():
if alias in kwargs:
if new in kwargs:
- raise TypeError(
+ raise ValueError(
f"{func_name} received both {alias} and {new} as arguments!"
f" {alias} is deprecated, use {new} instead."
)
diff --git a/pyotb/functions.py b/pyotb/functions.py
index ad2f24c..8fde2cd 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -247,7 +247,7 @@ def run_tf_function(func):
"""
# Getting the string definition of the tf function (e.g. "def multiply(x1, x2):...")
- # TODO: maybe not entirely foolproof, maybe we should use dill instead? but it would add a dependency
+ # Maybe not entirely foolproof, maybe we should use dill instead? but it would add a dependency
func_def_str = inspect.getsource(func)
func_name = func.__name__
@@ -342,7 +342,7 @@ def run_tf_function(func):
for i, inp in enumerate(raster_inputs):
model_serve.set_parameters({f"source{i + 1}.il": [inp]})
model_serve.execute()
- # TODO: handle the deletion of the temporary model ?
+ # Possible ENH: handle the deletion of the temporary model ?
return model_serve
@@ -403,7 +403,7 @@ def define_processing_area(
)
# Handling different spatial footprints
- # TODO: there seems to have a bug, ImageMetaData is not updated when running an app,
+ # TODO: find possible bug - ImageMetaData is not updated when running an app
# cf https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2234. Should we use ImageOrigin instead?
if not all(
md["UpperLeftCorner"] == any_metadata["UpperLeftCorner"]
@@ -432,10 +432,10 @@ def define_processing_area(
lry = metadatas[reference_window_input]["LowerRightCorner"][1]
uly = metadatas[reference_window_input]["UpperLeftCorner"][1]
elif window_rule == "specify":
- # TODO : when the user explicitly specifies the bounding box -> add some arguments in the function
+ # When the user explicitly specifies the bounding box -> add some arguments in the function
...
elif window_rule == "union":
- # TODO : when the user wants the final bounding box to be the union of all bounding box
+ # When the user wants the final bounding box to be the union of all bounding box
# It should replace any 'outside' pixel by some NoData -> add `fillvalue` argument in the function
...
@@ -458,7 +458,7 @@ def define_processing_area(
"mode.extent.lry": lry,
}
new_input = App("ExtractROI", params, quiet=True)
- # TODO: OTB 7.4 fixes this bug, how to handle different versions of OTB?
+ # OTB 7.4 fixes this bug, how to handle different versions of OTB?
new_inputs.append(new_input)
# Potentially update the reference inputs for later resampling
if str(inp) == str(reference_pixel_size_input):
@@ -495,8 +495,9 @@ def define_processing_area(
elif pixel_size_rule == "same_as_input":
reference_input = reference_pixel_size_input
elif pixel_size_rule == "specify":
- pass
- # TODO : when the user explicitly specify the pixel size -> add argument inside the function
+ # When the user explicitly specify the pixel size -> add argument inside the function
+ ...
+
pixel_size = metadatas[reference_input]["GeoTransform"][1]
# Perform resampling on inputs that do not comply with the target pixel size
diff --git a/pyotb/helpers.py b/pyotb/helpers.py
index e0feb63..0e6ea2a 100644
--- a/pyotb/helpers.py
+++ b/pyotb/helpers.py
@@ -60,6 +60,10 @@ def find_otb(prefix: str = OTB_ROOT, scan: bool = True):
Returns:
otbApplication module
+ Raises:
+ SystemError: is OTB is not found (when using interactive mode)
+ SystemExit: if OTB is not found, since pyotb won't be usable
+
"""
otb = None
# Try OTB_ROOT env variable first (allow override default OTB version)
@@ -125,6 +129,9 @@ def set_environment(prefix: str):
Args:
prefix: path to OTB root directory
+ Raises:
+ SystemError: if OTB or GDAL is not found
+
"""
logger.info("Preparing environment for OTB in %s", prefix)
# OTB root directory
@@ -138,16 +145,18 @@ def set_environment(prefix: str):
lib_dir = __find_lib(prefix)
if not lib_dir:
raise SystemError("Can't find OTB external libraries")
- # This does not seems to work
+ # LD library path : this does not seems to work
if sys.platform == "linux" and built_from_source:
new_ld_path = f"{lib_dir}:{os.environ.get('LD_LIBRARY_PATH') or ''}"
os.environ["LD_LIBRARY_PATH"] = new_ld_path
+
# Add python bindings directory first in PYTHONPATH
otb_api = __find_python_api(lib_dir)
if not otb_api:
raise SystemError("Can't find OTB Python API")
if otb_api not in sys.path:
sys.path.insert(0, otb_api)
+
# Add /bin first in PATH, in order to avoid conflicts with another GDAL install
os.environ["PATH"] = f"{prefix / 'bin'}{os.pathsep}{os.environ['PATH']}"
# Ensure APPLICATION_PATH is set
@@ -159,6 +168,7 @@ def set_environment(prefix: str):
os.environ["LC_NUMERIC"] = "C"
os.environ["GDAL_DRIVER_PATH"] = "disable"
+ # Find GDAL libs
if (prefix / "share/gdal").exists():
# Local GDAL (OTB Superbuild, .run, .exe)
gdal_data = str(prefix / "share/gdal")
@@ -186,7 +196,7 @@ def __find_lib(prefix: str = None, otb_module=None):
otb_module: try with otbApplication library path if found, else None
Returns:
- lib path
+ lib path, or None if not found
"""
if prefix is not None:
@@ -216,7 +226,7 @@ def __find_python_api(lib_dir: Path):
prefix: prefix
Returns:
- python API path if found, else None
+ OTB python API path, or None if not found
"""
otb_api = lib_dir / "python"
@@ -235,7 +245,7 @@ def __find_apps_path(lib_dir: Path):
lib_dir: library path
Returns:
- application path if found, else empty string
+ application path, or empty string if not found
"""
if lib_dir.exists():
@@ -251,7 +261,7 @@ def __find_otb_root():
"""Search for OTB root directory in well known locations.
Returns:
- str path of the OTB directory
+ str path of the OTB directory, or None if not found
"""
prefix = None
@@ -339,5 +349,5 @@ def __suggest_fix_import(error_message: str, prefix: str):
# This part of pyotb is the first imported during __init__ and checks if OTB is found
-# If OTB is not found, a SystemExit is raised, to prevent execution of the core module
+# If OTB isn't found, a SystemExit is raised to prevent execution of the core module
find_otb()
diff --git a/pyotb/install.py b/pyotb/install.py
index 47a6ee7..219967a 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -33,15 +33,17 @@ def otb_latest_release_tag():
return releases[-1]
-def check_versions(sysname: str, python_minor: int, otb_major: int):
+def check_versions(sysname: str, python_minor: int, otb_major: int) -> tuple[bool, int]:
"""Verify if python version is compatible with major OTB version.
Args:
sysname: OTB's system name convention (Linux64, Darwin64, Win64)
python_minor: minor version of python
otb_major: major version of OTB to be installed
+
Returns:
(True, 0) or (False, expected_version) if case of version conflict
+
"""
if sysname == "Win64":
expected = 5 if otb_major in (6, 7) else 7
@@ -103,6 +105,10 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True):
Returns:
full path of the new installation
+ Raises:
+ SystemExit: if python version is not compatible with major OTB version
+ SystemError: if automatic env config failed
+
"""
# Read env config
if sys.version_info.major == 2:
@@ -120,6 +126,7 @@ def install_otb(version: str = "latest", path: str = "", edit_env: bool = True):
raise SystemExit(
f"Python 3.{expected} is required to import bindings on Windows."
)
+
# Fetch archive and run installer
filename = f"OTB-{version}-{sysname}.{ext}"
url = f"https://www.orfeo-toolbox.org/packages/archives/OTB/{filename}"
--
GitLab
From 3da1b9f61c955faa7e3a9db7fa0ac1217dc0bcd8 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Nov 2023 17:14:13 +0100
Subject: [PATCH 365/399] CI: codespell typos
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 0f27547..d6336b5 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -706,7 +706,7 @@ class App(OTBObject):
Raises:
KeyError: when the parameter name wasn't recognized
- RuntimeError: failed to set parameter valie
+ RuntimeError: failed to set parameter value
"""
parameters = kwargs
--
GitLab
From 0770dac0db935663b42193ed457ffb6ca97af7ab Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Nov 2023 17:16:32 +0100
Subject: [PATCH 366/399] ENH: use integer in check_versions
---
pyotb/install.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 219967a..2d5561c 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -33,7 +33,7 @@ def otb_latest_release_tag():
return releases[-1]
-def check_versions(sysname: str, python_minor: int, otb_major: int) -> tuple[bool, int]:
+def check_versions(sysname: str, python_minor: int, otb_major: int) -> tuple[int]:
"""Verify if python version is compatible with major OTB version.
Args:
@@ -42,22 +42,22 @@ def check_versions(sysname: str, python_minor: int, otb_major: int) -> tuple[boo
otb_major: major version of OTB to be installed
Returns:
- (True, 0) or (False, expected_version) if case of version conflict
+ (1, 0) or (0, expected_version) if case of version conflict
"""
if sysname == "Win64":
expected = 5 if otb_major in (6, 7) else 7
if python_minor == expected:
- return True, 0
+ return 1, 0
elif sysname == "Darwin64":
expected = 7, 0
if python_minor == expected:
- return True, 0
+ return 1, 0
elif sysname == "Linux64":
expected = 5 if otb_major in (6, 7) else 8
if python_minor == expected:
- return True, 0
- return False, expected
+ return 1, 0
+ return 0, expected
def env_config_unix(otb_path: Path):
--
GitLab
From 853b206fba8701ff304a726fc44c8de8dd761159 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Nov 2023 17:28:47 +0100
Subject: [PATCH 367/399] FIX: regression in __set_param
---
pyotb/core.py | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index d6336b5..80916d0 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1034,12 +1034,9 @@ class App(OTBObject):
f" found in '{key}' list: {inp}"
)
# List of any other types (str, int...)
- elif self.is_key_list(key):
- self.app.SetParameterValue(key, obj)
else:
- raise TypeError(
- f"{self.name}: wrong input parameter type ({type(obj)}) for '{key}'"
- )
+ # TODO: use self.is_key_list, but this is not working for ExtractROI param "cl" which is ParameterType_UNKNOWN
+ self.app.SetParameterValue(key, obj)
def __sync_parameters(self):
"""Save app parameters in _auto_parameters or data dict.
--
GitLab
From e1791ea4b0ceb3069ae6012a11b8c9842f70a944 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Nov 2023 18:06:47 +0100
Subject: [PATCH 368/399] BUG: add missing annotations import from __future__
---
pyotb/install.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/pyotb/install.py b/pyotb/install.py
index 2d5561c..5a8256a 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -1,4 +1,6 @@
"""This module contains functions for interactive auto installation of OTB."""
+from __future__ import annotations
+
import json
import os
import re
--
GitLab
From d09c718ad945fc0a0632a1248cd28fef9fe415c2 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Nov 2023 18:07:39 +0100
Subject: [PATCH 369/399] DOC: remove useless comment since bug is fixed
---
pyotb/functions.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/pyotb/functions.py b/pyotb/functions.py
index 8fde2cd..fe93641 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -458,7 +458,6 @@ def define_processing_area(
"mode.extent.lry": lry,
}
new_input = App("ExtractROI", params, quiet=True)
- # OTB 7.4 fixes this bug, how to handle different versions of OTB?
new_inputs.append(new_input)
# Potentially update the reference inputs for later resampling
if str(inp) == str(reference_pixel_size_input):
--
GitLab
From 5c8f8dcccc73ddbbe1042208488f299aea44c03f Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Nov 2023 18:52:24 +0100
Subject: [PATCH 370/399] ENH: back to bool since type hints bug is fixed
---
pyotb/install.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/pyotb/install.py b/pyotb/install.py
index 5a8256a..0cf139e 100644
--- a/pyotb/install.py
+++ b/pyotb/install.py
@@ -35,7 +35,7 @@ def otb_latest_release_tag():
return releases[-1]
-def check_versions(sysname: str, python_minor: int, otb_major: int) -> tuple[int]:
+def check_versions(sysname: str, python_minor: int, otb_major: int) -> tuple[bool, int]:
"""Verify if python version is compatible with major OTB version.
Args:
@@ -44,22 +44,22 @@ def check_versions(sysname: str, python_minor: int, otb_major: int) -> tuple[int
otb_major: major version of OTB to be installed
Returns:
- (1, 0) or (0, expected_version) if case of version conflict
+ (True, 0) if compatible or (False, expected_version) in case of conflicts
"""
if sysname == "Win64":
expected = 5 if otb_major in (6, 7) else 7
if python_minor == expected:
- return 1, 0
+ return True, 0
elif sysname == "Darwin64":
expected = 7, 0
if python_minor == expected:
- return 1, 0
+ return True, 0
elif sysname == "Linux64":
expected = 5 if otb_major in (6, 7) else 8
if python_minor == expected:
- return 1, 0
- return 0, expected
+ return True, 0
+ return False, expected
def env_config_unix(otb_path: Path):
--
GitLab
From 014a07574aa05a0cffb7c573564b5815e77ef0a8 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Fri, 3 Nov 2023 19:35:57 +0100
Subject: [PATCH 371/399] ENH: remove useless exceptions since already checked
in set_parameters
---
pyotb/core.py | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 80916d0..df3297a 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1025,14 +1025,9 @@ class App(OTBObject):
self.app.ConnectImage(key, inp.app, inp.output_image_key)
elif isinstance(inp, otb.Application):
self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0])
- elif isinstance(inp, (str, Path)):
+ else:
# Append `input` to the list, do not overwrite any previously set element of the image list
self.app.AddParameterStringList(key, inp)
- else:
- raise TypeError(
- f"{self.name}: wrong input parameter type ({type(inp)})"
- f" found in '{key}' list: {inp}"
- )
# List of any other types (str, int...)
else:
# TODO: use self.is_key_list, but this is not working for ExtractROI param "cl" which is ParameterType_UNKNOWN
--
GitLab
From 848f4e4aab6d81ad773f81fcf15cef9d2e23fb87 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Nov 2023 14:15:31 +0100
Subject: [PATCH 372/399] ENH: add exception, use if self.is_key_list(key) in
__set_param
---
pyotb/core.py | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index df3297a..c0a824b 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -534,11 +534,12 @@ class App(OTBObject):
otb.ParameterType_InputImageList,
otb.ParameterType_InputFilenameList,
]
- INPUT_LIST_TYPES = [
+ INPUT_LIST_TYPES = INPUT_IMAGES_LIST_TYPES + [
otb.ParameterType_StringList,
otb.ParameterType_ListView,
otb.ParameterType_InputVectorDataList,
- ] + INPUT_IMAGES_LIST_TYPES
+ otb.ParameterType_Band,
+ ]
def __init__(
self,
@@ -1025,13 +1026,17 @@ class App(OTBObject):
self.app.ConnectImage(key, inp.app, inp.output_image_key)
elif isinstance(inp, otb.Application):
self.app.ConnectImage(key, obj, get_out_images_param_keys(inp)[0])
+ # Here inp is either str or Path, already checked by __check_*_param
else:
- # Append `input` to the list, do not overwrite any previously set element of the image list
+ # Append it to the list, do not overwrite any previously set element of the image list
self.app.AddParameterStringList(key, inp)
# List of any other types (str, int...)
- else:
- # TODO: use self.is_key_list, but this is not working for ExtractROI param "cl" which is ParameterType_UNKNOWN
+ elif self.is_key_list(key):
self.app.SetParameterValue(key, obj)
+ else:
+ raise TypeError(
+ f"{self.name}: wrong parameter type ({type(obj)}) for '{key}'"
+ )
def __sync_parameters(self):
"""Save app parameters in _auto_parameters or data dict.
--
GitLab
From 8b257b5073c5ce5ce142a73eff8e3f32cb569c14 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Nov 2023 14:40:24 +0100
Subject: [PATCH 373/399] ENH: type hints (float | int) -> (float)
---
pyotb/core.py | 40 ++++++++++++++++++++--------------------
1 file changed, 20 insertions(+), 20 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index c0a824b..f03e9e1 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -294,35 +294,35 @@ class OTBObject(ABC):
return NotImplemented # this enables to fallback on numpy emulation thanks to __array_ufunc__
return op_cls(name, x, y)
- def __add__(self, other: OTBObject | str | int | float) -> Operation:
+ def __add__(self, other: OTBObject | str | float) -> Operation:
"""Addition."""
return self.__create_operator(Operation, "+", self, other)
- def __sub__(self, other: OTBObject | str | int | float) -> Operation:
+ def __sub__(self, other: OTBObject | str | float) -> Operation:
"""Subtraction."""
return self.__create_operator(Operation, "-", self, other)
- def __mul__(self, other: OTBObject | str | int | float) -> Operation:
+ def __mul__(self, other: OTBObject | str | float) -> Operation:
"""Multiplication."""
return self.__create_operator(Operation, "*", self, other)
- def __truediv__(self, other: OTBObject | str | int | float) -> Operation:
+ def __truediv__(self, other: OTBObject | str | float) -> Operation:
"""Division."""
return self.__create_operator(Operation, "/", self, other)
- def __radd__(self, other: OTBObject | str | int | float) -> Operation:
+ def __radd__(self, other: OTBObject | str | float) -> Operation:
"""Right addition."""
return self.__create_operator(Operation, "+", other, self)
- def __rsub__(self, other: OTBObject | str | int | float) -> Operation:
+ def __rsub__(self, other: OTBObject | str | float) -> Operation:
"""Right subtraction."""
return self.__create_operator(Operation, "-", other, self)
- def __rmul__(self, other: OTBObject | str | int | float) -> Operation:
+ def __rmul__(self, other: OTBObject | str | float) -> Operation:
"""Right multiplication."""
return self.__create_operator(Operation, "*", other, self)
- def __rtruediv__(self, other: OTBObject | str | int | float) -> Operation:
+ def __rtruediv__(self, other: OTBObject | str | float) -> Operation:
"""Right division."""
return self.__create_operator(Operation, "/", other, self)
@@ -330,35 +330,35 @@ class OTBObject(ABC):
"""Absolute value."""
return Operation("abs", self)
- def __ge__(self, other: OTBObject | str | int | float) -> LogicalOperation:
+ def __ge__(self, other: OTBObject | str | float) -> LogicalOperation:
"""Greater of equal than."""
return self.__create_operator(LogicalOperation, ">=", self, other)
- def __le__(self, other: OTBObject | str | int | float) -> LogicalOperation:
+ def __le__(self, other: OTBObject | str | float) -> LogicalOperation:
"""Lower of equal than."""
return self.__create_operator(LogicalOperation, "<=", self, other)
- def __gt__(self, other: OTBObject | str | int | float) -> LogicalOperation:
+ def __gt__(self, other: OTBObject | str | float) -> LogicalOperation:
"""Greater than."""
return self.__create_operator(LogicalOperation, ">", self, other)
- def __lt__(self, other: OTBObject | str | int | float) -> LogicalOperation:
+ def __lt__(self, other: OTBObject | str | float) -> LogicalOperation:
"""Lower than."""
return self.__create_operator(LogicalOperation, "<", self, other)
- def __eq__(self, other: OTBObject | str | int | float) -> LogicalOperation:
+ def __eq__(self, other: OTBObject | str | float) -> LogicalOperation:
"""Equality."""
return self.__create_operator(LogicalOperation, "==", self, other)
- def __ne__(self, other: OTBObject | str | int | float) -> LogicalOperation:
+ def __ne__(self, other: OTBObject | str | float) -> LogicalOperation:
"""Inequality."""
return self.__create_operator(LogicalOperation, "!=", self, other)
- def __or__(self, other: OTBObject | str | int | float) -> LogicalOperation:
+ def __or__(self, other: OTBObject | str | float) -> LogicalOperation:
"""Logical or."""
return self.__create_operator(LogicalOperation, "||", self, other)
- def __and__(self, other: OTBObject | str | int | float) -> LogicalOperation:
+ def __and__(self, other: OTBObject | str | float) -> LogicalOperation:
"""Logical and."""
return self.__create_operator(LogicalOperation, "&&", self, other)
@@ -1085,7 +1085,7 @@ class App(OTBObject):
# Special functions
def __getitem__(
self, key: str | tuple
- ) -> Any | list[int | float] | int | float | Slicer:
+ ) -> Any | list[float] | float | Slicer:
"""This function is called when we use App()[...].
We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer
@@ -1255,7 +1255,7 @@ class Operation(App):
appname, il=self.unique_inputs, exp=self.exp, quiet=True, name=name
)
- def get_nb_bands(self, inputs: list[OTBObject | str | int | float]) -> int:
+ def get_nb_bands(self, inputs: list[OTBObject | str | float]) -> int:
"""Guess the number of bands of the output image, from the inputs.
Args:
@@ -1284,7 +1284,7 @@ class Operation(App):
def build_fake_expressions(
self,
operator: str,
- inputs: list[OTBObject | str | int | float],
+ inputs: list[OTBObject | str | float],
nb_bands: int = None,
):
"""Create a list of 'fake' expressions, one for each band.
@@ -1455,7 +1455,7 @@ class LogicalOperation(Operation):
def build_fake_expressions(
self,
operator: str,
- inputs: list[OTBObject | str | int | float],
+ inputs: list[OTBObject | str | float],
nb_bands: int = None,
):
"""Create a list of 'fake' expressions, one for each band.
--
GitLab
From d4384d14129d1c5843e8bb191a52bb24a879b6e1 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Nov 2023 15:06:48 +0100
Subject: [PATCH 374/399] ENH: summarize comments
---
pyotb/core.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index f03e9e1..b614a4d 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1758,6 +1758,7 @@ def summarize(
return [summarize(o) for o in obj]
if isinstance(obj, Output):
return summarize(obj.parent_pyotb_app)
+ # => This is the deepest recursion level
if not isinstance(obj, App):
return obj
@@ -1769,8 +1770,8 @@ def summarize(
return param.split("?")[0]
# Call / top level of recursion : obj is an App
- # We need to return parameters values, summarized if param is an App
parameters = {}
+ # We need to return parameters values, summarized if param is an App
for key, param in obj.parameters.items():
if strip_inpath and obj.is_input(key) or strip_outpath and obj.is_output(key):
parameters[key] = strip_path(param)
--
GitLab
From b219d44312a63d0e01372f3d1310c5a4f40e2d9b Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Nov 2023 15:06:50 +0100
Subject: [PATCH 375/399] DOC: avoid multiline argument text in docstrings
---
pyotb/core.py | 104 +++++++++++++++++++++++++-------------------------
1 file changed, 51 insertions(+), 53 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index b614a4d..5528039 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -136,14 +136,14 @@ class OTBObject(ABC):
return App("ComputeImagesStatistics", self, quiet=True).data
def get_values_at_coords(
- self, row: int, col: int, bands: int = None
+ self, row: int, col: int, bands: int | list[int] = None
) -> list[float] | float:
"""Get pixel value(s) at a given YX coordinates.
Args:
row: index along Y / latitude axis
col: index along X / longitude axis
- bands: band number, list or slice to fetch values from
+ bands: band number(s) to fetch values from
Returns:
single numerical value or a list of values for each band
@@ -209,8 +209,7 @@ class OTBObject(ABC):
Args:
key: parameter key to export, if None then the default one will be used
- preserve_dtype: when set to True, the numpy array is converted to the same pixel type as
- the App first output. Default is True
+ preserve_dtype: convert the array to the same pixel type as the App first output
Returns:
the exported numpy array
@@ -231,12 +230,13 @@ class OTBObject(ABC):
) -> np.ndarray:
"""Export a pyotb object to numpy array.
+ A copy is avoided by default, but may be required if preserve_dtype is False
+ and the source app reference is lost.
+
Args:
key: the output parameter name to export as numpy array
- preserve_dtype: when set to True, the numpy array is converted to the same pixel type as
- the App first output. Default is True
- copy: whether to copy the output array, default is False
- required to True if preserve_dtype is False and the source app reference is lost
+ preserve_dtype: convert the array to the same pixel type as the App first output
+ copy: whether to copy the output array instead of returning a reference
Returns:
a numpy array that may already have been cached in self.exports_dic
@@ -378,12 +378,12 @@ class OTBObject(ABC):
"""This is called whenever a numpy function is called on a pyotb object.
Operation is performed in numpy, then imported back to pyotb with the same georeference as input.
+ At least one obj is unputs has to be an OTBObject.
Args:
ufunc: numpy function
method: an internal numpy argument
- inputs: inputs, at least one being pyotb object. If there are several pyotb objects, they must all have
- the same georeference and pixel size.
+ inputs: inputs, with equal shape in case of several images / OTBObject
**kwargs: kwargs of the numpy function
Returns:
@@ -552,18 +552,19 @@ class App(OTBObject):
):
"""Common constructor for OTB applications. Handles in-memory connection between apps.
+ There are several ways to pass parameters to init the app. *args can be :
+ - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
+ (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
+ - string, App or Output, useful when the user wants to specify the input "in"
+ - list, useful when the user wants to specify the input list 'il'
+
Args:
appname: name of the OTB application to initialize, e.g. 'BandMath'
- *args: used for passing application parameters. Can be :
- - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
- (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
- - string, App or Output, useful when the user wants to specify the input "in"
- - list, useful when the user wants to specify the input list 'il'
+ *args: used to pass an app input as argument and ommiting the key
frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___
quiet: whether to print logs of the OTB app
name: custom name that will show up in logs, appname will be used if not provided
- **kwargs: used for passing application parameters.
- e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif'
+ **kwargs: used for passing application parameters (e.g. il=["image_1.tif", "image_1.tif"])
"""
# Attributes and data structures used by properties
@@ -700,10 +701,8 @@ class App(OTBObject):
instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths
Args:
- *args: Can be : - dictionary containing key-arguments enumeration. Useful when a keyword is reserved (e.g. "in")
- - string or OTBObject, useful when the user implicitly wants to set the param "in"
- - list, useful when the user implicitly wants to set the param "il"
- **kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif'
+ *args: any input OTBObject, filepath or images list, or a dict of parameters
+ **kwargs: app parameters, with "_" instead of dots e.g. io_in="image.tif"
Raises:
KeyError: when the parameter name wasn't recognized
@@ -818,20 +817,19 @@ class App(OTBObject):
) -> bool:
"""Set output pixel type and write the output raster files.
+ The first argument is expected to be:
+ - filepath, useful when there is only one output, e.g. 'output.tif'
+ - dictionary containing output filepath
+ - None if output file was passed during App init
+ In case of multiple outputs, pixel_type may also be a dictionary with parameter names as keys.
+ Accepted pixel types : uint8, uint16, uint32, int16, int32, float, double, cint16, cint32, cfloat, cdouble
+
Args:
- path: Can be : - filepath, useful when there is only one output, e.g. 'output.tif'
- - dictionary containing key-arguments enumeration. Useful when a key contains
- non-standard characters such as a point, e.g. {'io.out':'output.tif'}
- - None if output file was passed during App init
- pixel_type: Can be : - dictionary {out_param_key: pixeltype} when specifying for several outputs
- - str (e.g. 'uint16') or otbApplication.ImagePixelType_... When there are several
- outputs, all outputs are written with this unique type.
- Valid pixel types are uint8, uint16, uint32, int16, int32, float, double,
- cint16, cint32, cfloat, cdouble. (Default value = None)
+ path: output filepath or dict of filepath with param keys
+ pixel_type: pixel type string representation
preserve_dtype: propagate main input pixel type to outputs, in case pixel_type is None
- ext_fname: Optional, an extended filename as understood by OTB (e.g. "&gdal:co:TILED=YES")
- Will be used for all outputs (Default value = None)
- **kwargs: keyword arguments e.g. out='output.tif'
+ ext_fname: an OTB extended filename, will be applied to every output (but won't overwrite existing keys in output filepath)
+ **kwargs: keyword arguments e.g. out='output.tif' or io_out='output.tif'
Returns:
True if all files are found on disk
@@ -1125,7 +1123,7 @@ class Slicer(App):
obj: input
rows: slice along Y / Latitude axis
cols: slice along X / Longitude axis
- channels: bands to extract. can be slicing, list or int
+ channels: bands to extract
Raises:
TypeError: if channels param isn't slice, list or int
@@ -1218,7 +1216,7 @@ class Operation(App):
Args:
operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
- *inputs: operands, can be OTBObject, filepath, int or float
+ *inputs: operands of the expression to build
nb_bands: optionally specify the output nb of bands - used only internally by pyotb.where
name: override the default Operation name
@@ -1292,7 +1290,7 @@ class Operation(App):
E.g for the operation input1 + input2, we create a fake expression that is like "str(input1) + str(input2)"
Args:
- operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
+ operator: one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
inputs: inputs. Can be OTBObject, filepath, int or float
nb_bands: to specify the output nb of bands. Optional. Used only internally by pyotb.where
@@ -1375,14 +1373,14 @@ class Operation(App):
"""This an internal function, only to be used by `build_fake_expressions`.
Enable to create a fake expression just for one input and one band.
+ Regarding the "keep_logical" param:
+ - if True, for `input1 > input2`, returned fake expression is "str(input1) > str(input2)"
+ - if False, for `input1 > input2`, returned fake exp is "str(input1) > str(input2) ? 1 : 0"] Default False
Args:
x: input
band: which band to consider (bands start at 1)
keep_logical: whether to keep the logical expressions "as is" in case the input is a logical operation.
- ex: if True, for `input1 > input2`, returned fake expression is "str(input1) > str(input2)"
- if False, for `input1 > input2`, returned fake exp is "str(input1) > str(input2) ? 1 : 0"]
- Default False
Returns:
fake_exp: the fake expression for this band and input
@@ -1498,7 +1496,7 @@ class Input(App):
"""Default constructor.
Args:
- filepath: Anything supported by GDAL (local file on the filesystem, remote resource e.g. /vsicurl/.., etc.)
+ filepath: Anything supported by GDAL (local file on the filesystem, remote resource, etc.)
"""
super().__init__("ExtractROI", {"in": filepath}, quiet=True, frozen=True)
@@ -1625,7 +1623,7 @@ def get_nbchannels(inp: str | Path | OTBObject) -> int:
"""Get the nb of bands of input image.
Args:
- inp: can be filepath or OTBObject object
+ inp: input file or OTBObject
Returns:
number of bands in image
@@ -1651,14 +1649,16 @@ def get_nbchannels(inp: str | Path | OTBObject) -> int:
def get_pixel_type(inp: str | Path | OTBObject) -> str:
- """Get the encoding of input image pixels.
+ """Get the encoding of input image pixels as integer enum.
+
+ OTB enum e.g. `otbApplication.ImagePixelType_uint8'.
+ For an OTBObject with several outputs, only the pixel type of the first output is returned
Args:
- inp: can be filepath or pyotb object
+ inp: input file or OTBObject
Returns:
- pixel_type: OTB enum e.g. `otbApplication.ImagePixelType_uint8', which actually is an int.
- For an OTBObject with several outputs, only the pixel type of the first output is returned
+ OTB enum
Raises:
TypeError: if inp pixel type cannot be retrieved
@@ -1685,7 +1685,7 @@ def parse_pixel_type(pixel_type: str | int) -> int:
"""Convert one str pixel type to OTB integer enum if necessary.
Args:
- pixel_type: pixel type. can be int or str (either OTB or numpy convention)
+ pixel_type: pixel type to parse
Returns:
pixel_type OTB enum integer value
@@ -1739,21 +1739,19 @@ def summarize(
At the deepest recursion level, this function just return any parameter value,
path stripped if needed, or app summarized in case of a pipeline.
+ If strip_path is enabled, paths are truncated after the first "?" character.
+ Can be useful to remove URLs tokens from inputs (e.g. SAS or S3 credentials),
+ or extended filenames from outputs.
Args:
obj: input object / parameter value to summarize
- strip_inpath: strip all input paths: If enabled, paths related to
- inputs are truncated after the first "?" character. Can be useful
- to remove URLs tokens (e.g. SAS or S3 credentials).
- strip_outpath: strip all output paths: If enabled, paths related
- to outputs are truncated after the first "?" character. Can be
- useful to remove extended filenames.
+ strip_inpath: strip all input paths
+ strip_outpath: strip all output paths
Returns:
nested dictionary containing name and parameters of an app and its parents
"""
- # This is the deepest recursion level
if isinstance(obj, list):
return [summarize(o) for o in obj]
if isinstance(obj, Output):
--
GitLab
From d729f02568fc4949538be2b413630efc418c0976 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Nov 2023 15:11:40 +0100
Subject: [PATCH 376/399] STYLE: apply black
---
pyotb/core.py | 12 ++++--------
1 file changed, 4 insertions(+), 8 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 5528039..5c33706 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1081,9 +1081,7 @@ class App(OTBObject):
self.data[key] = value
# Special functions
- def __getitem__(
- self, key: str | tuple
- ) -> Any | list[float] | float | Slicer:
+ def __getitem__(self, key: str | tuple) -> Any | list[float] | float | Slicer:
"""This function is called when we use App()[...].
We allow to return attr if key is a parameter, or call OTBObject __getitem__ for pixel values or Slicer
@@ -1270,9 +1268,7 @@ class Operation(App):
return 1
# Check that all inputs have the same band count
nb_bands_list = [
- get_nbchannels(inp)
- for inp in inputs
- if not isinstance(inp, (float, int))
+ get_nbchannels(inp) for inp in inputs if not isinstance(inp, (float, int))
]
all_same = all(x == nb_bands_list[0] for x in nb_bands_list)
if len(nb_bands_list) > 1 and not all_same:
@@ -1373,7 +1369,7 @@ class Operation(App):
"""This an internal function, only to be used by `build_fake_expressions`.
Enable to create a fake expression just for one input and one band.
- Regarding the "keep_logical" param:
+ Regarding the "keep_logical" param:
- if True, for `input1 > input2`, returned fake expression is "str(input1) > str(input2)"
- if False, for `input1 > input2`, returned fake exp is "str(input1) > str(input2) ? 1 : 0"] Default False
@@ -1689,7 +1685,7 @@ def parse_pixel_type(pixel_type: str | int) -> int:
Returns:
pixel_type OTB enum integer value
-
+
Raises:
KeyError: if pixel_type name is unknown
TypeError: if type(pixel_type) isn't int or str
--
GitLab
From 09bfa60e027fb6bc7fd6b036e2d97649e4eb5952 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Nov 2023 15:17:17 +0100
Subject: [PATCH 377/399] DOC: typo
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 5c33706..b6655f3 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -560,7 +560,7 @@ class App(OTBObject):
Args:
appname: name of the OTB application to initialize, e.g. 'BandMath'
- *args: used to pass an app input as argument and ommiting the key
+ *args: used to pass an app input as argument and omitting the key
frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___
quiet: whether to print logs of the OTB app
name: custom name that will show up in logs, appname will be used if not provided
--
GitLab
From 3ee79f5672397fd4d985a7c43f96d9c6a0182482 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Nov 2023 15:38:43 +0100
Subject: [PATCH 378/399] DOC: test move docs from __init__ to class docstring
---
pyotb/core.py | 37 +++++++++++++++++++------------------
1 file changed, 19 insertions(+), 18 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index b6655f3..f578a66 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -513,7 +513,24 @@ class OTBObject(ABC):
class App(OTBObject):
- """Base class that gathers common operations for any OTB application."""
+ """Wrapper around otb.Application to handle settings and execution.
+
+ Base class that gathers common operations for any OTB application lifetime (settings, exec, export, etc.)
+ There are several ways to pass parameters to init the app. *args can be :
+ - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
+ (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
+ - string, App or Output, useful when the user wants to specify the input "in"
+ - list, useful when the user wants to specify the input list 'il'
+
+ Args:
+ appname: name of the OTB application to initialize, e.g. 'BandMath'
+ *args: used to pass an app input as argument and omitting the key
+ frozen: freeze OTB app in order avoid blocking during __init___
+ quiet: whether to print logs of the OTB app
+ name: custom name that will show up in logs, appname will be used if not provided
+ **kwargs: used for passing application parameters (e.g. il=["image_1.tif", "image_1.tif"])
+
+ """
INPUT_IMAGE_TYPES = [
otb.ParameterType_InputImage,
@@ -550,23 +567,7 @@ class App(OTBObject):
name: str = "",
**kwargs,
):
- """Common constructor for OTB applications. Handles in-memory connection between apps.
-
- There are several ways to pass parameters to init the app. *args can be :
- - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
- (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
- - string, App or Output, useful when the user wants to specify the input "in"
- - list, useful when the user wants to specify the input list 'il'
-
- Args:
- appname: name of the OTB application to initialize, e.g. 'BandMath'
- *args: used to pass an app input as argument and omitting the key
- frozen: freeze OTB app in order to use execute() later and avoid blocking process during __init___
- quiet: whether to print logs of the OTB app
- name: custom name that will show up in logs, appname will be used if not provided
- **kwargs: used for passing application parameters (e.g. il=["image_1.tif", "image_1.tif"])
-
- """
+ """Common constructor for OTB applications, automatically handles in-memory connections."""
# Attributes and data structures used by properties
create = (
otb.Registry.CreateApplicationWithoutLogger
--
GitLab
From d44ffae4691678b535c067effa31a3d4677e5371 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Nov 2023 15:51:57 +0100
Subject: [PATCH 379/399] ENH: App docstring
---
pyotb/core.py | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index f578a66..d6a60d6 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -16,7 +16,7 @@ from .depreciation import deprecated_alias, depreciation_warning, deprecated_att
class OTBObject(ABC):
- """Abstraction of an image object."""
+ """Abstraction of an image object, for a whole app or one specific output."""
@property
@abstractmethod
@@ -269,7 +269,7 @@ class OTBObject(ABC):
y: latitude or projected Y
Returns:
- pixel index: (row, col)
+ pixel index as (row, col)
"""
spacing_x, _, origin_x, _, spacing_y, origin_y = self.transform
@@ -516,17 +516,17 @@ class App(OTBObject):
"""Wrapper around otb.Application to handle settings and execution.
Base class that gathers common operations for any OTB application lifetime (settings, exec, export, etc.)
- There are several ways to pass parameters to init the app. *args can be :
- - dictionary containing key-arguments enumeration. Useful when a key is python-reserved
- (e.g. "in") or contains reserved characters such as a point (e.g."mode.extent.unit")
- - string, App or Output, useful when the user wants to specify the input "in"
- - list, useful when the user wants to specify the input list 'il'
+ Any app parameter may be passed either using a dict of parameters or keyword argument.
+ The first argument can be :
+ - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in")
+ - string, App or Output, useful when the user wants to specify the input "in"
+ - list, useful when the user wants to specify the input list 'il'
Args:
appname: name of the OTB application to initialize, e.g. 'BandMath'
*args: used to pass an app input as argument and omitting the key
frozen: freeze OTB app in order avoid blocking during __init___
- quiet: whether to print logs of the OTB app
+ quiet: whether to print logs of the OTB app and the default progress bar
name: custom name that will show up in logs, appname will be used if not provided
**kwargs: used for passing application parameters (e.g. il=["image_1.tif", "image_1.tif"])
--
GitLab
From 12881408bcb8308f9834d0df6e9eaebb6ed3e147 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Nov 2023 15:57:02 +0100
Subject: [PATCH 380/399] DOC: enh class docstring
---
pyotb/core.py | 106 ++++++++++++++++++++++++--------------------------
1 file changed, 51 insertions(+), 55 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index d6a60d6..29535f3 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -517,7 +517,7 @@ class App(OTBObject):
Base class that gathers common operations for any OTB application lifetime (settings, exec, export, etc.)
Any app parameter may be passed either using a dict of parameters or keyword argument.
- The first argument can be :
+ The first argument can be :
- dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in")
- string, App or Output, useful when the user wants to specify the input "in"
- list, useful when the user wants to specify the input list 'il'
@@ -1103,7 +1103,22 @@ class App(OTBObject):
class Slicer(App):
- """Slicer objects i.e. when we call something like raster[:, :, 2] from Python."""
+ """Slicer objects, automatically created when using slicing e.g. app[:, :, 2].
+
+ It contains :
+ - an ExtractROI app that handles extracting bands and ROI and can be written to disk or used in pipelines
+ - in case the user only wants to extract one band, an expression such as "im1b#"
+
+ Args:
+ obj: input
+ rows: slice along Y / Latitude axis
+ cols: slice along X / Longitude axis
+ channels: bands to extract
+
+ Raises:
+ TypeError: if channels param isn't slice, list or int
+
+ """
def __init__(
self,
@@ -1112,22 +1127,7 @@ class Slicer(App):
cols: slice,
channels: slice | list[int] | int,
):
- """Create a slicer object, that can be used directly for writing or inside a BandMath.
-
- It contains :
- - an ExtractROI app that handles extracting bands and ROI and can be written to disk or used in pipelines
- - in case the user only wants to extract one band, an expression such as "im1b#"
-
- Args:
- obj: input
- rows: slice along Y / Latitude axis
- cols: slice along X / Longitude axis
- channels: bands to extract
-
- Raises:
- TypeError: if channels param isn't slice, list or int
-
- """
+ """Create a slicer object, that can be used directly for writing or inside a BandMath."""
super().__init__(
"ExtractROI",
obj,
@@ -1189,6 +1189,16 @@ class Slicer(App):
class Operation(App):
"""Class for arithmetic/math operations done in Python.
+ Given some inputs and an operator, this object enables to python operator to a BandMath operation.
+ Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator.
+ It can have 3 inputs for the ternary operator `cond ? x : y`.
+
+ Args:
+ operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
+ *inputs: operands of the expression to build
+ nb_bands: optionally specify the output nb of bands - used only internally by pyotb.where
+ name: override the default Operation name
+
Example:
Consider the python expression (input1 + 2 * input2) > 0.
This class enables to create a BandMathX app, with expression such as (im2 + 2 * im1) > 0 ? 1 : 0
@@ -1208,18 +1218,7 @@ class Operation(App):
"""
def __init__(self, operator: str, *inputs, nb_bands: int = None, name: str = None):
- """Given some inputs and an operator, this object enables to python operator to a BandMath operation.
-
- Operations generally involve 2 inputs (+, -...). It can have only 1 input for `abs` operator.
- It can have 3 inputs for the ternary operator `cond ? x : y`.
-
- Args:
- operator: (str) one of +, -, *, /, >, <, >=, <=, ==, !=, &, |, abs, ?
- *inputs: operands of the expression to build
- nb_bands: optionally specify the output nb of bands - used only internally by pyotb.where
- name: override the default Operation name
-
- """
+ """Operation constructor, one part of the logic is handled by App.__create_operator"""
self.operator = operator
# We first create a 'fake' expression. E.g for the operation `input1 + input2`
# we create a fake expression like "str(input1) + str(input2)"
@@ -1426,21 +1425,18 @@ class LogicalOperation(Operation):
"""A specialization of Operation class for boolean logical operations.
Supported operators are >, <, >=, <=, ==, !=, `&` and `|`.
-
The only difference is that not only the BandMath expression is saved (e.g. "im1b1 > 0 ? 1 : 0"),
but also the logical expression (e.g. "im1b1 > 0")
+ Args:
+ operator: string operator (one of >, <, >=, <=, ==, !=, &, |)
+ *inputs: inputs
+ nb_bands: optionally specify the output nb of bands - used only by pyotb.where
+
"""
def __init__(self, operator: str, *inputs, nb_bands: int = None):
- """Constructor for a LogicalOperation object.
-
- Args:
- operator: string operator (one of >, <, >=, <=, ==, !=, &, |)
- *inputs: inputs
- nb_bands: optionally specify the output nb of bands - used only by pyotb.where
-
- """
+ """Constructor for a LogicalOperation object."""
self.logical_fake_exp_bands = []
super().__init__(operator, *inputs, nb_bands=nb_bands, name="LogicalOperation")
self.logical_exp_bands, self.logical_exp = self.get_real_exp(
@@ -1487,15 +1483,15 @@ class LogicalOperation(Operation):
class Input(App):
- """Class for transforming a filepath to pyOTB object."""
+ """Class for transforming a filepath to pyOTB object.
- def __init__(self, filepath: str):
- """Default constructor.
+ Args:
+ filepath: Anything supported by GDAL (local file on the filesystem, remote resource, etc.)
- Args:
- filepath: Anything supported by GDAL (local file on the filesystem, remote resource, etc.)
+ """
- """
+ def __init__(self, filepath: str):
+ """Initialize an ExtractROI OTB app from a filepath, set dtype and store filepath."""
super().__init__("ExtractROI", {"in": filepath}, quiet=True, frozen=True)
self._name = f"Input from {filepath}"
if not filepath.startswith(("/vsi", "http://", "https://", "ftp://")):
@@ -1510,7 +1506,15 @@ class Input(App):
class Output(OTBObject):
- """Object that behave like a pointer to a specific application in-memory output or file."""
+ """Object that behave like a pointer to a specific application in-memory output or file.
+
+ Args:
+ pyotb_app: The pyotb App to store reference from
+ param_key: Output parameter key of the target app
+ filepath: path of the output file (if not memory)
+ mkdir: create missing parent directories
+
+ """
_filepath: str | Path = None
@@ -1522,15 +1526,7 @@ class Output(OTBObject):
filepath: str = None,
mkdir: bool = True,
):
- """Constructor for an Output object.
-
- Args:
- pyotb_app: The pyotb App to store reference from
- param_key: Output parameter key of the target app
- filepath: path of the output file (if not memory)
- mkdir: create missing parent directories
-
- """
+ """Constructor for an Output object, initialized during App.__init__."""
self.parent_pyotb_app = pyotb_app # keep trace of parent app
self.param_key = param_key
self.filepath = filepath
--
GitLab
From 212e40feebade49a6cb65a35bd788fb3805050a5 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Nov 2023 16:04:18 +0100
Subject: [PATCH 381/399] DOC: format indent to fix bulletpoints
---
pyotb/core.py | 20 +++++++++-----------
1 file changed, 9 insertions(+), 11 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 29535f3..7f60d0b 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -467,10 +467,8 @@ class OTBObject(ABC):
"""Override the default __getitem__ behaviour.
This function enables 2 things :
- - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3]
- selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]]
- selecting 1000x1000 subset : object[:1000, :1000]
- - access pixel value(s) at a specified row, col index
+ - slicing, i.e. selecting ROI/bands
+ - access pixel value(s) at a specified row, col index
Args:
key: attribute key
@@ -517,10 +515,10 @@ class App(OTBObject):
Base class that gathers common operations for any OTB application lifetime (settings, exec, export, etc.)
Any app parameter may be passed either using a dict of parameters or keyword argument.
- The first argument can be :
- - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in")
- - string, App or Output, useful when the user wants to specify the input "in"
- - list, useful when the user wants to specify the input list 'il'
+ The first argument can be:
+ - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in")
+ - string, App or Output, useful when the user wants to specify the input "in"
+ - list, useful when the user wants to specify the input list 'il'
Args:
appname: name of the OTB application to initialize, e.g. 'BandMath'
@@ -1105,9 +1103,9 @@ class App(OTBObject):
class Slicer(App):
"""Slicer objects, automatically created when using slicing e.g. app[:, :, 2].
- It contains :
- - an ExtractROI app that handles extracting bands and ROI and can be written to disk or used in pipelines
- - in case the user only wants to extract one band, an expression such as "im1b#"
+ It contains:
+ - an ExtractROI app that handles extracting bands and ROI and can be written to disk or used in pipelines
+ - in case the user only wants to extract one band, an expression such as "im1b#"
Args:
obj: input
--
GitLab
From 36d0892b9303c6a948e8a3af16465898f0885ce7 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Nov 2023 16:17:21 +0100
Subject: [PATCH 382/399] DOC: enh docstrings with bulletpoints
---
pyotb/core.py | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 7f60d0b..47109da 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -515,6 +515,7 @@ class App(OTBObject):
Base class that gathers common operations for any OTB application lifetime (settings, exec, export, etc.)
Any app parameter may be passed either using a dict of parameters or keyword argument.
+
The first argument can be:
- dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in")
- string, App or Output, useful when the user wants to specify the input "in"
@@ -820,6 +821,7 @@ class App(OTBObject):
- filepath, useful when there is only one output, e.g. 'output.tif'
- dictionary containing output filepath
- None if output file was passed during App init
+
In case of multiple outputs, pixel_type may also be a dictionary with parameter names as keys.
Accepted pixel types : uint8, uint16, uint32, int16, int32, float, double, cint16, cint32, cfloat, cdouble
@@ -1103,9 +1105,8 @@ class App(OTBObject):
class Slicer(App):
"""Slicer objects, automatically created when using slicing e.g. app[:, :, 2].
- It contains:
- - an ExtractROI app that handles extracting bands and ROI and can be written to disk or used in pipelines
- - in case the user only wants to extract one band, an expression such as "im1b#"
+ Can be used to select a subset of pixel and / or bands in the image.
+ This is a shortcut to an ExtractROI app that can be written to disk or used in pipelines.
Args:
obj: input
@@ -1423,8 +1424,8 @@ class LogicalOperation(Operation):
"""A specialization of Operation class for boolean logical operations.
Supported operators are >, <, >=, <=, ==, !=, `&` and `|`.
- The only difference is that not only the BandMath expression is saved (e.g. "im1b1 > 0 ? 1 : 0"),
- but also the logical expression (e.g. "im1b1 > 0")
+ The only difference is that not only the BandMath expression is saved
+ (e.g. "im1b1 > 0 ? 1 : 0"), but also the logical expression (e.g. "im1b1 > 0")
Args:
operator: string operator (one of >, <, >=, <=, ==, !=, &, |)
@@ -1449,8 +1450,8 @@ class LogicalOperation(Operation):
):
"""Create a list of 'fake' expressions, one for each band.
- e.g for the operation input1 > input2, we create a fake expression that is like
- "str(input1) > str(input2) ? 1 : 0" and a logical fake expression that is like "str(input1) > str(input2)"
+ For the operation input1 > input2, we create a fake expression like `str(input1) > str(input2) ? 1 : 0`
+ and a logical fake expression like `str(input1) > str(input2)`
Args:
operator: str (one of >, <, >=, <=, ==, !=, &, |)
@@ -1630,9 +1631,7 @@ def get_nbchannels(inp: str | Path | OTBObject) -> int:
try:
info = App("ReadImageInfo", inp, quiet=True)
return info["numberbands"]
- except (
- RuntimeError
- ) as info_err: # this happens when we pass a str that is not a filepath
+ except RuntimeError as info_err: # e.g. file is missing
raise TypeError(
f"Could not get the number of channels file '{inp}' ({info_err})"
) from info_err
--
GitLab
From 8a131d01d0b642a8971489763973a77821a4b422 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Nov 2023 16:23:29 +0100
Subject: [PATCH 383/399] DOC: add missing dot
---
pyotb/core.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 47109da..346dc31 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1217,7 +1217,7 @@ class Operation(App):
"""
def __init__(self, operator: str, *inputs, nb_bands: int = None, name: str = None):
- """Operation constructor, one part of the logic is handled by App.__create_operator"""
+ """Operation constructor, one part of the logic is handled by App.__create_operator."""
self.operator = operator
# We first create a 'fake' expression. E.g for the operation `input1 + input2`
# we create a fake expression like "str(input1) + str(input2)"
--
GitLab
From b16538db952eec7e012816f4e7d88993c1860d8a Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Nov 2023 16:37:59 +0100
Subject: [PATCH 384/399] ENH: docstrings
---
pyotb/core.py | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 346dc31..bc6529e 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -517,9 +517,9 @@ class App(OTBObject):
Any app parameter may be passed either using a dict of parameters or keyword argument.
The first argument can be:
- - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in")
- - string, App or Output, useful when the user wants to specify the input "in"
+ - string, App or Output, the main input parameter name is automatically set
- list, useful when the user wants to specify the input list 'il'
+ - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in", "map")
Args:
appname: name of the OTB application to initialize, e.g. 'BandMath'
@@ -695,8 +695,10 @@ class App(OTBObject):
return self._time_end - self._time_start
def set_parameters(self, *args, **kwargs):
- """Set some parameters of the app.
+ """Set parameters, using the right OTB API function depending on the key and type.
+ Parameters with dots may be passed as keyword arguments using "_", e.g. map_epsg_code=4326.
+ Additional checks are done for input and output (in-memory objects, remote filepaths, etc.).
When useful, e.g. for images list, this function appends the parameters
instead of overwriting them. Handles any parameters, i.e. in-memory & filepaths
@@ -1482,7 +1484,7 @@ class LogicalOperation(Operation):
class Input(App):
- """Class for transforming a filepath to pyOTB object.
+ """Class for transforming a filepath to pyotb object.
Args:
filepath: Anything supported by GDAL (local file on the filesystem, remote resource, etc.)
--
GitLab
From 238dce9911046b714eab8baea38ed24dedaeeaf1 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Nov 2023 16:49:11 +0100
Subject: [PATCH 385/399] ENH: App docstring
---
pyotb/core.py | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index bc6529e..8afd168 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -517,17 +517,18 @@ class App(OTBObject):
Any app parameter may be passed either using a dict of parameters or keyword argument.
The first argument can be:
- - string, App or Output, the main input parameter name is automatically set
- - list, useful when the user wants to specify the input list 'il'
- - dictionary containing key-arguments enumeration. Useful when a key is python-reserved (e.g. "in", "map")
+ - filepath or OTBObject, the main input parameter name is automatically used
+ - list of inputs, useful when the user wants to specify the input list `il`
+ - dictionary of parameters, useful when a key is python-reserved (e.g. `in`, `map`)
+ Any key except reserved keywards may also be passed via kwargs, if you replace dots with "_" e.g `map_epsg_code=4326`
Args:
appname: name of the OTB application to initialize, e.g. 'BandMath'
- *args: used to pass an app input as argument and omitting the key
+ *args: can be a filepath, OTB object or a dict or parameters, several dicts will be merged in **kwargs
frozen: freeze OTB app in order avoid blocking during __init___
quiet: whether to print logs of the OTB app and the default progress bar
name: custom name that will show up in logs, appname will be used if not provided
- **kwargs: used for passing application parameters (e.g. il=["image_1.tif", "image_1.tif"])
+ **kwargs: any OTB application parameter key is accepted except "in"
"""
--
GitLab
From e55ec155aa0ca9338a66c02d53b6341a081c2fbb Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Nov 2023 17:00:25 +0100
Subject: [PATCH 386/399] STYLE: use comprehensive list instead of filter +
whitespace
---
pyotb/core.py | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 8afd168..2b7af8c 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -581,6 +581,7 @@ class App(OTBObject):
self._time_start, self._time_end = 0.0, 0.0
self.data, self.outputs = {}, {}
self.quiet, self.frozen = quiet, frozen
+
# Param keys and types
self.parameters_keys = tuple(self.app.GetParametersKeys())
self._all_param_types = {
@@ -596,15 +597,18 @@ class App(OTBObject):
for key in self.parameters_keys
if self.app.GetParameterType(key) == otb.ParameterType_Choice
}
+
# Init, execute and write (auto flush only when output param was provided)
if args or kwargs:
self.set_parameters(*args, **kwargs)
# Create Output image objects
- for key in filter(
- lambda k: self._out_param_types[k] == otb.ParameterType_OutputImage,
- self._out_param_types,
+ for key in (
+ key
+ for key, param in self._out_param_types.items()
+ if param == otb.ParameterType_OutputImage
):
self.outputs[key] = Output(self, key, self._settings.get(key))
+
if not self.frozen:
self.execute()
if any(key in self._settings for key in self._out_param_types):
--
GitLab
From e18bcfc3394a4b1cca5fbe4c22eaee74cb328cf2 Mon Sep 17 00:00:00 2001
From: Vincent Delbar <vincent.delbar@latelescop.fr>
Date: Mon, 6 Nov 2023 18:37:43 +0100
Subject: [PATCH 387/399] CI: typo
---
pyotb/core.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyotb/core.py b/pyotb/core.py
index 2b7af8c..9cd9d4e 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -520,7 +520,7 @@ class App(OTBObject):
- filepath or OTBObject, the main input parameter name is automatically used
- list of inputs, useful when the user wants to specify the input list `il`
- dictionary of parameters, useful when a key is python-reserved (e.g. `in`, `map`)
- Any key except reserved keywards may also be passed via kwargs, if you replace dots with "_" e.g `map_epsg_code=4326`
+ Any key except "in" or "map" can also be passed via kwargs, replace "." with "_" e.g `map_epsg_code=4326`
Args:
appname: name of the OTB application to initialize, e.g. 'BandMath'
@@ -528,7 +528,7 @@ class App(OTBObject):
frozen: freeze OTB app in order avoid blocking during __init___
quiet: whether to print logs of the OTB app and the default progress bar
name: custom name that will show up in logs, appname will be used if not provided
- **kwargs: any OTB application parameter key is accepted except "in"
+ **kwargs: any OTB application parameter key is accepted except "in" or "map"
"""
--
GitLab
From ccb4fe2a073604079fabf8cd1145144d394f1b67 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 8 Nov 2023 17:07:20 +0100
Subject: [PATCH 388/399] DOC: add sumarize() documentation
---
doc/index.md | 1 +
doc/summarize.md | 167 +++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 168 insertions(+)
create mode 100644 doc/summarize.md
diff --git a/doc/index.md b/doc/index.md
index 7861878..d76ef33 100644
--- a/doc/index.md
+++ b/doc/index.md
@@ -22,6 +22,7 @@ to make OTB more Python friendly.
## Advanced use
- [Comparison between pyotb and OTB native library](comparison_otb.md)
+- [Summarize applications](summarize.md)
- [OTB versions](otb_versions.md)
- [Managing loggers](managing_loggers.md)
- [Troubleshooting & limitations](troubleshooting.md)
diff --git a/doc/summarize.md b/doc/summarize.md
new file mode 100644
index 0000000..13f3021
--- /dev/null
+++ b/doc/summarize.md
@@ -0,0 +1,167 @@
+## Summarize applications
+
+pyotb enables to summarize applications as a dictionary with keys/values for
+parameters. This feature can be used to keep track of a process, composed of
+multiple applications chained together.
+
+### Single application
+
+Let's take the example of one single application.
+
+```python
+import pyotb
+
+app = pyotb.RigidTransformResample({
+ 'in': 'my_image.tif',
+ 'interpolator': 'linear',
+ 'transform.type.id.scaley': 0.5,
+ 'transform.type.id.scalex': 0.5
+})
+```
+
+The application can be summarized using `pyotb.summarize()` or
+`app.summary()`, which are equivalent.
+
+```python
+print(app.summarize())
+```
+
+Results in the following (lines have been pretty printed for the sake of
+documentation):
+
+```json lines
+{
+ 'name': 'RigidTransformResample',
+ 'parameters': {
+ 'transform.type': 'id',
+ 'in': 'my_image.tif',
+ 'interpolator': 'linear',
+ 'transform.type.id.scaley': 0.5,
+ 'transform.type.id.scalex': 0.5
+ }
+}
+```
+
+Note that we can also summarize an application after it has been executed:
+
+```python
+app.write('output.tif', pixel_type='uint16')
+print(app.summarize())
+```
+
+Which results in the following:
+
+```json lines
+{
+ 'name': 'RigidTransformResample',
+ 'parameters': {
+ 'transform.type': 'id',
+ 'in': 'my_image.tif',
+ 'interpolator': 'linear',
+ 'transform.type.id.scaley': 0.5,
+ 'transform.type.id.scalex': 0.5,
+ 'out': 'output.tif'
+ }
+}
+```
+
+Now `'output.tif'` has been added to the application parameters.
+
+### Multiple applications chained together (pipeline)
+
+When multiple applications are chained together, the summary of the last
+application will describe all upstream processes.
+
+```python
+import pyotb
+
+app1 = pyotb.RigidTransformResample({
+ 'in': 'my_image.tif',
+ 'interpolator': 'linear',
+ 'transform.type.id.scaley': 0.5,
+ 'transform.type.id.scalex': 0.5
+})
+app2 = pyotb.Smoothing(app1)
+print(app2.summarize())
+```
+
+Results in:
+
+```json lines
+{
+ 'name': 'Smoothing',
+ 'parameters': {
+ 'type': 'anidif',
+ 'type.anidif.timestep': 0.125,
+ 'type.anidif.nbiter': 10,
+ 'type.anidif.conductance': 1.0,
+ 'in': {
+ 'name': 'RigidTransformResample',
+ 'parameters': {
+ 'transform.type': 'id',
+ 'in': 'my_image.tif',
+ 'interpolator': 'linear',
+ 'transform.type.id.scaley': 0.5,
+ 'transform.type.id.scalex': 0.5
+ }
+ }
+ }
+}
+```
+
+### Remote files URL stripping
+
+Cloud-based raster URLs often include tokens or random strings resulting from
+the URL signing.
+Those can be removed from the summarized paths, using the `strip_inpath`
+and/or `strip_outpath` arguments respectively for inputs and/or outputs.
+
+Here is an example with Microsoft Planetary Computer:
+
+```python
+import planetary_computer
+import pyotb
+
+url = (
+ "https://sentinel2l2a01.blob.core.windows.net/sentinel2-l2/31/N/EA/2023/"
+ "11/03/S2A_MSIL2A_20231103T095151_N0509_R079_T31NEA_20231103T161409.SAFE/"
+ "GRANULE/L2A_T31NEA_A043691_20231103T100626/IMG_DATA/R10m/T31NEA_20231103"
+ "T095151_B02_10m.tif"
+)
+signed_url = planetary_computer.sign_inplace(url)
+app = pyotb.Smoothing(signed_url)
+```
+
+By default, the summary does not strip the URL.
+
+```python
+print(app.summarize()["parameters"]["in"])
+```
+
+This results in:
+
+```
+/vsicurl/https://sentinel2l2a01.blob.core.windows.net/sentinel2-l2/31/N/EA/...
+2023/11/03/S2A_MSIL2A_20231103T095151_N0509_R079_T31NEA_20231103T161409.SAFE...
+/GRANULE/L2A_T31NEA_A043691_20231103T100626/IMG_DATA/R10m/T31NEA_20231103T...
+095151_B02_10m.tif?st=2023-11-07T15%3A52%3A47Z&se=2023-11-08T16%3A37%3A47Z&...
+sp=rl&sv=2021-06-08&sr=c&skoid=c85c15d6-d1ae-42d4-af60-e2ca0f81359b&sktid=...
+72f988bf-86f1-41af-91ab-2d7cd011db47&skt=2023-11-08T11%3A41%3A41Z&ske=2023-...
+11-15T11%3A41%3A41Z&sks=b&skv=2021-06-08&sig=xxxxxxxxxxx...xxxxx
+```
+
+Now we can strip the URL to keep only the resource identifier and get rid of
+the token:
+
+```python
+print(app.summarize(strip_inpath=True)["parameters"]["in"])
+```
+
+Which now results in:
+
+```
+/vsicurl/https://sentinel2l2a01.blob.core.windows.net/sentinel2-l2/31/N/EA/...
+2023/11/03/S2A_MSIL2A_20231103T095151_N0509_R079_T31NEA_20231103T161409.SAFE...
+/GRANULE/L2A_T31NEA_A043691_20231103T100626/IMG_DATA/R10m/T31NEA_20231103T...
+095151_B02_10m.tif
+```
\ No newline at end of file
--
GitLab
From bd2bbc419dc80b584886773f293c183c47101786 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 8 Nov 2023 18:11:20 +0100
Subject: [PATCH 389/399] DOC: features (dtype, transform, etc)
---
doc/features.md | 58 ++++++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 52 insertions(+), 6 deletions(-)
diff --git a/doc/features.md b/doc/features.md
index 2058af5..6af1355 100644
--- a/doc/features.md
+++ b/doc/features.md
@@ -32,18 +32,64 @@ inp = pyotb.Input('my_image.tif')
inp[:, :, :3] # selecting first 3 bands
inp[:, :, [0, 1, 4]] # selecting bands 1, 2 & 5
-inp[:1000, :1000] # selecting 1000x1000 subset, same as inp[:1000, :1000, :]
+inp[:, :, 1:-1] # removing first and last band
+inp[:, :, ::2] # selecting one band every 2 bands
+inp[:100, :100] # selecting 100x100 subset, same as inp[:100, :100, :]
inp[:100, :100].write('my_image_roi.tif') # write cropped image to disk
```
-## Shape attributes
+## Retrieving a pixel location in image coordinates
-You can access the shape of any in-memory pyotb object.
+One can retrieve a pixel location in image coordinates (i.e. row and column
+indices) using `get_rowcol_from_xy()`:
```python
-import pyotb
+inp.get_rowcol_from_xy(760086.0, 6948092.0) # (333, 5)
+```
-# transforming filepath to pyotb object
-inp = pyotb.Input('my_image.tif')
+## Reading a pixel value
+
+One can read a pixel value of a pyotb object using brackets, as if it was a
+common array. Returned is a list of pixel values for each band:
+
+```python
+inp[10, 10] # [217, 202, 182, 255]
+```
+
+!!! warning
+
+ Accessing multiple pixels values if not computationally efficient. Please
+ use this with moderation, or consider numpy or pyotb applications to
+ process efficiently blocks of pixels.
+
+## Attributes
+
+### Shape
+
+The shape of pyotb objects can be retrieved using `shape`.
+
+```python
print(inp[:1000, :500].shape) # (1000, 500, 4)
```
+
+### Pixel type
+
+The pixel type of pyotb objects can be retrieved using `dtype`.
+
+```python
+inp.dtype # e.g. 'uint8'
+```
+
+!!! note
+
+ The `dtype` returns a `str` corresponding to values accepted by the
+ `pixel_type` of `write()`
+
+### Transform
+
+The transform, as defined in GDAL, can be retrieved with the `transform`
+attribute:
+
+```python
+inp.transform # (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0)
+```
\ No newline at end of file
--
GitLab
From 4e9d21ee76eae47f5521178e58b0c08af37e8b29 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 8 Nov 2023 18:12:27 +0100
Subject: [PATCH 390/399] DOC: sections titles
---
doc/interaction.md | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/doc/interaction.md b/doc/interaction.md
index 0195b70..dcb8729 100644
--- a/doc/interaction.md
+++ b/doc/interaction.md
@@ -1,4 +1,6 @@
-## Export to Numpy
+## Numpy
+
+### Export to numpy arrays
pyotb objects can be exported to numpy array.
@@ -16,7 +18,7 @@ arr = np.asarray(calibrated)
arr = calibrated.to_numpy()
```
-## Interaction with Numpy
+### Interact with numpy functions
pyotb objects can be transparently used in numpy functions.
@@ -47,9 +49,9 @@ noisy_image.write('image_plus_noise.tif')
- The georeference can not be modified. Thus, numpy operations can not
change the image or pixel size
-## Export to rasterio
+## Rasterio
-pyotb objects can also be exported in a format that is usable by rasterio.
+pyotb objects can also be exported in a format usable by rasterio.
For example:
@@ -88,7 +90,7 @@ as the user gets the `profile` dictionary.
If the georeference or pixel size is modified, the user can update the
`profile` accordingly.
-## Interaction with Tensorflow
+## Tensorflow
We saw that numpy operations had some limitations. To bypass those
limitations, it is possible to use some Tensorflow operations on pyotb objects.
--
GitLab
From 1f718193d6c20c9bae2a699597260bf372811d59 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 8 Nov 2023 18:13:11 +0100
Subject: [PATCH 391/399] DOC: note for linked python API
---
doc/otb_versions.md | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/doc/otb_versions.md b/doc/otb_versions.md
index 50f30ee..aa8889d 100644
--- a/doc/otb_versions.md
+++ b/doc/otb_versions.md
@@ -40,8 +40,10 @@ Here is the path precedence for this automatic env configuration :
OR (for windows) : C:/Program Files
```
-N.B. : in case `otbApplication` is found in `PYTHONPATH` (and if `OTB_ROOT`
-was not set), the OTB which the python API is linked to will be used.
+!!! Note
+
+ When `otbApplication` is found in `PYTHONPATH` (and `OTB_ROOT` not set),
+ the OTB installation where the python API is linked, will be used.
## Fresh OTB installation
--
GitLab
From a8b5bf9550fc2b1efd97319c785cc3df694b1d44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr>
Date: Wed, 8 Nov 2023 17:43:19 +0000
Subject: [PATCH 392/399] DOC: Update RELEASE_NOTES.txt
---
RELEASE_NOTES.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index 746a2a6..9b1bad1 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -11,7 +11,7 @@
- Easy access to pixel coordinates
- Add function to transform x,y coordinates into row, col
- Native support of vsicurl inputs
-- Fixes in `summarize()`
+- Fixes and enhancements in `summarize()`
- Fixes in `shape`
- Add typing to function defs to enhance documentation
--
GitLab
From 87ab40a12c9062c6f6be7d7ff0e94e7bb7dc0d6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr>
Date: Thu, 9 Nov 2023 12:50:10 +0000
Subject: [PATCH 393/399] CI: doc stage rules
---
.gitlab-ci.yml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 27cb2d6..2a2bf38 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -119,6 +119,8 @@ docs:
stage: Documentation
rules:
- changes:
+ - *.txt
+ - *.md
- mkdocs.yml
- doc/*
- pyotb/*.py
--
GitLab
From 4f6874d46b10c72d192036d378253fa3e821fb55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr>
Date: Thu, 9 Nov 2023 12:50:56 +0000
Subject: [PATCH 394/399] CI: doc stage rules
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2a2bf38..57c9c42 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -119,8 +119,8 @@ docs:
stage: Documentation
rules:
- changes:
- - *.txt
- - *.md
+ - "*.txt"
+ - "*.md"
- mkdocs.yml
- doc/*
- pyotb/*.py
--
GitLab
From 0221a9e3dc4feab370292f5ef44b2912905a7f69 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 9 Nov 2023 14:57:21 +0100
Subject: [PATCH 395/399] DOC: update readme
---
README.md | 46 +++++++++++++---------------------------------
1 file changed, 13 insertions(+), 33 deletions(-)
diff --git a/README.md b/README.md
index 6659aab..f5aa184 100644
--- a/README.md
+++ b/README.md
@@ -1,19 +1,20 @@
-# pyotb: a pythonic extension of Orfeo Toolbox
+# pyotb: Orfeo ToolBox for Python
[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/releases)
[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop)
[](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb/-/commits/develop)
[](https://pyotb.readthedocs.io/en/master/)
-**pyotb** wraps the [Orfeo Toolbox](https://www.orfeo-toolbox.org/) (OTB)
-python bindings to make it more developer friendly.
+**pyotb** wraps the [Orfeo Toolbox](https://www.orfeo-toolbox.org/) in a pythonic, developer friendly
+fashion.
## Key features
-- Easy use of OTB applications from python
+- Easy use of Orfeo ToolBox (OTB) applications from python
- Simplify common sophisticated I/O features of OTB
-- Lazy execution of in-memory pipelines with OTB streaming mechanism
-- Interoperable with popular python libraries (numpy, rasterio)
+- Lazy execution of operations thanks to OTB streaming mechanism
+- Interoperable with popular python libraries ([numpy](https://numpy.org/) and
+[rasterio](https://rasterio.readthedocs.io/))
- Extensible
Documentation hosted at [pyotb.readthedocs.io](https://pyotb.readthedocs.io/).
@@ -25,44 +26,23 @@ Building a simple pipeline with OTB applications
```py
import pyotb
-# RigidTransformResample application, with input parameters as dict
+# RigidTransformResample, with input parameters as dict
resampled = pyotb.RigidTransformResample({
- "in": "https://some.remote.data/input.tif", # Note: no /vsicurl/...
+ "in": "https://myserver.ia/input.tif", # Note: no /vsicurl/
"interpolator": "linear",
"transform.type.id.scaley": 0.5,
"transform.type.id.scalex": 0.5
})
-# OpticalCalibration, with automatic input parameters resolution
+# OpticalCalibration, with input parameters as args
calib = pyotb.OpticalCalibration(resampled)
-# BandMath, with input parameters passed as kwargs
+# BandMath, with input parameters as kwargs
ndvi = pyotb.BandMath(calib, exp="ndvi(im1b1, im1b4)")
-# Pythonic slicing using lazy computation (no memory used)
+# Pythonic slicing
roi = ndvi[20:586, 9:572]
-# Pipeline execution
-# The actual computation happens here !
+# Pipeline execution. The actual computation happens here!
roi.write("output.tif", "float")
```
-
-pyotb's objects also enable easy interoperability with
-[numpy](https://numpy.org/) and [rasterio](https://rasterio.readthedocs.io/):
-
-```python
-# Numpy and RasterIO style attributes
-print(roi.shape, roi.dtype, roi.transform)
-print(roi.metadata)
-
-# Other useful information
-print(roi.get_infos())
-print(roi.get_statistics())
-
-array = roi.to_numpy()
-array, profile = roi.to_rasterio()
-```
-
-## Contributing
-
-Contributions are welcome on [Github](https://github.com/orfeotoolbox/pyotb) or the source repository hosted on the Orfeo ToolBox [GitLab](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb).
--
GitLab
From 1e2330cd78d2a39542d248508f0c4c19cdc73586 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 9 Nov 2023 14:57:48 +0100
Subject: [PATCH 396/399] DOC: add metadata, get_statistics(), get_info()
---
doc/features.md | 90 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 90 insertions(+)
diff --git a/doc/features.md b/doc/features.md
index 6af1355..3a55c03 100644
--- a/doc/features.md
+++ b/doc/features.md
@@ -92,4 +92,94 @@ attribute:
```python
inp.transform # (6.0, 0.0, 760056.0, 0.0, -6.0, 6946092.0)
+```
+
+### Metadata
+
+Images metadata can be retrieved with the `metadata` attribute:
+
+```python
+print(inp.metadata)
+```
+
+Gives:
+
+```
+{
+ 'DataType': 1.0,
+ 'DriverLongName': 'GeoTIFF',
+ 'DriverShortName': 'GTiff',
+ 'GeoTransform': (760056.0, 6.0, 0.0, 6946092.0, 0.0, -6.0),
+ 'LowerLeftCorner': (760056.0, 6944268.0),
+ 'LowerRightCorner': (761562.0, 6944268.0),
+ 'AREA_OR_POINT': 'Area',
+ 'TIFFTAG_SOFTWARE': 'CSinG - 13 SEPTEMBRE 2012',
+ 'ProjectionRef': 'PROJCS["RGF93 v1 / Lambert-93",\n...',
+ 'ResolutionFactor': 0,
+ 'SubDatasetIndex': 0,
+ 'UpperLeftCorner': (760056.0, 6946092.0),
+ 'UpperRightCorner': (761562.0, 6946092.0),
+ 'TileHintX': 251.0,
+ 'TileHintY': 8.0
+}
+```
+
+## Information
+
+The information fetched by the `ReadImageInfo` OTB application is available
+through `get_info()`:
+
+```python
+print(inp.get_info())
+```
+
+Gives:
+
+```json lines
+{
+ 'indexx': 0,
+ 'indexy': 0,
+ 'sizex': 251,
+ 'sizey': 304,
+ 'spacingx': 6.0,
+ 'spacingy': -6.0,
+ 'originx': 760059.0,
+ 'originy': 6946089.0,
+ 'estimatedgroundspacingx': 5.978403091430664,
+ 'estimatedgroundspacingy': 5.996793270111084,
+ 'numberbands': 4,
+ 'datatype': 'unsigned_char',
+ 'ullat': 0.0,
+ 'ullon': 0.0,
+ 'urlat': 0.0,
+ 'urlon': 0.0,
+ 'lrlat': 0.0,
+ 'lrlon': 0.0,
+ 'lllat': 0.0,
+ 'lllon': 0.0,
+ 'rgb.r': 0,
+ 'rgb.g': 1,
+ 'rgb.b': 2,
+ 'projectionref': 'PROJCS["RGF93 v1 ..."EPSG","2154"]]',
+ 'gcp.count': 0
+}
+```
+
+## Statistics
+
+Image statistics can be computed on-the-fly using `get_statistics()`:
+
+```python
+print(inp.get_statistics())
+```
+
+Gives:
+
+```json lines
+{
+ 'out.mean': [79.5505, 109.225, 115.456, 249.349],
+ 'out.min': [33, 64, 91, 47],
+ 'out.max': [255, 255, 230, 255],
+ 'out.std': [51.0754, 35.3152, 23.4514, 20.3827]
+}
```
\ No newline at end of file
--
GitLab
From b44e5089ad4a6170725ee07eafa3d11e58368d88 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 9 Nov 2023 14:58:17 +0100
Subject: [PATCH 397/399] DOC: move contribute section in index
---
doc/index.md | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
diff --git a/doc/index.md b/doc/index.md
index d76ef33..4ecfaf4 100644
--- a/doc/index.md
+++ b/doc/index.md
@@ -1,4 +1,4 @@
-# Pyotb: Orfeo Toolbox for Python
+# pyotb: Orfeo Toolbox for Python
pyotb is a Python extension of Orfeo Toolbox. It has been built on top of the
existing Python API of OTB, in order
@@ -9,7 +9,7 @@ to make OTB more Python friendly.
## Get started
- [Installation](installation.md)
-- [How to use pyotb](quickstart.md)
+- [Quick start](quickstart.md)
- [Useful features](features.md)
- [Functions](functions.md)
- [Interaction with Python libraries (numpy, rasterio, tensorflow)](interaction.md)
@@ -27,8 +27,17 @@ to make OTB more Python friendly.
- [Managing loggers](managing_loggers.md)
- [Troubleshooting & limitations](troubleshooting.md)
-
## API
- See the API reference. If you have any doubts or questions, feel free to ask
-on github or gitlab!
\ No newline at end of file
+on github or gitlab!
+
+## Contribute
+
+Contributions are welcome !
+Open a PR/MR, or file an issue if you spot a bug or have any suggestion:
+
+- [Github](https://github.com/orfeotoolbox/pyotb)
+- [Orfeo ToolBox GitLab instance](https://gitlab.orfeo-toolbox.org/nicolasnn/pyotb).
+
+Thank you!
\ No newline at end of file
--
GitLab
From 47bc22af9c25ff76ff21df6961f60d5748b648ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr>
Date: Thu, 23 Nov 2023 14:58:06 +0000
Subject: [PATCH 398/399] Update RELEASE_NOTES.txt
---
RELEASE_NOTES.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index 9b1bad1..d7f6094 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -1,5 +1,5 @@
---------------------------------------------------------------------
-2.00 (Oct XX, 2023) - Changes since version 1.5.4
+2.00 (Nov 23, 2023) - Changes since version 1.5.4
- Major refactoring (see troubleshooting/migration)
- Pythonic extended filenames (can use dict, etc)
--
GitLab
From 8ca3c1072ad7373abda257668952e6290d6a9c61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9mi=20Cresson?= <remi.cresson@inrae.fr>
Date: Thu, 23 Nov 2023 15:09:34 +0000
Subject: [PATCH 399/399] Update __init__.py
---
pyotb/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index 5e74f37..5b455ab 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
"""This module provides convenient python wrapping of otbApplications."""
-__version__ = "2.0.0.dev8"
+__version__ = "2.0.0"
from .install import install_otb
from .helpers import logger, set_logger_level
--
GitLab