diff --git a/.dockerignore b/.dockerignore
index 2ca1beb2a56a4cb45c526deab8306417c095f6fe..85bef57491e7b9179dbcd5e3d2f084d84b715a1a 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,2 +1,7 @@
-.git
-python/__pycache__
+.git*
+Dockerfile
+**/__pycache__
+.readthedocs.yaml
+doc/*
+mkdocs.yml
+RELEASE_NOTES.txt
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bd41bbf9a6151e4c3b0f47b7eab4147f0aa263d4..35dfe14298ee0b3506fd6023f6e6c40755f15681 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,269 +1,257 @@
-variables:
- OTBTF_VERSION: 4.3.1
- OTB_BUILD: /src/otb/build/OTB/build # Local OTB build directory
- OTBTF_SRC: /src/otbtf # Local OTBTF source directory
- OTB_TEST_DIR: $OTB_BUILD/Testing/Temporary # OTB testing directory
- ARTIFACT_TEST_DIR: $CI_PROJECT_DIR/testing
- CRC_BOOK_TMP: /tmp/crc_book_tests_tmp
- API_TEST_TMP: /tmp/api_tests_tmp
- DATADIR: $CI_PROJECT_DIR/test/data
- DOCKER_BUILDKIT: 1
- DOCKER_DRIVER: overlay2
- CACHE_IMAGE_BASE: $CI_REGISTRY_IMAGE:otbtf-base
- CACHE_IMAGE_BUILDER: $CI_REGISTRY_IMAGE:builder
- BRANCH_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
- DEV_IMAGE: $CI_REGISTRY_IMAGE:cpu-basic-dev-testing
- CI_REGISTRY_PUBIMG: $CI_REGISTRY_IMAGE:$OTBTF_VERSION
- DOCKERHUB_BASE: mdl4eo/otbtf
- DOCKERHUB_IMAGE_BASE: ${DOCKERHUB_BASE}:${OTBTF_VERSION}
- CPU_BASE_IMG: ubuntu:22.04
- GPU_BASE_IMG: nvidia/cuda:12.0.1-cudnn8-devel-ubuntu22.04
-
-image: $BRANCH_IMAGE
-
workflow:
rules:
- - if: $CI_MERGE_REQUEST_ID || $CI_COMMIT_REF_NAME =~ /master/ # Execute jobs in merge request context, or commit in master branch
-
+ # Ignore pipeline for filthy commits
+ - if: $CI_COMMIT_MESSAGE =~ /^wip.*/i
+ when: never
+ # Execute for MR, but avoid duplicated due to branch triggers in MR
+ - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
+ when: never
+ - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+ # Else execute on tags or protected branches
+ - if: $CI_COMMIT_TAG
+ - if: $CI_COMMIT_REF_PROTECTED == "true"
+ auto_cancel:
+ on_new_commit: interruptible
+
stages:
- Build
- Static Analysis
- Documentation
- Test
- Applications Test
- - Update dev image
- Ship
+variables:
+ OTBTF_VERSION: 5.0.0-rc2
+ OTBTF_SRC: /src/otbtf # OTBTF source directory path in image
+ DATADIR: $CI_PROJECT_DIR/test/data
+ CACHE_IMAGE_CPU: $CI_REGISTRY_IMAGE:build-cache-cpu
+ CACHE_IMAGE_GPU: $CI_REGISTRY_IMAGE:build-cache-gpu
+ BRANCH_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+ BUILDX_BUILDER: container
+ BUILDX_NO_DEFAULT_ATTESTATIONS: 1
+
+default:
+ tags: [ godzilla ]
+ interruptible: true
+ image:
+ name: $BRANCH_IMAGE
+ pull_policy: always
+
.docker_build_base:
- allow_failure: false
- tags: [godzilla]
- image: docker:latest
+ image: docker:27.5.1
services:
- - name: docker:dind
+ - name: docker:27.5.1-dind
before_script:
- echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- timeout: 10 hours
-
+ - docker buildx create --name=container --driver=docker-container --use --bootstrap
+ after_script:
+ - docker buildx rm --keep-state container
-docker image:
- extends: .docker_build_base
+docker_image:
stage: Build
- except:
+ only:
+ - merge_requests
- develop
+ extends: .docker_build_base
script:
- >
- docker build
- --target otbtf-base
- --cache-from $CACHE_IMAGE_BASE
- --tag $CACHE_IMAGE_BASE
- --build-arg BASE_IMG=$CPU_BASE_IMG
- --build-arg BUILDKIT_INLINE_CACHE=1
- "."
- - docker push $CACHE_IMAGE_BASE
- - >
- docker build
- --target builder
- --cache-from $CACHE_IMAGE_BASE
- --cache-from $CACHE_IMAGE_BUILDER
- --tag $CACHE_IMAGE_BUILDER
- --build-arg KEEP_SRC_OTB="true"
- --build-arg BZL_CONFIGS=""
- --build-arg BASE_IMG=$CPU_BASE_IMG
- --build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=$BAZELCACHE"
- --build-arg OTBTESTS="true"
+ docker buildx build --push -t $BRANCH_IMAGE
+ --cache-from type=registry,ref=$CACHE_IMAGE_CPU
+ --cache-to type=registry,ref=$CACHE_IMAGE_CPU,mode=max
+ --build-arg DEV_IMAGE=true
"."
- - docker push $CACHE_IMAGE_BUILDER
- - >
- docker build
- --cache-from $CACHE_IMAGE_BASE
- --cache-from $CACHE_IMAGE_BUILDER
- --cache-from $BRANCH_IMAGE
- --cache-from $DEV_IMAGE
- --tag $BRANCH_IMAGE
- --build-arg KEEP_SRC_OTB="true"
- --build-arg BZL_CONFIGS=""
- --build-arg BASE_IMG=$CPU_BASE_IMG
- --build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=$BAZELCACHE"
- --build-arg OTBTESTS="true"
- "."
- - docker push $BRANCH_IMAGE
.static_analysis_base:
stage: Static Analysis
allow_failure: true
+ only:
+ - merge_requests
flake8:
extends: .static_analysis_base
script:
- - sudo pip install flake8
- flake8 $OTBTF_SRC/otbtf --exclude=tensorflow_v1x
pylint:
extends: .static_analysis_base
script:
- - sudo pip install pylint
- pylint $OTBTF_SRC/otbtf --ignore=tensorflow_v1x
codespell:
+ rules:
extends: .static_analysis_base
script:
- - sudo pip install codespell
- - codespell otbtf
- - codespell doc
+ - codespell otbtf doc README.md
cppcheck:
extends: .static_analysis_base
script:
- - sudo apt update && sudo apt install cppcheck -y
- cd $OTBTF_SRC/ && cppcheck --enable=all --error-exitcode=1 -I include/ --suppress=missingInclude --suppress=unusedFunction .
-.doc_base:
+docs:
stage: Documentation
+ tags: [ stable ]
+ needs: []
+ image: python:3.10-slim
+ variables:
+ PTH: public_test
+ rules:
+ - if: $CI_COMMIT_TAG
+ when: never
+ - if: $CI_COMMIT_REF_NAME == /master/
+ variables:
+ PTH: public
+ - changes:
+ - doc/**/*
+ - "*.{md,txt}"
+ - mkdocs.yml
+ - .readthedocs.yaml
before_script:
- pip install -r doc/doc_requirements.txt
- artifacts:
- paths:
- - public
- - public_test
-
-pages_test:
- extends: .doc_base
- except:
- - master
script:
- - mkdocs build --site-dir public_test
-
-pages:
- extends: .doc_base
- only:
- - master
- script:
- - mkdocs build --site-dir public
+ - mkdocs build --site-dir $PTH
artifacts:
paths:
- public
+ - public_test
.tests_base:
- tags: [godzilla]
+ only:
+ - merge_requests
artifacts:
- paths:
- - $ARTIFACT_TEST_DIR/*.*
+ reports:
+ junit: report_*.xml
expire_in: 1 week
when: on_failure
ctest:
+ stage: Test
extends: .tests_base
+ needs: ["docker_image"]
+ variables:
+ CTEST_OUTPUT_ON_FAILURE: 1
+ OTB_TEST_UNITS: "Tensorflow|PanSharpening|Projection|Transform|IOGDAL"
+ OTB_BUILD: /src/otb/build/OTB/build
+ script:
+ - cd /src/otb/otb && git lfs pull
+ - ln -s $CI_PROJECT_DIR/test/data $OTBTF_SRC/test/data
+ - ln -s $CI_PROJECT_DIR/test/models $OTBTF_SRC/test/models
+ - cd $OTB_BUILD
+ - ctest -L $OTB_TEST_UNITS --output-junit $CI_PROJECT_DIR/report_ctest.xml
+ after_script:
+ - cp -r $OTB_BUILD/Testing/Temporary/* $CI_PROJECT_DIR/artifacts_ctest
+ artifacts:
+ paths:
+ - $CI_PROJECT_DIR/artifacts_ctest
+
+python_api:
stage: Test
+ extends: .tests_base
+ variables:
+ API_TEST_TMP: /tmp/api_tests_tmp
script:
- - sudo apt update && sudo apt install -y git-lfs
- - cd /src/otb/otb && sudo git lfs fetch --all && sudo git lfs pull
- - cd $OTB_BUILD/
- - sudo ctest -L OTBTensorflow
- - sudo ctest -L OTBPanSharpening
- - sudo ctest -L OTBProjection
- - sudo ctest -L OTBTransform
- - sudo ctest -L OTBIOGDAL
+ - mkdir $API_TEST_TMP
+ - TMPDIR=$API_TEST_TMP python -m pytest -svv --junitxml=report_api.xml test/api_unittest.py
after_script:
- - cp -r $OTB_TEST_DIR $ARTIFACT_TEST_DIR
+ - saved_model_cli show --dir $API_TEST_TMP/model_from_pimg --all
+ - cp -r $API_TEST_TMP $CI_PROJECT_DIR/artifacts_test_api
+ artifacts:
+ paths:
+ - $CI_PROJECT_DIR/artifacts_test_api
.applications_test_base:
- extends: .tests_base
stage: Applications Test
- before_script:
- - pip install pytest pytest-cov pytest-order
- - mkdir -p $ARTIFACT_TEST_DIR
- - cd $CI_PROJECT_DIR
+ extends: .tests_base
+ needs: ["ctest", "python_api"]
+ #when: manual
crc_book:
extends: .applications_test_base
+ when: manual
+ allow_failure: true
+ variables:
+ CRC_BOOK_TMP: /tmp/crc_book_tests_tmp
script:
- mkdir -p $CRC_BOOK_TMP
- - TMPDIR=$CRC_BOOK_TMP python -m pytest --junitxml=$CI_PROJECT_DIR/report_tutorial.xml $OTBTF_SRC/test/tutorial_unittest.py
+ - TMPDIR=$CRC_BOOK_TMP python -m pytest -v --junitxml=report_tutorial.xml test/tutorial_unittest.py
after_script:
- - cp $CRC_BOOK_TMP/*.* $ARTIFACT_TEST_DIR/
-
+ - cp -r $CRC_BOOK_TMP $CI_PROJECT_DIR/artifacts_crc_book
+ artifacts:
+ paths:
+ - $CI_PROJECT_DIR/artifacts_crc_book
+
+decloud:
+ extends: .applications_test_base
+ when: manual
+ allow_failure: true
+ variables:
+ DATASET_DECLOUD: https://nextcloud.inrae.fr/s/aNTWLcH28zNomqk/download
+ DECLOUD_DATA_DIR: $CI_PROJECT_DIR/decloud_data
+ script:
+ - git clone https://forgemia.inra.fr/umr-tetis/decloud.git -b keras3
+ - pip install -r $PWD/decloud/requirements.txt
+ - wget -q $DATASET_DECLOUD -O file.zip && unzip file.zip
+ - pytest -v decloud/tests/train_from_tfrecords_unittest.py
+
sr4rs:
extends: .applications_test_base
+ variables:
+ DATASET_S2: https://nextcloud.inrae.fr/s/EZL2JN7SZyDK8Cf/download/sr4rs_sentinel2_bands4328_france2020_savedmodel.zip
+ DATASET_SR4RS: https://nextcloud.inrae.fr/s/kDms9JrRMQE2Q5z/download
script:
- - wget -O sr4rs_sentinel2_bands4328_france2020_savedmodel.zip
- https://nextcloud.inrae.fr/s/EZL2JN7SZyDK8Cf/download/sr4rs_sentinel2_bands4328_france2020_savedmodel.zip
+ - wget -qO sr4rs_sentinel2_bands4328_france2020_savedmodel.zip $DATASET_S2
- unzip -o sr4rs_sentinel2_bands4328_france2020_savedmodel.zip
- - wget -O sr4rs_data.zip https://nextcloud.inrae.fr/s/kDms9JrRMQE2Q5z/download
+ - wget -qO sr4rs_data.zip $DATASET_SR4RS
- unzip -o sr4rs_data.zip
- rm -rf sr4rs
- git clone https://github.com/remicres/sr4rs.git
- export PYTHONPATH=$PYTHONPATH:$PWD/sr4rs
- - python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_sr4rs.xml $OTBTF_SRC/test/sr4rs_unittest.py
-
-decloud:
- extends: .applications_test_base
- script:
- - git clone https://github.com/CNES/decloud.git
- - pip install -r $PWD/decloud/docker/requirements.txt
- - wget https://nextcloud.inrae.fr/s/aNTWLcH28zNomqk/download -O file.zip && unzip file.zip
- - export DECLOUD_DATA_DIR="$PWD/decloud_data"
- - pytest decloud/tests/train_from_tfrecords_unittest.py
-
-otbtf_api:
- extends: .applications_test_base
- script:
- - mkdir $API_TEST_TMP
- - TMPDIR=$API_TEST_TMP python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_api.xml $OTBTF_SRC/test/api_unittest.py
- after_script:
- - cp $API_TEST_TMP/*.* $ARTIFACT_TEST_DIR/
+ - python -m pytest -v --junitxml=report_sr4rs.xml test/sr4rs_unittest.py
geos_enabled:
extends: .applications_test_base
script:
- - python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_geos_enabled.xml $OTBTF_SRC/test/geos_test.py
+ - python -m pytest -v --junitxml=report_geos_enabled.xml test/geos_test.py
planetary_computer:
extends: .applications_test_base
script:
- pip install pystac_client planetary_computer
- - python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_pc_enabled.xml $OTBTF_SRC/test/pc_test.py
-
-imports:
- extends: .applications_test_base
- script:
- - python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_imports.xml $OTBTF_SRC/test/imports_test.py
+ - python -m pytest -v --junitxml=report_pc_enabled.xml test/pc_test.py
numpy_gdal_otb:
extends: .applications_test_base
script:
- - python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_numpy.xml $OTBTF_SRC/test/numpy_test.py
+ - python -m pytest -v --junitxml=report_numpy.xml test/numpy_test.py
rio:
extends: .applications_test_base
script:
- - sudo pip install rasterio
- - python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_rio.xml $OTBTF_SRC/test/rio_test.py
+ - pip install rasterio --no-binary rasterio
+ - python -m pytest -v --junitxml=report_rio.xml test/rio_test.py
nodata:
extends: .applications_test_base
script:
- - python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_nodata.xml $OTBTF_SRC/test/nodata_test.py
+ - python -m pytest -v --junitxml=report_nodata.xml test/nodata_test.py
-deploy_cpu-dev-testing:
- stage: Update dev image
+.ship_base:
extends: .docker_build_base
- except:
- - master
- script:
- - docker pull $BRANCH_IMAGE
- - docker tag $BRANCH_IMAGE $DEV_IMAGE
- - docker push $DEV_IMAGE
-
-.ship base:
- extends: .docker_build_base
- stage: Ship
only:
- - master
+ - tags
+ variables:
+ DOCKERHUB_BASE: mdl4eo/otbtf
+ CI_REGISTRY_PUBIMG: $CI_REGISTRY_IMAGE:$OTBTF_VERSION
+ DOCKERHUB_IMAGE_BASE: $DOCKERHUB_BASE:$OTBTF_VERSION
+ before_script:
+ - echo -n $DOCKERHUB_TOKEN | docker login -u mdl4eo --password-stdin
+ - echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+ - docker buildx create --name=container --driver=docker-container --use --bootstrap
deploy_cpu:
- extends: .ship base
+ stage: Ship
+ extends: .ship_base
variables:
IMAGE_CPU: $CI_REGISTRY_PUBIMG-cpu
IMAGE_CPUDEV: $CI_REGISTRY_PUBIMG-cpu-dev
@@ -272,51 +260,49 @@ deploy_cpu:
DOCKERHUB_LATEST: $DOCKERHUB_BASE:latest
script:
# cpu
- - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_CPU --build-arg BASE_IMG=$CPU_BASE_IMG --build-arg BZL_CONFIGS="" .
- - docker push $IMAGE_CPU
+ - >
+ docker buildx build --push -t $IMAGE_CPU
+ --cache-from type=registry,ref=$CACHE_IMAGE_CPU
+ .
# cpu-dev
- - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_CPUDEV --build-arg BASE_IMG=$CPU_BASE_IMG --build-arg BZL_CONFIGS="" --build-arg KEEP_SRC_OTB=true .
- - docker push $IMAGE_CPUDEV
+ - >
+ docker buildx build --push -t $IMAGE_CPUDEV
+ --cache-from type=registry,ref=$CACHE_IMAGE_CPU
+ --build-arg DEV_IMAGE=true
+ .
# push images on dockerhub
- - echo -n $DOCKERHUB_TOKEN | docker login -u mdl4eo --password-stdin
- - docker tag $IMAGE_CPU $DOCKERHUB_CPU
- - docker push $DOCKERHUB_CPU
- - docker tag $IMAGE_CPUDEV $DOCKERHUB_CPUDEV
- - docker push $DOCKERHUB_CPUDEV
+ - docker tag $IMAGE_CPU $DOCKERHUB_CPU && docker push $DOCKERHUB_CPU
+ - docker tag $IMAGE_CPUDEV $DOCKERHUB_CPUDEV && docker push $DOCKERHUB_CPUDEV
# latest = cpu image
- - docker tag $IMAGE_CPU $DOCKERHUB_LATEST
- - docker push $DOCKERHUB_LATEST
+ - docker tag $IMAGE_CPU $DOCKERHUB_LATEST && docker push $DOCKERHUB_LATEST
deploy_gpu:
- extends: .ship base
+ stage: Ship
+ extends: .ship_base
variables:
IMAGE_GPU: $CI_REGISTRY_PUBIMG-gpu
IMAGE_GPUDEV: $CI_REGISTRY_PUBIMG-gpu-dev
- IMAGE_GPUOPT: $CI_REGISTRY_PUBIMG-gpu-opt
- IMAGE_GPUOPTDEV: $CI_REGISTRY_PUBIMG-gpu-opt-dev
DOCKERHUB_GPU: $DOCKERHUB_IMAGE_BASE-gpu
DOCKERHUB_GPUDEV: $DOCKERHUB_IMAGE_BASE-gpu-dev
DOCKERHUB_GPULATEST: $DOCKERHUB_BASE:latest-gpu
script:
- # gpu-opt
- - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_GPUOPT --build-arg BASE_IMG=$GPU_BASE_IMG .
- - docker push $IMAGE_GPUOPT
- # gpu-opt-dev
- - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_GPUOPTDEV --build-arg BASE_IMG=$GPU_BASE_IMG --build-arg KEEP_SRC_OTB=true .
- - docker push $IMAGE_GPUOPTDEV
- # gpu-basic
- - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_GPU --build-arg BASE_IMG=$GPU_BASE_IMG --build-arg BZL_CONFIGS="" .
- - docker push $IMAGE_GPU
- # gpu-basic-dev
- - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_GPUDEV --build-arg BZL_CONFIGS="" --build-arg BASE_IMG=$GPU_BASE_IMG --build-arg KEEP_SRC_OTB=true .
- - docker push $IMAGE_GPUDEV
- # push gpu-basic* images on dockerhub
- - echo -n $DOCKERHUB_TOKEN | docker login -u mdl4eo --password-stdin
- - docker tag $IMAGE_GPU $DOCKERHUB_GPU
- - docker push $DOCKERHUB_GPU
- - docker tag $IMAGE_GPUDEV $DOCKERHUB_GPUDEV
- - docker push $DOCKERHUB_GPUDEV
+ # gpu
+ - >
+ docker buildx build --push -t $IMAGE_GPU
+ --cache-from type=registry,ref=$CACHE_IMAGE_GPU
+ --cache-from type=registry,ref=$CACHE_IMAGE_CPU
+ --cache-to type=registry,ref=$CACHE_IMAGE_GPU,mode=max
+ --build-arg WITH_CUDA=true
+ .
+ # gpu-dev
+ - >
+ docker buildx build --push -t $IMAGE_GPUDEV
+ --cache-from type=registry,ref=$CACHE_IMAGE_GPU
+ --cache-from type=registry,ref=$CACHE_IMAGE_CPU
+ --build-arg WITH_CUDA=true --build-arg DEV_IMAGE=true
+ .
+ # push gpu-* images on dockerhub
+ - docker tag $IMAGE_GPU $DOCKERHUB_GPU && docker push $DOCKERHUB_GPU
+ - docker tag $IMAGE_GPUDEV $DOCKERHUB_GPUDEV && docker push $DOCKERHUB_GPUDEV
# latest-gpu = gpu image
- - docker tag $IMAGE_GPU $DOCKERHUB_GPULATEST
- - docker push $DOCKERHUB_GPULATEST
-
+ - docker tag $IMAGE_GPU $DOCKERHUB_GPULATEST && docker push $DOCKERHUB_GPULATEST
diff --git a/Dockerfile b/Dockerfile
index 904431e35349f5a27f35250e834594f5baa3f4d5..0aacaa5dae9321c3897e5c2564cfe17ca17c6fbf 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,114 +1,109 @@
-##### Configurable Dockerfile with multi-stage build - Author: Vincent Delbar
-## Mandatory
-ARG BASE_IMG
-
+##### OTBTF configurable Dockerfile with multi-stage build
# ----------------------------------------------------------------------------
-# Init base stage - will be cloned as intermediate build env
-FROM $BASE_IMG AS otbtf-base
+# Init base stage - used for intermediate build env and final image
+
+# Freeze ubuntu version to avoid surprise rebuild
+FROM ubuntu:jammy-20250126 AS base-stage
+
WORKDIR /tmp
-### System packages
-COPY tools/docker/build-deps-*.txt ./
+# System packages
ARG DEBIAN_FRONTEND=noninteractive
+COPY system-dependencies.txt .
RUN apt-get update -y && apt-get upgrade -y \
- && cat build-deps-cli.txt | xargs apt-get install --no-install-recommends -y \
+ && cat system-dependencies.txt | xargs apt-get install --no-install-recommends -y \
&& apt-get clean && rm -rf /var/lib/apt/lists/*
-### Python3 links and pip packages
-RUN ln -s /usr/bin/python3 /usr/local/bin/python && ln -s /usr/bin/pip3 /usr/local/bin/pip
-# Upgrade pip
-RUN pip install --no-cache-dir pip --upgrade
-# In case NumPy version is conflicting with system's gdal dep and may require venv
-ARG NUMPY_SPEC=""
-# This is to avoid https://github.com/tensorflow/tensorflow/issues/61551
-ARG PROTO_SPEC="==4.23.*"
-RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" packaging requests \
- && pip install --no-cache-dir --no-deps keras_applications keras_preprocessing
-# ----------------------------------------------------------------------------
-# Tmp builder stage - dangling cache should persist until "docker builder prune"
-FROM otbtf-base AS builder
-# A smaller value may be required to avoid OOM errors when building OTB
+# Env required during build and for the final image
+ENV PY=3.10
+ENV VIRTUAL_ENV=/opt/otbtf/venv
+ENV PATH="$VIRTUAL_ENV/bin:/opt/otbtf/bin:$PATH"
+ENV PYTHON_SITE_PACKAGES="$VIRTUAL_ENV/lib/python$PY/site-packages"
+ENV LD_LIBRARY_PATH=/opt/otbtf/lib
+# A smaller value may be used to limit bazel or to avoid OOM errors while building OTB
ARG CPU_RATIO=1
-RUN mkdir -p /src/tf /opt/otbtf/bin /opt/otbtf/include /opt/otbtf/lib/python3
+# ----------------------------------------------------------------------------
+# Builder stage: bazel clang tensorflow
+FROM base-stage AS tf-build
WORKDIR /src/tf
+RUN mkdir -p /opt/otbtf/bin /opt/otbtf/lib /opt/otbtf/include
-RUN git config --global advice.detachedHead false
-
-### TF
-
-ARG TF=v2.14.0
-ARG TENSORRT
-
-# Install bazelisk (will read .bazelversion and download the right bazel binary - latest by default)
-RUN wget -qO /opt/otbtf/bin/bazelisk https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-amd64 \
- && chmod +x /opt/otbtf/bin/bazelisk \
- && ln -s /opt/otbtf/bin/bazelisk /opt/otbtf/bin/bazel
-
-ARG BZL_TARGETS="//tensorflow:libtensorflow_cc.so //tensorflow/tools/pip_package:build_pip_package"
-
-# "--config=opt" will enable 'march=native'
-# (otherwise read comments about CPU compatibility and edit CC_OPT_FLAGS in
-# build-env-tf.sh)
-ARG BZL_CONFIGS="--config=nogcp --config=noaws --config=nohdfs --config=opt"
-
-# "--compilation_mode opt" is already enabled by default (see tf repo .bazelrc
-# and configure.py)
-ARG BZL_OPTIONS="--verbose_failures --remote_cache=http://localhost:9090"
-
-# Build
-ARG ZIP_TF_BIN=false
-COPY tools/docker/build-env-tf.sh ./
-RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.git
-RUN cd tensorflow \
- && export PATH=$PATH:/opt/otbtf/bin \
- && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/otbtf/lib \
- && bash -c '\
- source ../build-env-tf.sh \
- && ./configure \
- && export TMP=/tmp/bazel \
- && BZL_CMD="build $BZL_TARGETS $BZL_CONFIGS $BZL_OPTIONS" \
- && bazel $BZL_CMD --jobs="HOST_CPUS*$CPU_RATIO" '
-
-# Installation
-RUN apt update && apt install -y patchelf
-RUN cd tensorflow \
- && ./bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg \
- && pip3 install --no-cache-dir --prefix=/opt/otbtf /tmp/tensorflow_pkg/tensorflow*.whl \
- && ln -s /opt/otbtf/local/lib/python3.*/* /opt/otbtf/lib/python3 \
- && ln -s /opt/otbtf/local/bin/* /opt/otbtf/bin \
- && ln -s $(find /opt/otbtf -type d -wholename "*/dist-packages/tensorflow/include") /opt/otbtf/include/tf \
- # The only missing header in the wheel
- && cp tensorflow/cc/saved_model/tag_constants.h /opt/otbtf/include/tf/tensorflow/cc/saved_model/ \
- && cp tensorflow/cc/saved_model/signature_constants.h /opt/otbtf/include/tf/tensorflow/cc/saved_model/ \
- # Symlink external libs (required for MKL - libiomp5)
- && for f in $(find -L /opt/otbtf/include/tf -wholename "*/external/*/*.so"); do ln -s $f /opt/otbtf/lib/; done \
- # Compress and save TF binaries
- && ( ! $ZIP_TF_BIN || zip -9 -j --symlinks /opt/otbtf/tf-$TF.zip tensorflow/cc/saved_model/tag_constants.h tensorflow/cc/saved_model/signature_constants.h bazel-bin/tensorflow/libtensorflow_cc.so* /tmp/tensorflow_pkg/tensorflow*.whl ) \
- # Cleaning
- && rm -rf bazel-* /src/tf /root/.cache/ /tmp/*
+# Clang + LLVM
+ARG LLVM=18
-### OTB
+ADD https://apt.llvm.org/llvm.sh llvm.sh
+RUN bash ./llvm.sh $LLVM
+ENV CC=/usr/bin/clang-$LLVM
+ENV CXX=/usr/bin/clang++-$LLVM
+ENV BAZEL_COMPILER="/usr/bin/clang-$LLVM"
+RUN apt-get update -y && apt-get upgrade -y \
+ && apt-get install -y lld-$LLVM libomp-$LLVM-dev \
+ && apt-get clean && rm -rf /var/lib/apt/lists/*
-ARG OTB=release-9.0
-ARG OTBTESTS=false
+### Python venv and packages
+RUN virtualenv $VIRTUAL_ENV
+RUN pip install --no-cache-dir -U pip wheel
+# Numpy 2 support in TF is planned for 2.18, but isn't supported by most libraries for now
+ARG NUMPY="1.26.4"
+RUN pip install --no-cache-dir -U mock six future tqdm deprecated numpy==$NUMPY packaging requests
+
+# TensorFlow build arguments
+ARG TF=v2.18.0
+ARG WITH_CUDA=false
+# Custom compute capabilities, else use default one from .bazelrc
+ARG CUDA_CC
+ARG WITH_XLA=true
+ARG WITH_MKL=false
+
+# Install bazelisk: will read .bazelversion and download the right bazel binary
+ADD https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-amd64 /opt/otbtf/bin/bazelisk
+RUN chmod +x /opt/otbtf/bin/bazelisk && ln -s /opt/otbtf/bin/bazelisk /opt/otbtf/bin/bazel
+
+# Build and install tf wheel
+ADD https://github.com/tensorflow/tensorflow.git#$TF tensorflow
+ARG BZL_TARGETS="//tensorflow:libtensorflow_cc.so //tensorflow/tools/pip_package:wheel"
+# You can use --build-arg BZL_OPTIONS="--remote_cache=http://..." at build time
+ARG BZL_OPTIONS
+
+# Run build with local bazel cache using docker mount
+RUN --mount=type=cache,target=/root/.cache/bazel \
+ cd tensorflow \
+ && BZL_CONFIGS="--repo_env=WHEEL_NAME=tensorflow_cpu --config=release_cpu_linux" \
+ && if [ "$WITH_CUDA" = "true" ] ; then BZL_CONFIGS="--repo_env=WHEEL_NAME=tensorflow --config=release_gpu_linux --config=cuda_wheel" ; fi \
+ && if [ -n "$CUDA_CC" ] ; then BZL_CONFIGS="$BZL_CONFIGS --repo_env=HERMETIC_CUDA_COMPUTE_CAPABILITIES=$CUDA_CC"; fi \
+ && if [ "$WITH_MKL" = "true" ] ; then BZL_CONFIGS="$BZL_CONFIGS --config=mkl" ; fi \
+ && if [ "$WITH_XLA" = "true" ] ; then BZL_CONFIGS="$BZL_CONFIGS --config=xla" ; fi \
+ && export HERMETIC_PYTHON_VERSION=$PY \
+ && echo "Build env:" && env \
+ && BZL_CMD="build $BZL_TARGETS $BZL_CONFIGS --announce_rc --verbose_failures $BZL_OPTIONS" \
+ && echo "Starting build with cmd: \"bazel $BZL_CMD\"" \
+ && bazel $BZL_CMD --jobs="HOST_CPUS*$CPU_RATIO" \
+ && TF_WHEEL=$(find bazel-bin/tensorflow/tools/pip_package/wheel_house/ -type f -name "tensorflow*.whl") \
+ && pip install --no-cache-dir "$TF_WHEEL$(! $WITH_CUDA || echo '[and-cuda]')" \
+ && ln -s $PYTHON_SITE_PACKAGES/tensorflow/include /opt/otbtf/include/tf \
+ && for f in $(find -L /opt/otbtf/include/tf -wholename "*/external/*/*.so"); do ln -s $f /opt/otbtf/lib/; done \
+ && TF_MISSING_HEADERS="tensorflow/cc/saved_model/tag_constants.h tensorflow/cc/saved_model/signature_constants.h" \
+ && cp $TF_MISSING_HEADERS /opt/otbtf/include/tf/tensorflow/cc/saved_model/ \
+ && mkdir /tmp/artifacts && mv bazel-bin/tensorflow/libtensorflow_cc.so* $TF_WHEEL $TF_MISSING_HEADERS /tmp/artifacts \
+ && rm -rf bazel-* /src/tf
-RUN mkdir /src/otb
+# ----------------------------------------------------------------------------
+# Builder stage: cmake gcc otb
+FROM base-stage AS otb-build
WORKDIR /src/otb
+COPY --from=tf-build /opt/otbtf /opt/otbtf
+
# SuperBuild OTB
-COPY tools/docker/build-flags-otb.txt ./
-RUN apt-get update -y \
- && apt-get install --reinstall ca-certificates -y \
- && update-ca-certificates \
- && git clone https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb.git \
- && cd otb && git checkout $OTB
-
-# <---------------------------------------- Begin dirty hack
+ARG OTB=release-9.1
+ADD --keep-git-dir=true https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb.git#$OTB otb
+
+# <------------------------------------------
# This is a dirty hack for release 4.0.0alpha
# We have to wait that OTB moves from C++14 to C++17
# See https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2338
-RUN cd /src/otb/otb \
+RUN cd otb \
&& sed -i 's/CMAKE_CXX_STANDARD 14/CMAKE_CXX_STANDARD 17/g' CMakeLists.txt \
&& echo "" > Modules/Core/ImageManipulation/test/CMakeLists.txt \
&& echo "" > Modules/Core/Conversion/test/CMakeLists.txt \
@@ -116,81 +111,85 @@ RUN cd /src/otb/otb \
&& echo "" > Modules/Core/Edge/test/CMakeLists.txt \
&& echo "" > Modules/Core/ImageBase/test/CMakeLists.txt \
&& echo "" > Modules/Learning/DempsterShafer/test/CMakeLists.txt \
-# <---------------------------------------- End dirty hack
&& cd .. \
- && mkdir -p build \
+ && mkdir -p build /tmp/SuperBuild-downloads \
&& cd build \
- && if $OTBTESTS; then \
- echo "-DBUILD_TESTING=ON" >> ../build-flags-otb.txt; fi \
- # Possible ENH: superbuild-all-dependencies switch, with separated build-deps-minimal.txt and build-deps-otbcli.txt)
- #&& if $OTB_SUPERBUILD_ALL; then sed -i -r "s/-DUSE_SYSTEM_([A-Z0-9]*)=ON/-DUSE_SYSTEM_\1=OFF/ " ../build-flags-otb.txt; fi \
- && OTB_FLAGS=$(cat "../build-flags-otb.txt") \
- && cmake ../otb/SuperBuild -DCMAKE_INSTALL_PREFIX=/opt/otbtf $OTB_FLAGS \
- && make -j $(python -c "import os; print(round( os.cpu_count() * $CPU_RATIO ))")
-
-### OTBTF - copy (without .git/) or clone repository
-COPY . /src/otbtf
+ && cmake ../otb/SuperBuild \
+ -DCMAKE_INSTALL_PREFIX=/opt/otbtf \
+ -DOTB_BUILD_FeaturesExtraction=ON \
+ -DOTB_BUILD_Hyperspectral=ON \
+ -DOTB_BUILD_Learning=ON \
+ -DOTB_BUILD_Miscellaneous=ON \
+ -DOTB_BUILD_RemoteModules=ON \
+ -DOTB_BUILD_SAR=ON \
+ -DOTB_BUILD_Segmentation=ON \
+ -DOTB_BUILD_StereoProcessing=ON \
+ -DDOWNLOAD_LOCATION=/tmp/SuperBuild-downloads \
+ && make -j $(python -c "import os; print(round( os.cpu_count() * $CPU_RATIO ))") \
+ && rm -rf /tmp/SuperBuild-downloads
+
+# Copy cpp and cmake files from build context (TODO: use `COPY --parents` feature when released)
+WORKDIR /src/otbtf
+COPY app ./app
+COPY include ./include
+COPY CMakeLists.txt otb-module.cmake ./
+RUN mkdir test
+COPY test/CMakeLists.txt test/*.cxx test/
+
+# Build OTBTF cpp
+ARG DEV_IMAGE=false
RUN ln -s /src/otbtf /src/otb/otb/Modules/Remote/otbtf
-
-# Rebuild OTB with module
-ARG KEEP_SRC_OTB=false
RUN cd /src/otb/build/OTB/build \
- && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/otbtf/lib \
- && export PATH=$PATH:/opt/otbtf/bin \
&& cmake /src/otb/otb \
-DCMAKE_INSTALL_PREFIX=/opt/otbtf \
- -DOTB_WRAP_PYTHON=ON -DPYTHON_EXECUTABLE=/usr/bin/python3 \
- -DOTB_USE_TENSORFLOW=ON -DModule_OTBTensorflow=ON \
+ -DOTB_WRAP_PYTHON=ON \
+ -DPython_EXECUTABLE=$(which python) \
+ -DOTB_USE_TENSORFLOW=ON \
+ -DModule_OTBTensorflow=ON \
-Dtensorflow_include_dir=/opt/otbtf/include/tf \
- -DTENSORFLOW_CC_LIB=/opt/otbtf/local/lib/python3.10/dist-packages/tensorflow/libtensorflow_cc.so.2 \
- -DTENSORFLOW_FRAMEWORK_LIB=/opt/otbtf/local/lib/python3.10/dist-packages/tensorflow/libtensorflow_framework.so.2 \
+ -DTENSORFLOW_CC_LIB=$PYTHON_SITE_PACKAGES/tensorflow/libtensorflow_cc.so.2 \
+ -DTENSORFLOW_FRAMEWORK_LIB=$PYTHON_SITE_PACKAGES/tensorflow/libtensorflow_framework.so.2 \
+ $( [ "$DEV_IMAGE" != "true" ] || echo "-DBUILD_TESTING=ON" ) \
&& make install -j $(python -c "import os; print(round( os.cpu_count() * $CPU_RATIO ))") \
- # Cleaning
- && ( $KEEP_SRC_OTB || rm -rf /src/otb ) \
+ && ( [ "$DEV_IMAGE" = "true" ] || rm -rf /src/otb ) \
&& rm -rf /root/.cache /tmp/*
-# Symlink executable python files in PATH
-RUN for f in /src/otbtf/python/*.py; do if [ -x $f ]; then ln -s $f /opt/otbtf/bin/; fi; done
-
# ----------------------------------------------------------------------------
-# Final stage
-FROM otbtf-base
+# Final stage: copy binaries from middle layers and install python module
+FROM base-stage AS final-stage
LABEL maintainer="Remi Cresson <remi.cresson[at]inrae[dot]fr>"
-# Copy files from intermediate stage
-COPY --from=builder /opt/otbtf /opt/otbtf
-COPY --from=builder /src /src
-
# System-wide ENV
-ENV PATH="/opt/otbtf/bin:$PATH"
-ENV LD_LIBRARY_PATH="/opt/otbtf/lib:$LD_LIBRARY_PATH"
-ENV PYTHONPATH="/opt/otbtf/lib/python3/dist-packages:/opt/otbtf/lib/otb/python"
-ENV OTB_APPLICATION_PATH="/opt/otbtf/lib/otb/applications"
-RUN pip install -e /src/otbtf
-
-# Default user, directory and command (bash is the entrypoint when using
-# 'docker create')
-RUN useradd -s /bin/bash -m otbuser
-WORKDIR /home/otbuser
+ENV OTB_INSTALL_DIR=/opt/otbtf
+ENV OTB_APPLICATION_PATH=/opt/otbtf/lib/otb/applications
+# For otbApplication and osgeo modules
+ENV PYTHONPATH="/opt/otbtf/lib/otb/python:/opt/otbtf/lib/python$PY/site-packages"
-# Admin rights without password
-ARG SUDO=true
-RUN if $SUDO; then \
- usermod -a -G sudo otbuser \
- && echo "otbuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers; fi
+# Add a standard user - this won't prevent ownership issues with volumes if you're not UID 1000
+RUN useradd -s /bin/bash -m otbuser
-# Set /src/otbtf ownership to otbuser (but you still need 'sudo -i' in order
-# to rebuild TF or OTB)
-RUN chown -R otbuser:otbuser /src/otbtf
+# Admin rights without password (not recommended, use `docker run -u root` instead)
+ARG SUDO=false
+RUN ! $SUDO || usermod -a -G sudo otbuser && echo "otbuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
-# This won't prevent ownership problems with volumes if you're not UID 1000
+# Allow user to install packages in prefix /opt/otbtf and venv without being root
+COPY --from=otb-build --chown=otbuser:otbuser /opt/otbtf /opt/otbtf
+COPY --from=otb-build --chown=otbuser:otbuser /src /src
USER otbuser
-# User-only ENV
-ENV PATH="/home/otbuser/.local/bin:$PATH"
+# Install OTBTF python module
+WORKDIR /src/otbtf
+COPY otbtf ./otbtf
+COPY README.md pyproject.toml .
+RUN pip install -e .
+
+# Install test packages for dev image
+RUN ! $DEV_IMAGE || pip install codespell flake8 pylint pytest pytest-cov pytest-order
+
+WORKDIR /home/otbuser
# Test python imports
-RUN python -c "import tensorflow"
-RUN python -c "import otbtf, tricks"
+RUN python -c "import tensorflow, keras"
RUN python -c "import otbApplication as otb; otb.Registry.CreateApplication('ImageClassifierFromDeepFeatures')"
+RUN python -c "import otbtf"
RUN python -c "from osgeo import gdal"
diff --git a/README.md b/README.md
index 9704bb430444a7e944c914c6d601ba1fae71103f..dc7683d14455d472e31576826a8b4fe38a1c332a 100644
--- a/README.md
+++ b/README.md
@@ -34,8 +34,8 @@ The documentation is available on [otbtf.readthedocs.io](https://otbtf.readthedo
You can use our latest GPU enabled docker images.
```bash
-docker run --runtime=nvidia -ti mdl4eo/otbtf:latest-gpu otbcli_PatchesExtraction
-docker run --runtime=nvidia -ti mdl4eo/otbtf:latest-gpu python -c "import otbtf"
+docker run --gpus=all -ti mdl4eo/otbtf:latest-gpu otbcli_PatchesExtraction
+docker run --gpus=all -ti mdl4eo/otbtf:latest-gpu python -c "import otbtf"
```
You can also build OTBTF from sources (see the documentation)
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index af4e4429edcf715d6e99305ab5ed096cce5f21d1..75d6e4b1c0cc4a7e715337a4281e4741d8f81cb8 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -1,3 +1,18 @@
+Version 4.4.0 (?? ??? ????)
+----------------------------------------------------------------
+* Bump OTB version to 9.1.0
+* Bump TensorFlow version to 2.18.0
+* Move python pip install to virtualenv /opt/otbtf/venv
+* Lock numpy version < 2.0
+* Upgrade TF build workflow to latest specs:
+ - Drop Docker build argument BASE_IMG (ubuntu jammy is used for every build)
+ - Use official bazel configs (targets --config=release_{cpu,gpu}) for wheels
+ - Compile with LLVM Clang 18, use "HERMETIC_CUDA" 12.5
+ - Use default compute capabilities from .bazelrc (sm_60,sm_70,sm_80,sm_89,compute_90)
+ - Add Docker build arguments WITH_CUDA=false, WITH_MKL=false, WITH_XLA=true
+ - TODO: update python code for Keras 3
+* TODO: move packaging spec to pyproject.toml
+
Version 4.3.1 (02 jan 2024)
----------------------------------------------------------------
* Fix a bug with PROJ due to OTB 9 packaging
diff --git a/doc/api_distributed.md b/doc/api_distributed.md
index 09ea86c35d976ba5de5cfe70d612ddf06afa4020..786a6f5cc6d9cfded3c7ef812acbcb6b54de88ae 100644
--- a/doc/api_distributed.md
+++ b/doc/api_distributed.md
@@ -98,7 +98,7 @@ The rest of the code is identical.
!!! Warning
- Be careful when calling `mymodel.save()` to export the SavedModel. When
+ Be careful when calling `mymodel.export()` to export the SavedModel. When
multiple nodes are used in parallel, this can lead to a corrupt save.
One good practice is to defer the call only to the master worker (e.g. node
0). You can identify the master worker using `otbtf.model._is_chief()`.
diff --git a/doc/api_tutorial.md b/doc/api_tutorial.md
index cc08b9198469a97a083c16588976a9b2c9aa8785..9c47be74774afffd4d4cf21b59b5409b7796d69b 100644
--- a/doc/api_tutorial.md
+++ b/doc/api_tutorial.md
@@ -315,12 +315,24 @@ dataset
```python
model.compile(
- loss=tf.keras.losses.CategoricalCrossentropy(),
+ loss={TARGET_NAME: tf.keras.losses.CategoricalCrossentropy()},
optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4),
- metrics=[tf.keras.metrics.Precision(), tf.keras.metrics.Recall()]
+ metrics={
+ TARGET_NAME: [
+ tf.keras.metrics.Precision(),
+ tf.keras.metrics.Recall()
+ ]
+ }
)
```
+!!! Note
+
+ The losses and metrics must always be provided using a dict, to specify
+ which output to use. This is mandatory since Keras 3, since OTBTF generates
+ a bunch of extra outputs that are not used during optimization, but needed
+ in the inference step.
+
We can then train our model using Keras:
```python
diff --git a/doc/app_sampling.md b/doc/app_sampling.md
index b79ba7f4653e5c1951c85757571430b17fa63580..9a9fc047e2c85d4b3f410d28271b9e15c4ffa8d3 100644
--- a/doc/app_sampling.md
+++ b/doc/app_sampling.md
@@ -84,7 +84,7 @@ specific field of the input vector data.
Typically, the *class* field can be used to generate a dataset suitable for a
model that performs pixel wise classification.
-
+
The application description can be displayed using:
diff --git a/doc/app_training.md b/doc/app_training.md
index eb3f4708f37121d5f63181eb2c2b08a783da9df9..c1e731ef65b69f936291a74a5377a25c3e8d238f 100644
--- a/doc/app_training.md
+++ b/doc/app_training.md
@@ -47,7 +47,7 @@ patches images, a convenient method consist in reading patches images as numpy
arrays using OTB applications (e.g. `ExtractROI`) or GDAL, then do a
`numpy.reshape` to the dimensions wanted.
-
+
The application description can be displayed using:
diff --git a/doc/deprecated.md b/doc/deprecated.md
index 3f76c1dbd1d62e3313da1e85e4d3b8e8ecef8bdf..08f9e73ebef7fa2815cac8843ae0a63f51184206 100644
--- a/doc/deprecated.md
+++ b/doc/deprecated.md
@@ -35,4 +35,13 @@ training, etc. is done using the so-called `tensorflow.Strategy`
!!! Note
- Read our [tutorial](api_tutorial.html) to know more on working with Keras!
\ No newline at end of file
+ Read our [tutorial](api_tutorial.html) to know more on working with Keras!
+
+## Major changes between Keras 2 and 3
+
+- Use keras functions on keras objects, instead of tf ones
+- Most operations in `tf` namespace have moved to `keras.ops`
+- Function `model.save()` should be replaced by `model.export()`
+- Target layers for metrics must be explicitly named
+
+Read further instructions in the official [keras docs](https://keras.io/guides/migrating_to_keras_3/).
diff --git a/doc/docker_build.md b/doc/docker_build.md
index debc1ea8d5ae2994949b46a61e70a5ed14290ce3..80393b1eee5e3f8b37b764c2f30701c46b857154 100644
--- a/doc/docker_build.md
+++ b/doc/docker_build.md
@@ -1,162 +1,129 @@
# Build your own docker images
-Docker build has to be called from the root of the repository (i.e. `docker
-build .` or `bash tools/docker/multibuild.sh`).
-You can build a custom image using `--build-arg` and several config files :
+Docker build has to be called from the root of the repository
+ (i.e. `docker build .`.
+You can select target versions using `--build-arg`:
-- **Ubuntu** : `BASE_IMG` should accept any version, for additional packages
-see *tools/docker/build-deps-cli.txt* and *tools/docker/build-deps-gui.txt*.
-- **TensorFlow** : `TF` arg for the git branch or tag + *build-env-tf.sh* and
-BZL_* arguments for the build configuration. `ZIP_TF_BIN` allows you to save
-compiled binaries if you want to install it elsewhere.
-- **OrfeoToolBox** : `OTB` arg for the git branch or tag +
-*tools/docker/build-flags-otb.txt* to edit cmake flags. Set `KEEP_SRC_OTB` in
-order to preserve OTB git directory.
+- **TensorFlow** : `TF` arg for the git branch or tag
+- **OrfeoToolBox** : `OTB` arg for the git branch or tag,
+ set `KEEP_SRC_OTB` in order to preserve OTB sources
-### Base images
+## Default build arguments
```bash
-CPU_IMG=ubuntu:22.04
-GPU_IMG=nvidia/cuda:12.1.0-devel-ubuntu22.04
-```
-
-### Default arguments
-
-```bash
-BASE_IMG # mandatory
+# Limit CPU usage e.g. 0.75
CPU_RATIO=1
-GUI=false
-NUMPY_SPEC="==1.19.*"
-TF=v2.12.0
-OTB=8.1.0
-BZL_TARGETS="//tensorflow:libtensorflow_cc.so //tensorflow/tools/pip_package:build_pip_package"
-BZL_CONFIGS="--config=nogcp --config=noaws --config=nohdfs --config=opt"
-BZL_OPTIONS="--verbose_failures --remote_cache=http://localhost:9090"
-ZIP_TF_BIN=false
-KEEP_SRC_OTB=false
-SUDO=true
-
-# NumPy version requirement :
-# TF < 2.4 : "numpy<1.19.0,>=1.16.0"
-# TF >= 2.4 : "numpy==1.19.*"
-# TF >= 2.8 : "numpy==1.22.*"
+# Can be used to install a specific numpy version
+NUMPY="1.26.4"
+# Git branch or tag to checkout
+TF=v2.18.0
+# Build with XLA
+WITH_XLA=true
+# Build with Intel MKL support
+WITH_MKL=false
+# Set to true to enable Nvidia GPU support
+WITH_CUDA=false
+# Custom compute capabilities, default are defined in repo tensorflow/.bazelrc
+# Currently "sm_60,sm_70,sm_80,sm_89,compute_90"
+CUDA_COMPUTE_CAPABILITIES=
+# Targets for bazel build cmd
+BZL_TARGETS="//tensorflow:libtensorflow_cc.so //tensorflow/tools/pip_package:wheel"
+# Available for additional bazel options, e.g. --remote_cache
+BZL_OPTIONS=
+# Git branch or tag to checkout
+OTB=release-9.1
+# Keep OTB sources and build test
+DEV_IMAGE=false
+# Enable sudo without password for "otbuser"
+SUDO=false
```
-### Bazel remote cache daemon
+## Bazel remote cache daemon
-If you just need to rebuild with different GUI or KEEP_SRC arguments, or may
-be a different branch of OTB, bazel cache will help you to rebuild everything
-except TF, even if the docker cache was purged (after `docker
-[system|builder] prune`).
-In order to recycle the cache, bazel config and TF git tag should be exactly
-the same, any change in *tools/docker/build-env-tf.sh* and `--build-arg`
-(if related to bazel env, cuda, mkl, xla...) may result in a fresh new build.
+If you just need to rebuild with different arguments, or may be a different
+ branch of OTB, bazel cache will help you to rebuild everything except TF,
+ even if the docker cache was purged (after `docker [system|builder] prune`).
+In order to recycle the cache, bazel config and TF git tag should be exactly
+ the same, any change in Dockerfile or `--build-arg` would create a new build.
-Start a cache daemon - here with max 20GB but 10GB should be enough to save 2
-TF builds (GPU and CPU):
+Start a cache daemon - 10GB should be enough to save 2 TF builds (GPU and CPU):
```bash
mkdir -p $HOME/.cache/bazel-remote
-docker run --detach -u 1000:1000 -v $HOME/.cache/bazel-remote:/data \
- -p 9090:8080 buchgr/bazel-remote-cache --max_size=20
+docker run --detach -u $UID:$GID -v $HOME/.cache/bazel-remote:/data \
+ -p 9090:8080 buchgr/bazel-remote-cache --max_size=10
```
-Then just add ` --network='host'` to the docker build command, or connect
-bazel to a remote server - see 'BZL_OPTIONS'.
-The other way of docker is a virtual bridge, but you'll need to edit the IP
-address.
+Then just add ` --network='host'` to the docker build command, or connect
+ bazel to a remote server - see 'BZL_OPTIONS'.
+The other way of docker is a virtual bridge, but you'll need to edit the IP
+ address. Changing the BZL_OPTIONS will invalidate docker build cache.
## Images build examples
```bash
# Build for CPU using default Dockerfiles args (without AWS, HDFS or GCP
# support)
-docker build --network='host' -t otbtf:cpu --build-arg BASE_IMG=ubuntu:22.04 .
-
-# Clear bazel config var (deactivate default optimizations and unset
-# noaws/nogcp/nohdfs)
docker build --network='host' -t otbtf:cpu \
- --build-arg BASE_IMG=ubuntu:22.04 \
- --build-arg BZL_CONFIGS= .
+ --build-arg BZL_OPTIONS="--remote_cache=http://localhost:9090" .
-# Enable MKL
-MKL_CONFIG="--config=nogcp --config=noaws --config=nohdfs --config=opt --config=mkl"
-docker build --network='host' -t otbtf:cpu-mkl \
- --build-arg BZL_CONFIGS="$MKL_CONFIG" \
- --build-arg BASE_IMG=ubuntu:22.04 .
-
-# Build for GPU (if you're building for your system only you should edit
-# CUDA_COMPUTE_CAPABILITIES in build-env-tf.sh)
+# Build for GPU
docker build --network='host' -t otbtf:gpu \
- --build-arg BASE_IMG=nvidia/cuda:12.1.0-devel-ubuntu22.04 .
+ --build-arg BZL_OPTIONS="--remote_cache=http://localhost:9090" \
+ --build-arg WITH_CUDA=true \
+ .
# Build latest TF and OTB, set git branches/tags to clone
-docker build --network='host' -t otbtf:gpu-dev \
- --build-arg BASE_IMG=nvidia/cuda:12.1.0-devel-ubuntu22.04 \
- --build-arg KEEP_SRC_OTB=true \
+docker build --network='host' -t otbtf:gpu \
+ --build-arg BZL_OPTIONS="--remote_cache=http://localhost:9090" \
+ --build-arg WITH_CUDA=true \
+ --build-arg DEV_IMAGE=true \
--build-arg TF=nightly \
- --build-arg OTB=develop .
-
-# Build old release (TF-2.1)
-docker build --network='host' -t otbtf:oldstable-gpu \
- --build-arg BASE_IMG=nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04 \
- --build-arg TF=r2.1 \
- --build-arg NUMPY_SPEC="<1.19" \
- --build-arg BAZEL_OPTIONS="--noincompatible_do_not_split_linking_cmdline --verbose_failures --remote_cache=http://localhost:9090" .
-# You could edit the Dockerfile in order to clone an old branch of the repo
-# instead of copying files from the build context
+ --build-arg OTB=develop \
+ .
```
-### Build for another machine and save TF compiled files
-
-Example with TF 2.5
+### Build only tensorflow C++ lib and python wheel to install outside of Docker
```bash
-# Use same ubuntu and CUDA version than your target machine, beware of CC
-# optimization and CPU compatibility (set env variable CC_OPT_FLAGS and avoid
-# "-march=native" if your Docker's CPU is optimized with AVX2/AVX512 but your
-# target CPU isn't)
-docker build --network='host' -t otbtf:custom \
- --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 \
- --build-arg TF=v2.5.0 \
- --build-arg ZIP_TF_BIN=true .
-# Retrieve zip file
-docker run -v $HOME:/home/otbuser/volume otbtf:custom \
- cp /opt/otbtf/tf-v2.5.0.zip /home/otbuser/volume
+docker build --network='host' --target=tf-build -t tf:gpu \
+ --build-arg BZL_OPTIONS="--remote_cache=http://localhost:9090" \
+ --build-arg WITH_CUDA=true \
+ .
# Target machine shell
-cd $HOME
-unzip tf-v2.5.0.zip
-sudo mkdir -p /opt/tensorflow/lib
-sudo mv tf-v2.5.0/libtensorflow_cc* /opt/tensorflow/lib
+docker run -v gpu-build-artifacts:/artifacts tf:gpu mv /tmp/artifacts /artifacts
+cd gpu-build-artifacts
+sudo mv libtensorflow_cc* /usr/local/lib # Or another path you may add to LD_LIBRARY_PATH
# You may need to create a virtualenv, here TF and dependencies are installed
# next to user's pip packages
-pip3 install -U pip wheel mock six future deprecated "numpy==1.19.*"
+pip3 install -U pip wheel mock six future deprecated "numpy<2"
pip3 install --no-deps keras_applications keras_preprocessing
-pip3 install tf-v2.5.0/tensorflow-2.5.0-cp38-cp38-linux_x86_64.whl
+pip3 install tensorflow-v2.18.0-cp310-cp310-linux_x86_64.whl
-TF_WHEEL_DIR="$HOME/.local/lib/python3.8/site-packages/tensorflow"
+TF_WHEEL_DIR="$HOME/.local/lib/python3.10/site-packages/tensorflow"
# If you installed the wheel as regular user, with root pip it should be in
# /usr/local/lib/python3.*, or in your virtualenv lib/ directory
-mv tf-v2.5.0/tag_constants.h $TF_WHEEL_DIR/include/tensorflow/cc/saved_model/
-# Then recompile OTB with OTBTF using libraries in /opt/tensorflow/lib and
+mv tag_constants.h signature_constants.h $TF_WHEEL_DIR/include/tensorflow/cc/saved_model/
+# From an OTB git source tree
+# Recompile OTB with OTBTF using libraries in /opt/tensorflow/lib and
# instructions in build_from_sources.md.
cmake $OTB_GIT \
- -DOTB_USE_TENSORFLOW=ON -DModule_OTBTensorflow=ON \
- -DTENSORFLOW_CC_LIB=/opt/tensorflow/lib/libtensorflow_cc.so.2 \
- -Dtensorflow_include_dir=$TF_WHEEL_DIR/include \
- -DTENSORFLOW_FRAMEWORK_LIB=$TF_WHEEL_DIR/libtensorflow_framework.so.2 \
+ -DOTB_USE_TENSORFLOW=ON \
+ -DModule_OTBTensorflow=ON \
+ -DTENSORFLOW_CC_LIB="/usr/local/lib/libtensorflow_cc.so.2" \
+ -Dtensorflow_include_dir="$TF_WHEEL_DIR/include" \
+ -DTENSORFLOW_FRAMEWORK_LIB="$TF_WHEEL_DIR/libtensorflow_framework.so.2" \
&& make install -j
```
### Debug build
-
-If you fail to build, you can log into the last layer and check CMake logs.
-Run `docker images`, find the latest layer ID and run a tmp container
+If you fail to build, you can log into the last layer and check CMake logs.
+ Run `docker images`, find the latest layer ID and run a tmp container
(`docker run -it d60496d9612e bash`).
+**This is only possible when building with legacy docker config DOCKER_BUILDKIT=0**.
You may also need to split some multi-command layers in the Dockerfile.
-If you see OOM errors during SuperBuild you should decrease CPU_RATIO (e.g.
-0.75).
+ If you see OOM errors during SuperBuild you should decrease CPU_RATIO.
## Container examples
@@ -164,7 +131,7 @@ If you see OOM errors during SuperBuild you should decrease CPU_RATIO (e.g.
# Pull GPU image and create a new container with your home directory as volume
# (requires apt package nvidia-docker2 and CUDA>=11.0)
docker create --gpus=all --volume $HOME:/home/otbuser/volume -it \
- --name otbtf-gpu mdl4eo/otbtf:3.3.2-gpu
+ --name otbtf-gpu mdl4eo/otbtf:5.0.0-gpu
# Run interactive
docker start -i otbtf-gpu
@@ -180,32 +147,14 @@ docker exec otbtf-gpu \
Enter a development ready docker image:
```bash
-docker create --gpus=all -it --name otbtf-gpu-dev mdl4eo/otbtf:3.3.2-gpu-dev
-docker start -i otbtf-gpu-dev
-```
-
-Then, from the container shell:
-
-```bash
-sudo -i
+docker run -it --gpus=all -it --name otbtf-gpu-dev mdl4eo/otbtf:5.0.0-gpu-dev
+# Then, from the container shell:
cd /src/otb/otb/Modules/Remote
git clone https://gitlab.irstea.fr/raffaele.gaetano/otbSelectiveHaralickTextures.git
cd /src/otb/build/OTB/build
cmake -DModule_OTBAppSelectiveHaralickTextures=ON /src/otb/otb && make install -j
+exit
+docker container ls
```
-### Container with GUI
-
-GUI is disabled by default in order to save space, and because docker xvfb
-isn't working properly with OpenGL.
-OTB GUI seems OK but monteverdi isn't working
-
-```bash
-docker build --network='host' -t otbtf:cpu-gui \
- --build-arg BASE_IMG=ubuntu:22.04 \
- --build-arg GUI=true .
-docker create -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=$DISPLAY -it \
- --name otbtf-gui otbtf:cpu-gui
-docker start -i otbtf-gui
-$ mapla
-```
+Then you can user `docker commit` to save this container as a new image.
diff --git a/doc/docker_troubleshooting.md b/doc/docker_troubleshooting.md
index 4aa0d506d32aa42c264d580e5362fe37aa72e229..c5096717f8fcc43fa5cecef74f8cf680a69efae7 100644
--- a/doc/docker_troubleshooting.md
+++ b/doc/docker_troubleshooting.md
@@ -1,34 +1,24 @@
# Docker troubleshooting
-You can find plenty of help on the web about docker.
-This section only provides the basics for newcomers that are eager to use
-OTBTF!
+You can find plenty of help on the web about docker.
+This section only provides the basics for newcomers
+ that are eager to use OTBTF!
This section is largely inspired from the
-[moringa docker help](https://gitlab.irstea.fr/raffaele.gaetano/moringa/blob/develop/docker/README.md).
-Big thanks to the authors.
+[moringa docker help](https://gitlab.irstea.fr/raffaele.gaetano/moringa/blob/develop/docker/README.md).
+ Big thanks to the authors.
## Common errors
-### Manifest unknown
-
-```
-Error response from daemon:
-manifest for nvidia/cuda:11.0-cudnn8-devel-ubuntu20.04 not found:
-manifest unknown: manifest unknown
-```
-
-This means that the docker image is missing from dockerhub.
-
### failed call to cuInit
-```
+```raw
failed call to cuInit:
UNKNOWN ERROR (303) / no NVIDIA GPU device is present:
/dev/nvidia0 does not exist
```
-Nvidia driver is missing or disabled, make sure to add
-` --gpus=all` to your docker run or create command
+Nvidia driver is missing or disabled, make sure to add
+ ` --gpus=all` to your docker run or create command
## Useful diagnostic commands
@@ -87,7 +77,7 @@ docker create --interactive --tty --volume /home/$USER:/home/otbuser/ \
!!! warning
Beware of ownership issues, see
- [this section](#fix-volume-ownership-sissues).
+ [this section](#fix-volume-ownership-issues).
### Interactive session
diff --git a/doc/docker_use.md b/doc/docker_use.md
index 81211066432c95f60e546ae815ec31acaca8f593..0f9d95a935d71c17d7cce3cb2d9f3cf72e5db4ea 100644
--- a/doc/docker_use.md
+++ b/doc/docker_use.md
@@ -21,16 +21,18 @@ Read more in the following sections.
Here is the list of the latest OTBTF docker images hosted on
[dockerhub](https://hub.docker.com/u/mdl4eo).
Since OTBTF >= 3.2.1 you can find the latest docker images on
-[gitlab.irstea.fr](https://gitlab.irstea.fr/remi.cresson/otbtf/container_registry).
+[gitlab.irstea.fr](https://gitlab.irstea.fr/remi.cresson/otbtf/container_registry) for
+versions <= 4.3.0 and [forgemia.inra.fr](https://forgemia.inra.fr/orfeo-toolbox/otbtf/container_registry/) since version 4.3.1.
| Name | Os | TF | OTB | Description | Dev files | Compute capability |
|------------------------------------------------------------------------------------| ------------- |-------|-------| ---------------------- | --------- | ------------------ |
-| **mdl4eo/otbtf:4.3.1-cpu** | Ubuntu Jammy | r2.14 | 9.0.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:4.3.1-cpu-dev** | Ubuntu Jammy | r2.14 | 9.0.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:4.3.1-gpu** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:4.3.1-gpu-dev** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.3.1-gpu-opt** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.3.1-gpu-opt-dev** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:5.0.0-cpu** | Ubuntu Jammy | r2.18 | 9.1.0 | CPU | no | |
+| **mdl4eo/otbtf:5.0.0-cpu-dev** | Ubuntu Jammy | r2.18 | 9.1.0 | CPU (dev) | yes | |
+| **mdl4eo/otbtf:5.0.0-gpu** | Ubuntu Jammy | r2.18 | 9.1.0 | GPU | no | sm_60,sm_70,sm_80,sm_89,compute_90 |
+| **mdl4eo/otbtf:5.0.0-gpu-dev** | Ubuntu Jammy | r2.18 | 9.1.0 | GPU (dev) | yes | sm_60,sm_70,sm_80,sm_89,compute_90|
+[gitlab.irstea.fr](https://gitlab.irstea.fr/remi.cresson/otbtf/container_registry)
+(before otbtf 4.3.0) and [forgemia](https://forgemia.inra.fr/orfeo-toolbox/otbtf/container_registry)
+(since otbtf 4.3.1).
The list of older releases is available [here](#older-images).
@@ -43,10 +45,9 @@ The list of older releases is available [here](#older-images).
## GPU enabled docker
-In Linux, this is quite straightforward.
-Just follow the steps described in the
-[nvidia-docker documentation](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html).
-You can then use the OTBTF `gpu` tagged docker images with the **NVIDIA runtime** :
+In Linux, this is quite straightforward. Just follow the steps described in the
+ [nvidia-docker documentation](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html).
+You can then use the OTBTF `gpu` tagged docker images with the **NVIDIA runtime** :
With Docker version earlier than 19.03 :
@@ -57,7 +58,7 @@ docker run --runtime=nvidia -ti mdl4eo/otbtf:latest-gpu bash
With Docker version including and after 19.03 :
```bash
-docker run --gpus all -ti mdl4eo/otbtf:latest-gpu bash
+docker run --gpus=all -ti mdl4eo/otbtf:latest-gpu bash
```
You can find some details on the **GPU docker image** and some **docker tips
@@ -99,7 +100,6 @@ Troubleshooting:
- [WSL user guide](https://docs.nvidia.com/cuda/wsl-user-guide/index.html)
- [XSL GPU support](https://docs.docker.com/docker-for-windows/wsl/#gpu-support)
-
## Build your own images
If you want to use optimization flags, change GPUs compute capability, etc.
@@ -140,44 +140,30 @@ Here you can find the list of older releases of OTBTF:
| **mdl4eo/otbtf:3.3.0-cpu-dev** | Ubuntu Focal | r2.8 | 8.1.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.0-gpu** | Ubuntu Focal | r2.8 | 8.1.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.0-gpu-dev** | Ubuntu Focal | r2.8 | 8.1.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.0-gpu-opt** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.0-gpu-opt-dev** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.2-cpu** | Ubuntu Focal | r2.8 | 8.1.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.2-cpu-dev** | Ubuntu Focal | r2.8 | 8.1.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.2-gpu** | Ubuntu Focal | r2.8 | 8.1.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.2-gpu-dev** | Ubuntu Focal | r2.8 | 8.1.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.2-gpu-opt** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.2-gpu-opt-dev** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.3-cpu** | Ubuntu Focal | r2.8 | 8.1.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.3-cpu-dev** | Ubuntu Focal | r2.8 | 8.1.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.3-gpu** | Ubuntu Focal | r2.8 | 8.1.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.3.3-gpu-dev** | Ubuntu Focal | r2.8 | 8.1.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.3-gpu-opt** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.3-gpu-opt-dev** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.4.0-cpu** | Ubuntu Focal | r2.8 | 8.1.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.4.0-cpu-dev** | Ubuntu Focal | r2.8 | 8.1.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.4.0-gpu** | Ubuntu Focal | r2.8 | 8.1.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.4.0-gpu-dev** | Ubuntu Focal | r2.8 | 8.1.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.4.0-gpu-opt** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.4.0-gpu-opt-dev** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:4.0.0-cpu** | Ubuntu Jammy | r2.12 | 8.1.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:4.0.0-cpu-dev** | Ubuntu Jammy | r2.12 | 8.1.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:4.0.0-gpu** | Ubuntu Jammy | r2.12 | 8.1.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:4.0.0-gpu-dev** | Ubuntu Jammy | r2.12 | 8.1.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.0.0-gpu-opt** | Ubuntu Jammy | r2.12 | 8.1.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.0.0-gpu-opt-dev** | Ubuntu Jammy | r2.12 | 8.1.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:4.1.0-cpu** | Ubuntu Jammy | r2.12 | 8.1.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:4.1.0-cpu-dev** | Ubuntu Jammy | r2.12 | 8.1.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:4.1.0-gpu** | Ubuntu Jammy | r2.12 | 8.1.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:4.1.0-gpu-dev** | Ubuntu Jammy | r2.12 | 8.1.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.1.0-gpu-opt** | Ubuntu Jammy | r2.12 | 8.1.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.1.0-gpu-opt-dev** | Ubuntu Jammy | r2.12 | 8.1.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:4.2.0-cpu** | Ubuntu Jammy | r2.12 | 8.1.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:4.2.0-cpu-dev** | Ubuntu Jammy | r2.12 | 8.1.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:4.2.0-gpu** | Ubuntu Jammy | r2.12 | 8.1.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:4.2.0-gpu-dev** | Ubuntu Jammy | r2.12 | 8.1.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.2.0-gpu-opt** | Ubuntu Jammy | r2.12 | 8.1.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.2.0-gpu-opt-dev** | Ubuntu Jammy | r2.12 | 8.1.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:4.2.1-cpu** | Ubuntu Jammy | r2.12 | 8.1.2 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:4.2.1-cpu-dev** | Ubuntu Jammy | r2.12 | 8.1.2 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:4.2.1-gpu** | Ubuntu Jammy | r2.12 | 8.1.2 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
@@ -196,3 +182,7 @@ Here you can find the list of older releases of OTBTF:
| **mdl4eo/otbtf:4.3.0-gpu-dev** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.3.0-gpu-opt** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:4.3.0-gpu-opt-dev** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.3.1-cpu** | Ubuntu Jammy | r2.14 | 9.0.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.3.1-cpu-dev** | Ubuntu Jammy | r2.14 | 9.0.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.3.1-gpu** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:4.3.1-gpu-dev** | Ubuntu Jammy | r2.14 | 9.0.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
diff --git a/otbtf/__init__.py b/otbtf/__init__.py
index 1ce624226bcf921418b5de8731eb1566d1486055..633ccf1bbbc6c05f3e84536b25c7c8bbc3e06e25 100644
--- a/otbtf/__init__.py
+++ b/otbtf/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# ==========================================================================
#
# Copyright 2018-2019 IRSTEA
@@ -20,7 +19,9 @@
"""
OTBTF python module
"""
-import pkg_resources
+
+__version__ = "5.0.0-rc2"
+
try:
from otbtf.utils import read_as_np_arr, gdal_open # noqa
from otbtf.dataset import Buffer, PatchesReaderBase, PatchesImagesReader, \
@@ -34,4 +35,3 @@ except ImportError:
from otbtf.tfrecords import TFRecords # noqa
from otbtf.model import ModelBase # noqa
from otbtf import layers, ops # noqa
-__version__ = pkg_resources.require("otbtf")[0].version
diff --git a/otbtf/dataset.py b/otbtf/dataset.py
index cf2a0759e4c0b8e7788abb484737aae921e4727a..73ae88b9fccdad5ed78f9c37016acb4bebbdef29 100644
--- a/otbtf/dataset.py
+++ b/otbtf/dataset.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# ==========================================================================
#
# Copyright 2018-2019 IRSTEA
diff --git a/otbtf/examples/tensorflow_v1x/__init__.py b/otbtf/examples/tensorflow_v1x/__init__.py
index c77256a480843e5f109dfe04174daac4d76b44fb..796e4cbd9cd41936aef2c63a42a001a5c194dedc 100644
--- a/otbtf/examples/tensorflow_v1x/__init__.py
+++ b/otbtf/examples/tensorflow_v1x/__init__.py
@@ -38,7 +38,7 @@ Predicted label is a single pixel, for an input patch of size 16x16 (for an inpu
The learning rate of the training operator can be adjusted using the *lr* placeholder.
The following figure summarizes this architecture.
-<img src ="https://gitlab.irstea.fr/remi.cresson/otbtf/-/raw/develop/doc/images/savedmodel_simple_cnn.png" />
+<img src ="https://forgemia.inra.fr/orfeo-toolbox/otbtf/-/raw/develop/doc/images/savedmodel_simple_cnn.png" />
## Generate the model
@@ -143,7 +143,7 @@ otbcli_TensorflowModelServe \\
The `create_savedmodel_simple_fcn.py` script enables you to create a fully
convolutional model which does not use any stride.
-<img src ="https://gitlab.irstea.fr/remi.cresson/otbtf/-/raw/develop/doc/images/savedmodel_simple_fcnn.png" />
+<img src ="https://forgemia.inra.fr/orfeo-toolbox/otbtf/-/raw/develop/doc/images/savedmodel_simple_fcnn.png" />
Thanks to that, once trained this model can be applied on the image to produce
a landcover map at the same resolution as the input image, in a fully
@@ -208,7 +208,7 @@ available parameters.
Let's train the M3 model from time series (TS) and Very High Resolution
Satellite (VHRS) patches images.
-<img src ="https://gitlab.irstea.fr/remi.cresson/otbtf/-/raw/develop/doc/images/model_training.png" />
+<img src ="https://forgemia.inra.fr/orfeo-toolbox/otbtf/-/raw/develop/doc/images/model_training.png" />
First, tell OTBTF that we want two sources: one for time series + one for
VHR image
@@ -255,7 +255,7 @@ otbcli_TensorflowModelTrain \\
Let's produce a land cover map using the M3 model from time series (TS) and
Very High Resolution Satellite image (VHRS)
-<img src ="https://gitlab.irstea.fr/remi.cresson/otbtf/-/raw/develop/doc/images/classif_map.png" />
+<img src ="https://forgemia.inra.fr/orfeo-toolbox/otbtf/-/raw/develop/doc/images/classif_map.png" />
Since we provide time series as the reference source (*source1*), the output
classes are estimated at the same resolution. This model can be run in
@@ -363,7 +363,7 @@ See: Gaetano, R., Ienco, D., Ose, K., & Cresson, R. (2018). *A two-branch CNN
architecture for land cover classification of PAN and MS imagery*. Remote
Sensing, 10(11), 1746.
-<img src ="https://gitlab.irstea.fr/remi.cresson/otbtf/-/raw/develop/doc/images/savedmodel_simple_pxs_fcn.png" />
+<img src ="https://forgemia.inra.fr/orfeo-toolbox/otbtf/-/raw/develop/doc/images/savedmodel_simple_pxs_fcn.png" />
Use `create_savedmodel_pxs_fcn.py` to generate this model.
diff --git a/otbtf/examples/tensorflow_v2x/deterministic/__init__.py b/otbtf/examples/tensorflow_v2x/deterministic/__init__.py
index d0ec3db4654aa19a87b31cf32003b2c3a2d2418d..03bb3133f473b2304fc74c4ca17e3e1342e27dcd 100644
--- a/otbtf/examples/tensorflow_v2x/deterministic/__init__.py
+++ b/otbtf/examples/tensorflow_v2x/deterministic/__init__.py
@@ -29,11 +29,11 @@ import tensorflow as tf
x = tf.keras.Input(shape=[None, None, None], name="x") # [1, h, w, N]
# Compute norm on the last axis
-y = tf.norm(x, axis=-1)
+y = tf.keras.ops.norm(x, axis=-1)
# Create model
model = tf.keras.Model(inputs={"x": x}, outputs={"y": y})
-model.save("l2_norm_savedmodel")
+model.export("l2_norm_savedmodel")
```
Run the code. The *l2_norm_savedmodel* file is created.
@@ -65,18 +65,19 @@ Let's consider a simple model that inputs two multispectral image (*x1* and
The model is exported as a SavedModel named *scalar_product_savedmodel*
```python
-import tensorflow as tf
+import keras
# Input
-x1 = tf.keras.Input(shape=[None, None, None], name="x1") # [1, h, w, N]
-x2 = tf.keras.Input(shape=[None, None, None], name="x2") # [1, h, w, N]
+x1 = keras.Input(shape=[None, None, None], name="x1") # [1, h, w, N]
+x2 = keras.Input(shape=[None, None, None], name="x2") # [1, h, w, N]
# Compute scalar product
-y = tf.reduce_sum(tf.multiply(x1, x2), axis=-1)
+y = keras.ops.sum(keras.ops.multiply(x1, x2), axis=-1)
# Create model
-model = tf.keras.Model(inputs={"x1": x1, "x2": x2}, outputs={"y": y})
-model.save("scalar_product_savedmodel")
+model = keras.Model(inputs={"x1": x1, "x2": x2}, outputs={"y": y})
+model.export("scalar_product_savedmodel")
+
```
Run the code. The *scalar_product_savedmodel* file is created.
diff --git a/otbtf/examples/tensorflow_v2x/deterministic/l2_norm.py b/otbtf/examples/tensorflow_v2x/deterministic/l2_norm.py
index b23d86cb4016dcdd5eecb7eff65e8487392d4661..59e2b0a409de565b3810b22bf1cbeeb512bc06f5 100644
--- a/otbtf/examples/tensorflow_v2x/deterministic/l2_norm.py
+++ b/otbtf/examples/tensorflow_v2x/deterministic/l2_norm.py
@@ -14,14 +14,15 @@ otbcli_TensorflowModelServe \
```
"""
-import tensorflow as tf
+import keras
+
# Input
-x = tf.keras.Input(shape=[None, None, None], name="x") # [1, h, w, N]
+x = keras.Input(shape=[None, None, None], name="x") # [1, h, w, N]
# Compute norm on the last axis
-y = tf.norm(x, axis=-1)
+y = keras.ops.norm(x, axis=-1)
# Create model
-model = tf.keras.Model(inputs={"x": x}, outputs={"y": y})
-model.save("l2_norm_savedmodel")
+model = keras.Model(inputs={"x": x}, outputs={"y": y})
+model.export("l2_norm_savedmodel")
diff --git a/otbtf/examples/tensorflow_v2x/deterministic/scalar_prod.py b/otbtf/examples/tensorflow_v2x/deterministic/scalar_prod.py
index 57127c5e01667dceeef23720ba550fecb0451f2c..c75d36313dda16c00cb425a9933a6a5aa40f24c3 100644
--- a/otbtf/examples/tensorflow_v2x/deterministic/scalar_prod.py
+++ b/otbtf/examples/tensorflow_v2x/deterministic/scalar_prod.py
@@ -16,15 +16,16 @@ OTB_TF_NSOURCES=2 otbcli_TensorflowModelServe \
```
"""
-import tensorflow as tf
+
+import keras
# Input
-x1 = tf.keras.Input(shape=[None, None, None], name="x1") # [1, h, w, N]
-x2 = tf.keras.Input(shape=[None, None, None], name="x2") # [1, h, w, N]
+x1 = keras.Input(shape=[None, None, None], name="x1") # [1, h, w, N]
+x2 = keras.Input(shape=[None, None, None], name="x2") # [1, h, w, N]
# Compute scalar product
-y = tf.reduce_sum(tf.multiply(x1, x2), axis=-1)
+y = keras.ops.sum(keras.ops.multiply(x1, x2), axis=-1)
# Create model
-model = tf.keras.Model(inputs={"x1": x1, "x2": x2}, outputs={"y": y})
-model.save("scalar_product_savedmodel")
+model = keras.Model(inputs={"x1": x1, "x2": x2}, outputs={"y": y})
+model.export("scalar_product_savedmodel")
diff --git a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
index fcd14a2024e575f79a9f0c9a4a8e475a5e1d373b..4259390f3ea301fbd45aec50da462aef5d46c5e3 100644
--- a/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
+++ b/otbtf/examples/tensorflow_v2x/fcnn/fcnn_model.py
@@ -1,16 +1,18 @@
"""
Implementation of a small U-Net like model
"""
+
import logging
import tensorflow as tf
+import keras
from otbtf.model import ModelBase
logging.basicConfig(
- format='%(asctime)s %(levelname)-8s %(message)s',
+ format="%(asctime)s %(levelname)-8s %(message)s",
level=logging.INFO,
- datefmt='%Y-%m-%d %H:%M:%S'
+ datefmt="%Y-%m-%d %H:%M:%S",
)
# Number of classes estimated by the model
@@ -19,13 +21,13 @@ N_CLASSES = 2
# Name of the input in the `FCNNModel` instance, also name of the input node
# in the SavedModel
INPUT_NAME = "input_xs"
+INPUT_SIGNATURE = tf.TensorSpec(
+ shape=[None, None, None, 4], dtype=tf.float32, name=INPUT_NAME
+)
# Name of the output in the `FCNNModel` instance
TARGET_NAME = "predictions"
-# Name (prefix) of the output node in the SavedModel
-OUTPUT_SOFTMAX_NAME = "predictions_softmax_tensor"
-
class FCNNModel(ModelBase):
"""
@@ -51,7 +53,7 @@ class FCNNModel(ModelBase):
Returns:
dict of normalized inputs, ready to be used from `get_outputs()`
"""
- return {INPUT_NAME: tf.cast(inputs[INPUT_NAME], tf.float32) * 0.0001}
+ return {INPUT_NAME: keras.ops.cast(inputs[INPUT_NAME], tf.float32) * 0.0001}
def get_outputs(self, normalized_inputs: dict) -> dict:
"""
@@ -71,24 +73,24 @@ class FCNNModel(ModelBase):
norm_inp = normalized_inputs[INPUT_NAME]
def _conv(inp, depth, name):
- conv_op = tf.keras.layers.Conv2D(
+ conv_op = keras.layers.Conv2D(
filters=depth,
kernel_size=3,
strides=2,
activation="relu",
padding="same",
- name=name
+ name=name,
)
return conv_op(inp)
def _tconv(inp, depth, name, activation="relu"):
- tconv_op = tf.keras.layers.Conv2DTranspose(
+ tconv_op = keras.layers.Conv2DTranspose(
filters=depth,
kernel_size=3,
strides=2,
activation=activation,
padding="same",
- name=name
+ name=name,
)
return tconv_op(inp)
@@ -101,40 +103,31 @@ class FCNNModel(ModelBase):
out_tconv3 = _tconv(out_tconv2, 16, "tconv3") + out_conv1
out_tconv4 = _tconv(out_tconv3, N_CLASSES, "classifier", None)
- # Generally it is a good thing to name the final layers of the network
- # (i.e. the layers of which outputs are returned from
- # `MyModel.get_output()`). Indeed this enables to retrieve them for
- # inference time, using their name. In case your forgot to name the
- # last layers, it is still possible to look at the model outputs using
- # the `saved_model_cli show --dir /path/to/your/savedmodel --all`
- # command.
- #
- # Do not confuse **the name of the output layers** (i.e. the "name"
- # property of the tf.keras.layer that is used to generate an output
- # tensor) and **the key of the output tensor**, in the dict returned
- # from `MyModel.get_output()`. They are two identifiers with a
- # different purpose:
- # - the output layer name is used only at inference time, to identify
- # the output tensor from which generate the output image,
- # - the output tensor key identifies the output tensors, mainly to
- # fit the targets to model outputs during training process, but it
- # can also be used to access the tensors as tf/keras objects, for
- # instance to display previews images in TensorBoard.
- softmax_op = tf.keras.layers.Softmax(name=OUTPUT_SOFTMAX_NAME)
+ softmax_op = keras.layers.Softmax()
predictions = softmax_op(out_tconv4)
- # note that we could also add additional outputs, for instance the
- # argmax of the softmax:
+ # Model outputs are returned in a `dict`, where each key is an output
+ # name, and the value is the layer output. This naming have two
+ # functions:
+ # - the output layer name is used at inference time, to identify
+ # the output tensor from which generate the output image,
+ # - the output layer name identifies the output tensors, to fit the
+ # targets to model outputs, compute metrics, etc. during training
+ # process. It can also be used to access the tensors as tf/keras
+ # objects, for instance to display previews images in TensorBoard.
#
- # argmax_op = otbtf.layers.Argmax(name="labels")
- # labels = argmax_op(predictions)
- # return {TARGET_NAME: predictions, OUTPUT_ARGMAX_NAME: labels}
+ # Note that we could also add additional outputs, even outputs which
+ # are useless for the optimization process, for instance the argmax :
+ # ```
+ # argmax_op = otbtf.layers.Argmax()
+ # labels = argmax_op(predictions)
+ # return {TARGET_NAME: predictions, OUTPUT_ARGMAX_NAME: labels}
+ # ```
# The default extra outputs (i.e. output tensors with cropping in
# physical domain) are append by `otbtf.ModelBase` for all returned
# outputs of this function to be used at inference time (e.g.
- # "labels_crop32", "labels_crop64", ...,
- # "predictions_softmax_tensor_crop16", ..., etc).
-
+ # "labels_crop32", "labels_crop64", ..., "predictions__crop16", ...,
+ # etc).
return {TARGET_NAME: predictions}
@@ -158,10 +151,12 @@ def dataset_preprocessing_fn(examples: dict):
"""
return {
INPUT_NAME: examples["input_xs_patches"],
- TARGET_NAME: tf.one_hot(
- tf.squeeze(tf.cast(examples["labels_patches"], tf.int32), axis=-1),
- depth=N_CLASSES
- )
+ TARGET_NAME: keras.ops.one_hot(
+ keras.ops.squeeze(
+ keras.ops.cast(examples["labels_patches"], tf.int32), axis=-1
+ ),
+ N_CLASSES,
+ ),
}
@@ -185,23 +180,18 @@ def train(params, ds_train, ds_valid, ds_test):
model = FCNNModel(dataset_element_spec=ds_train.element_spec)
# Compile the model
- # It is a good practice to use a `dict` to explicitly name the outputs
- # over which the losses/metrics are computed.
- # This ensures a better optimization control, and also avoids lots of
- # useless outputs (e.g. metrics computed over extra outputs).
+ # Since Keras 3 it is mandatory to use a `dict` to explicitly name the
+ # outputs over which the losses/metrics are computed, e.g.
+ # `loss: {TARGET_NAME: "categorical_crossentropy"}`
model.compile(
- loss={
- TARGET_NAME: tf.keras.losses.CategoricalCrossentropy()
- },
- optimizer=tf.keras.optimizers.Adam(
- learning_rate=params.learning_rate
- ),
+ loss={TARGET_NAME: keras.losses.CategoricalCrossentropy()},
+ optimizer=keras.optimizers.Adam(learning_rate=params.learning_rate),
metrics={
TARGET_NAME: [
- tf.keras.metrics.Precision(class_id=1),
- tf.keras.metrics.Recall(class_id=1)
+ keras.metrics.Precision(class_id=1),
+ keras.metrics.Recall(class_id=1),
]
- }
+ },
)
# Summarize the model (in CLI)
@@ -215,4 +205,4 @@ def train(params, ds_train, ds_valid, ds_test):
model.evaluate(ds_test, batch_size=params.batch_size)
# Save trained model as SavedModel
- model.save(params.model_dir)
+ model.export(params.model_dir)
diff --git a/otbtf/layers.py b/otbtf/layers.py
index ef65ec1c5ce593ca70d6c316ae018f6a46efa596..be2ec04c1ae2ba6745487eb6f73b316536b95d0e 100644
--- a/otbtf/layers.py
+++ b/otbtf/layers.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# ==========================================================================
#
# Copyright 2018-2019 IRSTEA
@@ -25,13 +24,14 @@ The utils module provides some useful keras layers to build deep nets.
"""
from typing import List, Tuple, Any
import tensorflow as tf
+import keras
Tensor = Any
Scalars = List[float] | Tuple[float]
-class DilatedMask(tf.keras.layers.Layer):
+class DilatedMask(keras.layers.Layer):
"""Layer to dilate a binary mask."""
def __init__(self, nodata_value: float, radius: int, name: str = None):
"""
@@ -70,7 +70,7 @@ class DilatedMask(tf.keras.layers.Layer):
return tf.cast(conv2d_out, tf.uint8)
-class ApplyMask(tf.keras.layers.Layer):
+class ApplyMask(keras.layers.Layer):
"""Layer to apply a binary mask to one input."""
def __init__(self, out_nodata: float, name: str = None):
"""
@@ -95,7 +95,7 @@ class ApplyMask(tf.keras.layers.Layer):
return tf.where(mask == 1, float(self.out_nodata), inp)
-class ScalarsTile(tf.keras.layers.Layer):
+class ScalarsTile(keras.layers.Layer):
"""
Layer to duplicate some scalars in a whole array.
Simple example with only one scalar = 0.152:
@@ -127,7 +127,7 @@ class ScalarsTile(tf.keras.layers.Layer):
return tf.tile(inp, [1, tf.shape(ref)[1], tf.shape(ref)[2], 1])
-class Argmax(tf.keras.layers.Layer):
+class Argmax(keras.layers.Layer):
"""
Layer to compute the argmax of a tensor.
@@ -165,7 +165,7 @@ class Argmax(tf.keras.layers.Layer):
return argmax
-class Max(tf.keras.layers.Layer):
+class Max(keras.layers.Layer):
"""
Layer to compute the max of a tensor.
diff --git a/otbtf/model.py b/otbtf/model.py
index 9958510bdf32dd147df273a264714c93d2543ee1..e20ff81b13aa54b87c924754eff646934b80ea24 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# ==========================================================================
#
# Copyright 2018-2019 IRSTEA
@@ -27,6 +26,7 @@ from typing import List, Dict, Any
import abc
import logging
import tensorflow as tf
+import keras
Tensor = Any
TensorsDict = Dict[str, Tensor]
@@ -38,10 +38,10 @@ class ModelBase(abc.ABC):
"""
def __init__(
- self,
- dataset_element_spec: tf.TensorSpec,
- input_keys: List[str] = None,
- inference_cropping: List[int] = None
+ self,
+ dataset_element_spec: tf.TensorSpec,
+ input_keys: List[str] = None,
+ inference_cropping: List[int] = None,
):
"""
Model initializer, must be called **inside** the strategy.scope().
@@ -60,18 +60,14 @@ class ModelBase(abc.ABC):
"""
# Retrieve dataset inputs shapes
dataset_input_element_spec = dataset_element_spec[0]
- logging.info(
- "Dataset input element spec: %s", dataset_input_element_spec
- )
+ logging.info("Dataset input element spec: %s", dataset_input_element_spec)
if input_keys:
self.dataset_input_keys = input_keys
logging.info("Using input keys: %s", self.dataset_input_keys)
else:
self.dataset_input_keys = list(dataset_input_element_spec)
- logging.info(
- "Found dataset input keys: %s", self.dataset_input_keys
- )
+ logging.info("Found dataset input keys: %s", self.dataset_input_keys)
self.inputs_shapes = {
key: dataset_input_element_spec[key].shape[1:]
@@ -116,7 +112,7 @@ class ModelBase(abc.ABC):
if len(new_shape) > 2:
new_shape[0] = None
new_shape[1] = None
- placeholder = tf.keras.Input(shape=new_shape, name=key)
+ placeholder = keras.Input(shape=new_shape, name=key)
logging.info("New shape for input %s: %s", key, new_shape)
model_inputs.update({key: placeholder})
return model_inputs
@@ -158,10 +154,10 @@ class ModelBase(abc.ABC):
return inputs
def postprocess_outputs(
- self,
- outputs: TensorsDict,
- inputs: TensorsDict = None,
- normalized_inputs: TensorsDict = None
+ self,
+ outputs: TensorsDict,
+ inputs: TensorsDict = None,
+ normalized_inputs: TensorsDict = None,
) -> TensorsDict:
"""
Post-process the model outputs.
@@ -185,21 +181,21 @@ class ModelBase(abc.ABC):
for crop in self.inference_cropping:
extra_output_key = cropped_tensor_name(out_key, crop)
extra_output_name = cropped_tensor_name(
- out_tensor._keras_history.layer.name, crop
+ out_tensor._keras_history.operation.name, crop
)
logging.info(
"Adding extra output for tensor %s with crop %s (%s)",
- out_key, crop, extra_output_name
+ out_key,
+ crop,
+ extra_output_name,
)
cropped = out_tensor[:, crop:-crop, crop:-crop, :]
- identity = tf.keras.layers.Activation(
- 'linear', name=extra_output_name
- )
+ identity = keras.layers.Identity(name=extra_output_name)
extra_outputs[extra_output_key] = identity(cropped)
return extra_outputs
- def create_network(self) -> tf.keras.Model:
+ def create_network(self) -> keras.Model:
"""
This method returns the Keras model. This needs to be called
**inside** the strategy.scope(). Can be reimplemented depending on the
@@ -214,27 +210,28 @@ class ModelBase(abc.ABC):
logging.info("Model inputs: %s", inputs)
# Normalize the inputs
- normalized_inputs = self.normalize_inputs(inputs=inputs)
+ normalized_inputs = self.normalize_inputs(inputs)
logging.info("Normalized model inputs: %s", normalized_inputs)
# Build the model
- outputs = self.get_outputs(normalized_inputs=normalized_inputs)
+ outputs = self.get_outputs(normalized_inputs)
logging.info("Model outputs: %s", outputs)
# Post-processing for inference
postprocessed_outputs = self.postprocess_outputs(
- outputs=outputs,
- inputs=inputs,
- normalized_inputs=normalized_inputs
+ outputs, inputs, normalized_inputs
)
outputs.update(postprocessed_outputs)
+ # Since Keras 3, outputs are named after the key in the returned
+ # dict of `get_outputs()`
+ outputs = {
+ key: keras.layers.Identity(name=key)(prediction)
+ for key, prediction in outputs.items()
+ }
+
# Return the keras model
- return tf.keras.Model(
- inputs=inputs,
- outputs=outputs,
- name=self.__class__.__name__
- )
+ return keras.Model(inputs=inputs, outputs=outputs, name=self.__class__.__name__)
def summary(self, strategy=None):
"""
@@ -260,14 +257,13 @@ class ModelBase(abc.ABC):
show_shapes: annotate with shapes values (True or False)
"""
- assert self.model, "Plot() only works if create_network() has been " \
- "called beforehand"
+ assert self.model, (
+ "Plot() only works if create_network() has been " "called beforehand"
+ )
# When multiworker strategy, only plot if the worker is chief
if not strategy or _is_chief(strategy):
- tf.keras.utils.plot_model(
- self.model, output_path, show_shapes=show_shapes
- )
+ keras.utils.plot_model(self.model, output_path, show_shapes=show_shapes)
def _is_chief(strategy):
@@ -294,9 +290,11 @@ def _is_chief(strategy):
if strategy.cluster_resolver: # this means MultiWorkerMirroredStrategy
task_type = strategy.cluster_resolver.task_type
task_id = strategy.cluster_resolver.task_id
- return (task_type == 'chief') \
- or (task_type == 'worker' and task_id == 0) \
+ return (
+ (task_type == "chief")
+ or (task_type == "worker" and task_id == 0)
or task_type is None
+ )
# strategy with only one worker
return True
diff --git a/otbtf/ops.py b/otbtf/ops.py
index ef5c52b94060833c568dfabfcd54e98103ac85aa..5a473562ed842b268fea46e962c2ae5d8c81a31b 100644
--- a/otbtf/ops.py
+++ b/otbtf/ops.py
@@ -26,6 +26,7 @@ and train deep nets.
"""
from typing import List, Tuple, Any
import tensorflow as tf
+import keras
Tensor = Any
@@ -44,5 +45,7 @@ def one_hot(labels: Tensor, nb_classes: int):
one-hot encoded vector (shape [x, y, nb_classes])
"""
- labels_xy = tf.squeeze(tf.cast(labels, tf.int32), axis=-1) # shape [x, y]
- return tf.one_hot(labels_xy, depth=nb_classes) # shape [x, y, nb_classes]
+ # shape [x, y]
+ labels_xy = keras.ops.squeeze(keras.ops.cast(labels, tf.int32), axis=-1)
+ # shape [x, y, nb_classes]
+ return keras.ops.one_hot(labels_xy, nb_classes)
diff --git a/otbtf/tfrecords.py b/otbtf/tfrecords.py
index e5ac08417cd80d3a95e019205fbf61eb8dbeb830..8d2ce781f7a45b574ae0fa5e1143493ba73a615b 100644
--- a/otbtf/tfrecords.py
+++ b/otbtf/tfrecords.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# ==========================================================================
#
# Copyright 2018-2019 IRSTEA
diff --git a/otbtf/utils.py b/otbtf/utils.py
index 1c552fbd5c403cb048c8faa4160f62e5ae176a46..66b43ca7c62b521e9e433dd776881497ed6ac4ae 100644
--- a/otbtf/utils.py
+++ b/otbtf/utils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
# ==========================================================================
#
# Copyright 2018-2019 IRSTEA
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000000000000000000000000000000000000..19f7c077f70d0c37c45937fe1506dfd18fbde19a
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,39 @@
+[build-system]
+requires = ["setuptools>=61", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "otbtf"
+description = "OTBTF: Orfeo ToolBox meets TensorFlow"
+readme = "README.md"
+requires-python = ">=3.8"
+dynamic = ["version"]
+keywords = [
+ "remote sensing",
+ "otb",
+ "orfeotoolbox",
+ "orfeo toolbox",
+ "tensorflow",
+ "deep learning",
+ "machine learning"
+]
+authors = [
+ { name = "Remi Cresson", email = "remi.cresson@inrae.fr" },
+]
+urls = { "Homepage" = "https://github.com/remicres/otbtf" }
+classifiers = [
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Topic :: Scientific/Engineering :: GIS",
+ "Topic :: Scientific/Engineering :: Image Processing",
+ "License :: OSI Approved :: Apache Software License",
+ "Operating System :: OS Independent"
+]
+dependencies = [ "keras>=3" ]
+
+[tool.setuptools]
+packages = ["otbtf"]
+
+[tool.setuptools.dynamic]
+version = { attr = "otbtf.__version__" }
diff --git a/setup.py b/setup.py
deleted file mode 100644
index 8519fb1132c1f889eb2edb6853132ff08c747538..0000000000000000000000000000000000000000
--- a/setup.py
+++ /dev/null
@@ -1,35 +0,0 @@
-# -*- coding: utf-8 -*-
-import setuptools
-
-with open("README.md", "r", encoding="utf-8") as fh:
- long_description = fh.read()
-
-setuptools.setup(
- name="otbtf",
- version="4.3.1",
- author="Remi Cresson",
- author_email="remi.cresson@inrae.fr",
- description="OTBTF: Orfeo ToolBox meets TensorFlow",
- long_description=long_description,
- long_description_content_type="text/markdown",
- url="https://gitlab.irstea.fr/remi.cresson/otbtf",
- classifiers=[
- "Programming Language :: Python :: 3.8",
- "Programming Language :: Python :: 3.9",
- "Programming Language :: Python :: 3.10",
- "Topic :: Scientific/Engineering :: GIS",
- "Topic :: Scientific/Engineering :: Image Processing",
- "License :: OSI Approved :: Apache Software License",
- "Operating System :: OS Independent",
- ],
- packages=setuptools.find_packages(),
- python_requires=">=3.6",
- keywords=["remote sensing",
- "otb",
- "orfeotoolbox",
- "orfeo toolbox",
- "tensorflow",
- "deep learning",
- "machine learning"
- ],
-)
diff --git a/tools/docker/build-deps-cli.txt b/system-dependencies.txt
similarity index 73%
rename from tools/docker/build-deps-cli.txt
rename to system-dependencies.txt
index 49a4572b46825915b8b15812dc4ad3eb72b94b4c..53388447fbccf37f900d70ea6a073bfef9193b01 100644
--- a/tools/docker/build-deps-cli.txt
+++ b/system-dependencies.txt
@@ -1,25 +1,33 @@
apt-transport-https
ca-certificates
-curl
-cmake
+lsb-release
+software-properties-common
+gpg
file
-g++
-gcc
+sudo
+zip
+unzip
+curl
+wget
+vim
+nano
+
+pkg-config
git
-libc6-dev
+git-lfs
libtool
-lsb-release
+libc6-dev
+swig
+cppcheck
+cmake
make
-nano
patch
-pkg-config
+patchelf
+g++
+gcc
+
python3-dev
python3-pip
python3-setuptools
python3-venv
-swig
-unzip
-vim
-wget
-sudo
-zip
\ No newline at end of file
+virtualenv
diff --git a/test/api_unittest.py b/test/api_unittest.py
index 2fe3fe38632d8739dca7a61458ec95d398b0170e..b2e3ba8541ada8b1c327d3961cbc249475520515 100644
--- a/test/api_unittest.py
+++ b/test/api_unittest.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
import unittest
import pytest
@@ -7,8 +6,10 @@ import pytest
from otbtf.examples.tensorflow_v2x.fcnn import create_tfrecords
from otbtf.examples.tensorflow_v2x.fcnn import train_from_patchesimages
from otbtf.examples.tensorflow_v2x.fcnn import train_from_tfrecords
-from otbtf.examples.tensorflow_v2x.fcnn.fcnn_model import INPUT_NAME, \
- OUTPUT_SOFTMAX_NAME
+from otbtf.examples.tensorflow_v2x.fcnn.fcnn_model import (
+ INPUT_NAME,
+ TARGET_NAME
+)
from otbtf.model import cropped_tensor_name
from test_utils import resolve_paths, files_exist, run_command_and_compare
@@ -33,7 +34,7 @@ class APITest(unittest.TestCase):
])
train_from_patchesimages.train(params=params)
self.assertTrue(files_exist([
- '$TMPDIR/model_from_pimg/keras_metadata.pb',
+ '$TMPDIR/model_from_pimg/fingerprint.pb',
'$TMPDIR/model_from_pimg/saved_model.pb',
'$TMPDIR/model_from_pimg/variables/variables.data-00000-of-00001',
'$TMPDIR/model_from_pimg/variables/variables.index'
@@ -51,7 +52,7 @@ class APITest(unittest.TestCase):
f"-source1.placeholder {INPUT_NAME} "
"-model.dir $TMPDIR/model_from_pimg "
"-model.fullyconv on "
- f"-output.names {cropped_tensor_name(OUTPUT_SOFTMAX_NAME, 16)} "
+ f"-output.names {cropped_tensor_name(TARGET_NAME, 16)} "
"-output.efieldx 32 "
"-output.efieldy 32 "
"-out \"$TMPDIR/classif_model4_softmax.tif?&gdal:co:compress=deflate\" uint8",
@@ -68,7 +69,7 @@ class APITest(unittest.TestCase):
f"-source1.placeholder {INPUT_NAME} "
"-model.dir $TMPDIR/model_from_pimg "
"-model.fullyconv on "
- f"-output.names {cropped_tensor_name(OUTPUT_SOFTMAX_NAME, 32)} "
+ f"-output.names {cropped_tensor_name(TARGET_NAME, 32)} "
"-output.efieldx 64 "
"-output.efieldy 64 "
"-out \"$TMPDIR/classif_model4_softmax.tif?&gdal:co:compress=deflate\" uint8",
@@ -110,7 +111,7 @@ class APITest(unittest.TestCase):
])
train_from_tfrecords.train(params=params)
self.assertTrue(files_exist([
- '$TMPDIR/model_from_tfrecs/keras_metadata.pb',
+ '$TMPDIR/model_from_tfrecs/fingerprint.pb',
'$TMPDIR/model_from_tfrecs/saved_model.pb',
'$TMPDIR/model_from_tfrecs/variables/variables.data-00000-of-00001',
'$TMPDIR/model_from_tfrecs/variables/variables.index'
@@ -128,7 +129,7 @@ class APITest(unittest.TestCase):
f"-source1.placeholder {INPUT_NAME} "
"-model.dir $TMPDIR/model_from_pimg "
"-model.fullyconv on "
- f"-output.names {cropped_tensor_name(OUTPUT_SOFTMAX_NAME, 16)} "
+ f"-output.names {cropped_tensor_name(TARGET_NAME, 16)} "
"-output.efieldx 32 "
"-output.efieldy 32 "
"-out \"$TMPDIR/classif_model4_softmax.tif?&gdal:co:compress=deflate\" uint8",
@@ -148,7 +149,7 @@ class APITest(unittest.TestCase):
f"-source1.placeholder {INPUT_NAME} "
"-model.dir $TMPDIR/model_from_pimg "
"-model.fullyconv on "
- f"-output.names {cropped_tensor_name(OUTPUT_SOFTMAX_NAME, 32)} "
+ f"-output.names {cropped_tensor_name(TARGET_NAME, 32)} "
"-output.efieldx 64 "
"-output.efieldy 64 "
"-out \"$TMPDIR/classif_model4_softmax.tif?&gdal:co:compress=deflate\" uint8",
diff --git a/test/geos_test.py b/test/geos_test.py
index 7b8b39f9ec59822aff3c43c4ae62ed71f730e9a0..8cb36b4b1d93a709ee7ab570d609e57c2b791740 100644
--- a/test/geos_test.py
+++ b/test/geos_test.py
@@ -1,6 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-import pytest
import unittest
from osgeo import ogr
diff --git a/test/imports_test.py b/test/imports_test.py
deleted file mode 100644
index e745ed5baacd0a8a7a141891e3fb497ad537b81c..0000000000000000000000000000000000000000
--- a/test/imports_test.py
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-import pytest
-import unittest
-
-class ImportsTest(unittest.TestCase):
-
- def test_import_both1(self):
- import tensorflow
- self.assertTrue(tensorflow.__version__)
- import otbApplication
- self.assertTrue(otbApplication.Registry_GetAvailableApplications())
-
-
- def test_import_both2(self):
- import otbApplication
- self.assertTrue(otbApplication.Registry_GetAvailableApplications())
- import tensorflow
- self.assertTrue(tensorflow.__version__)
-
-
- def test_import_all(self):
- import otbApplication
- self.assertTrue(otbApplication.Registry_GetAvailableApplications())
- import tensorflow
- self.assertTrue(tensorflow.__version__)
- from osgeo import gdal
- self.assertTrue(gdal.__version__)
- import numpy
- self.assertTrue(numpy.__version__)
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/test/models/model5.py b/test/models/model5.py
index cc17d52edce5202befaee4f3fc8f7780fbc06f30..334d4f7989714f6d3f743958e99e07c5aa4bca6a 100644
--- a/test/models/model5.py
+++ b/test/models/model5.py
@@ -4,22 +4,21 @@ The input of this model must be a mono channel image.
All 4 different output shapes supported in OTBTF are tested.
"""
-import tensorflow as tf
+import keras
# Input
-x = tf.keras.Input(shape=[None, None, None], name="x") # [b, h, w, c=1]
+x = keras.Input(shape=[None, None, None], name="x") # [b, h, w, c=1]
# Create reshaped outputs
-shape = tf.shape(x)
+shape = keras.ops.shape(x)
b = shape[0]
h = shape[1]
w = shape[2]
-y1 = tf.reshape(x, shape=(b*h*w,)) # [b*h*w]
-y2 = tf.reshape(x, shape=(b*h*w, 1)) # [b*h*w, 1]
-y3 = tf.reshape(x, shape=(b, h, w)) # [b, h, w]
-y4 = tf.reshape(x, shape=(b, h, w, 1)) # [b, h, w, 1]
+y1 = keras.ops.reshape(x, shape=(b*h*w,)) # [b*h*w]
+y2 = keras.ops.reshape(x, shape=(b*h*w, 1)) # [b*h*w, 1]
+y3 = keras.ops.reshape(x, shape=(b, h, w)) # [b, h, w]
+y4 = keras.ops.reshape(x, shape=(b, h, w, 1)) # [b, h, w, 1]
# Create model
-model = tf.keras.Model(inputs={"x": x}, outputs={"y1": y1, "y2": y2, "y3": y3, "y4": y4})
-model.save("model5")
-
+model = keras.Model(inputs={"x": x}, outputs={"y1": y1, "y2": y2, "y3": y3, "y4": y4})
+model.export("model5")
diff --git a/test/nodata_test.py b/test/nodata_test.py
index c3892153401d26f000b82da59547dcbc8c890ea7..32878f435ea53977b4e05c6baa4b07bad8720e59 100644
--- a/test/nodata_test.py
+++ b/test/nodata_test.py
@@ -1,11 +1,8 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
import otbApplication
-import pytest
-import tensorflow as tf
+import keras
import unittest
-import otbtf
from test_utils import resolve_paths, compare
@@ -28,10 +25,10 @@ class NodataInferenceTest(unittest.TestCase):
sm_dir = resolve_paths("$TMPDIR/l2_norm_savedmodel")
# Create model
- x = tf.keras.Input(shape=[None, None, None], name="x")
- y = tf.norm(x, axis=-1)
- model = tf.keras.Model(inputs={"x": x}, outputs={"y": y})
- model.save(sm_dir)
+ x = keras.Input(shape=[None, None, None], name="x")
+ y = keras.ops.norm(x, axis=-1)
+ model = keras.Model(inputs={"x": x}, outputs={"y": y})
+ model.export(sm_dir)
# Input image: f(x, y) = x * y if x > y else 0
bmx = otbApplication.Registry.CreateApplication("BandMathX")
diff --git a/test/numpy_test.py b/test/numpy_test.py
index 55f0272ce4b2570c4ff0642ded73bf75baaa0d58..0d45a47f7019a7e617a138de00acbc614cfefca9 100644
--- a/test/numpy_test.py
+++ b/test/numpy_test.py
@@ -1,6 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-import pytest
import unittest
import otbApplication
from osgeo import gdal
diff --git a/test/pc_test.py b/test/pc_test.py
index 34544d02c56a4c3c4456aef5efa9f2d0815ef7e6..55a841d7cd85a088d6838736ebed1928d65e5d2f 100644
--- a/test/pc_test.py
+++ b/test/pc_test.py
@@ -1,6 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-import pytest
import unittest
import planetary_computer
import pystac_client
diff --git a/test/rio_test.py b/test/rio_test.py
index c6a0f2f1b4bd0f49ba68266f20005433987919ab..99bb0d57b5d4586035d48a83566f8ab996fbacb2 100644
--- a/test/rio_test.py
+++ b/test/rio_test.py
@@ -1,6 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-import pytest
import unittest
import rasterio
import rasterio.features
diff --git a/test/sr4rs_unittest.py b/test/sr4rs_unittest.py
index 89c3945f153304d04d35c23ef34513cf668120bc..87d887ddfc51ff7fb28d1caefc60d66ffac238ce 100644
--- a/test/sr4rs_unittest.py
+++ b/test/sr4rs_unittest.py
@@ -1,6 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
import unittest
import os
from pathlib import Path
diff --git a/test/test_utils.py b/test/test_utils.py
index 4554e28e3093e82d17faddf02ca0e14ad5536be1..2d9ba9c4d21abddb902b71bfb9dc0828183bf295 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1,7 +1,8 @@
-import otbApplication
import os
from pathlib import Path
+import otbApplication
+
def get_nb_of_channels(raster):
"""
diff --git a/test/tutorial_unittest.py b/test/tutorial_unittest.py
index af2b181c8442f8c7033b0e7ea94f39d03d262efc..886e017e94ddd4ca8c86427b3df6d2b5db7ad47f 100644
--- a/test/tutorial_unittest.py
+++ b/test/tutorial_unittest.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
import pytest
import unittest
from test_utils import run_command, run_command_and_test_exist, run_command_and_compare
diff --git a/tools/docker/build-env-tf.sh b/tools/docker/build-env-tf.sh
deleted file mode 100644
index e7703f0174a2f13dc5ec24939aeccffef4e01a7d..0000000000000000000000000000000000000000
--- a/tools/docker/build-env-tf.sh
+++ /dev/null
@@ -1,57 +0,0 @@
-### TF - bazel build env variables
-
-# As in official TF wheels, you'll need to remove "-march=native" to ensure
-# portability (avoid AVX2 / AVX512 compatibility issues)
-# You could also add CPUs instructions one by one, in this example to avoid
-# only AVX512 but enable commons optimizations like FMA, SSE4.2 and AVX2
-#export CC_OPT_FLAGS="-Wno-sign-compare --copt=-mavx --copt=-mavx2 --copt=-mfma --copt=-mfpmath=both --copt=-msse4.2"
-export CC_OPT_FLAGS="-march=native -Wno-sign-compare"
-export GCC_HOST_COMPILER_PATH=$(which gcc)
-export PYTHON_BIN_PATH=$(which python)
-export PYTHON_LIB_PATH="$($PYTHON_BIN_PATH -c 'import site; print(site.getsitepackages()[0])')"
-export TF_DOWNLOAD_CLANG=0
-export TF_ENABLE_XLA=1
-export TF_NEED_COMPUTECPP=0
-export TF_NEED_GDR=0
-export TF_NEED_JEMALLOC=1
-export TF_NEED_KAFKA=0
-export TF_NEED_MPI=0
-export TF_NEED_OPENCL=0
-export TF_NEED_OPENCL_SYCL=0
-export TF_NEED_VERBS=0
-export TF_SET_ANDROID_WORKSPACE=0
-export TF_NEED_CLANG=0
-# For MKL support BZL_CONFIGS+=" --config=mkl"
-#export TF_DOWNLOAD_MKL=1
-#export TF_NEED_MKL=0
-# Needed BZL_CONFIGS=" --config=nogcp --config=noaws --config=nohdfs"
-#export TF_NEED_S3=0
-#export TF_NEED_AWS=0
-#export TF_NEED_GCP=0
-#export TF_NEED_HDFS=0
-
-## GPU
-export TF_NEED_ROCM=0
-export TF_NEED_CUDA=0
-export CUDA_TOOLKIT_PATH=$(find /usr/local -maxdepth 1 -type d -name 'cuda-*')
-if [ ! -z $CUDA_TOOLKIT_PATH ] ; then
- if [ ! -z $TENSORRT ]; then
- echo "Building tensorflow with TensorRT support"
- apt install \
- libnvinfer8=$TENSORRT \
- libnvinfer-dev=$TENSORRT \
- libnvinfer-plugin8=$TENSORRT \
- libnvinfer-plugin-dev=$TENSORRT
- export TF_TENSORRT_VERSION=$(cat $(find /usr/ -type f -name NvInferVersion.h) | grep '#define NV_TENSORRT_MAJOR' | cut -f3 -d' ')
- export TF_NEED_TENSORRT=1
- fi
- export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$CUDA_TOOLKIT_PATH/lib64:$CUDA_TOOLKIT_PATH/lib64/stubs"
- export TF_CUDA_VERSION=$(echo $CUDA_TOOLKIT_PATH | sed -r 's/.*\/cuda-(.*)/\1/')
- export TF_CUDA_COMPUTE_CAPABILITIES="5.2,6.1,7.0,7.5,8.0,8.6,9.0"
- export TF_NEED_CUDA=1
- export TF_CUDA_CLANG=0
- export TF_NEED_TENSORRT=0
- export CUDNN_INSTALL_PATH="/usr/"
- export TF_CUDNN_VERSION=$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' $CUDNN_INSTALL_PATH/include/cudnn_version.h)
- export TF_NCCL_VERSION=2
-fi
diff --git a/tools/docker/build-flags-otb.txt b/tools/docker/build-flags-otb.txt
deleted file mode 100644
index 8c9b01233ed7754de0910fa011dc81e745ae7947..0000000000000000000000000000000000000000
--- a/tools/docker/build-flags-otb.txt
+++ /dev/null
@@ -1,8 +0,0 @@
--DOTB_BUILD_FeaturesExtraction=ON
--DOTB_BUILD_Hyperspectral=ON
--DOTB_BUILD_Learning=ON
--DOTB_BUILD_Miscellaneous=ON
--DOTB_BUILD_RemoteModules=ON
--DOTB_BUILD_SAR=ON
--DOTB_BUILD_Segmentation=ON
--DOTB_BUILD_StereoProcessing=ON
diff --git a/tools/docker/multibuild.sh b/tools/docker/multibuild.sh
deleted file mode 100644
index 9373d292469cbdd52cbf034c93d6edc8b9ba0869..0000000000000000000000000000000000000000
--- a/tools/docker/multibuild.sh
+++ /dev/null
@@ -1,75 +0,0 @@
-#!/bin/bash
-# Various docker builds using bazel cache
-RELEASE=3.5
-CPU_IMG=ubuntu:22.04
-GPU_IMG=nvidia/cuda:12.1.0-devel-ubuntu22.04
-
-## Bazel remote cache daemon
-mkdir -p $HOME/.cache/bazel-remote
-docker run -d -u 1000:1000 \
--v $HOME/.cache/bazel-remote:/data \
--p 9090:8080 \
-buchgr/bazel-remote-cache --max_size=20
-
-### CPU images
-
-# CPU-Dev
-docker build . \
---network='host' \
--t mdl4eo/otbtf:$RELEASE-cpu-dev \
---build-arg BASE_IMG=$CPU_IMG \
---build-arg KEEP_SRC_OTB=true
-
-# CPU
-docker build . \
---network='host' \
--t mdl4eo/otbtf:$RELEASE-cpu \
---build-arg BASE_IMG=$CPU_IMG
-
-# CPU-GUI
-docker build . \
---network='host' \
--t mdl4eo/otbtf:$RELEASE-cpu-gui \
---build-arg BASE_IMG=$CPU_IMG \
---build-arg GUI=true
-
-### CPU images with Intel MKL support
-MKL_CONF="--config=nogcp --config=noaws --config=nohdfs --config=mkl --config=opt"
-
-# CPU-MKL
-docker build . \
---network='host' \
--t mdl4eo/otbtf:$RELEASE-cpu-mkl \
---build-arg BASE_IMG=$CPU_IMG \
---build-arg BZL_CONFIGS="$MKL_CONF"
-
-# CPU-MKL-Dev
-docker build . \
---network='host' \
--t mdl4eo/otbtf:$RELEASE-cpu-mkl-dev \
---build-arg BASE_IMG=$CPU_IMG \
---build-arg BZL_CONFIGS="$MKL_CONF" \
---build-arg KEEP_SRC_OTB=true
-
-### GPU enabled images
-# Support is enabled if CUDA is found in /usr/local
-
-# GPU
-docker build . \
---network='host' \
--t mdl4eo/otbtf:$RELEASE-gpu-dev \
---build-arg BASE_IMG=$GPU_IMG \
---build-arg KEEP_SRC_OTB=true
-
-# GPU-Dev
-docker build . \
---network='host' \
--t mdl4eo/otbtf:$RELEASE-gpu \
---build-arg BASE_IMG=$GPU_IMG
-
-# GPU-GUI
-docker build . \
---network='host' \
--t mdl4eo/otbtf:$RELEASE-gpu-gui \
---build-arg BASE_IMG=$GPU_IMG \
---build-arg GUI=true
diff --git a/tricks/__init__.py b/tricks/__init__.py
deleted file mode 100644
index d22e7e96543aa57b3925a2738377110ab28943f4..0000000000000000000000000000000000000000
--- a/tricks/__init__.py
+++ /dev/null
@@ -1,96 +0,0 @@
-# -*- coding: utf-8 -*-
-# ==========================================================================
-#
-# Copyright 2018-2019 IRSTEA
-# Copyright 2020-2021 INRAE
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0.txt
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# ==========================================================================*/
-"""
-This module contains a set of python functions to interact with geospatial data
-and TensorFlow models.
-Starting from OTBTF >= 3.0.0, tricks is only used as a backward compatible stub
-for TF 1.X versions.
-"""
-import tensorflow.compat.v1 as tf
-from deprecated import deprecated
-from otbtf.utils import gdal_open, read_as_np_arr as read_as_np_arr_from_gdal_ds
-tf.disable_v2_behavior()
-
-
-@deprecated(version="3.0.0", reason="Please use otbtf.read_image_as_np() instead")
-def read_image_as_np(filename, as_patches=False):
- """
- Read a patches-image as numpy array.
- :param filename: File name of the patches-image
- :param as_patches: True if the image must be read as patches
- :return 4D numpy array [batch, h, w, c] (batch = 1 when as_patches is False)
- """
-
- # Open a GDAL dataset
- gdal_ds = gdal_open(filename)
-
- # Return patches
- return read_as_np_arr_from_gdal_ds(gdal_ds=gdal_ds, as_patches=as_patches)
-
-
-@deprecated(version="3.0.0", reason="Please consider using TensorFlow >= 2 to build your nets")
-def create_savedmodel(sess, inputs, outputs, directory):
- """
- Create a SavedModel from TF 1.X graphs
- :param sess: The Tensorflow V1 session
- :param inputs: List of inputs names (e.g. ["x_cnn_1:0", "x_cnn_2:0"])
- :param outputs: List of outputs names (e.g. ["prediction:0", "features:0"])
- :param directory: Path for the generated SavedModel
- """
- print("Create a SavedModel in " + directory)
- graph = tf.compat.v1.get_default_graph()
- inputs_names = {i: graph.get_tensor_by_name(i) for i in inputs}
- outputs_names = {o: graph.get_tensor_by_name(o) for o in outputs}
- tf.compat.v1.saved_model.simple_save(sess, directory, inputs=inputs_names, outputs=outputs_names)
-
-
-@deprecated(version="3.0.0", reason="Please consider using TensorFlow >= 2 to build and save your nets")
-def ckpt_to_savedmodel(ckpt_path, inputs, outputs, savedmodel_path, clear_devices=False):
- """
- Read a Checkpoint and build a SavedModel for some TF 1.X graph
- :param ckpt_path: Path to the checkpoint file (without the ".meta" extension)
- :param inputs: List of inputs names (e.g. ["x_cnn_1:0", "x_cnn_2:0"])
- :param outputs: List of outputs names (e.g. ["prediction:0", "features:0"])
- :param savedmodel_path: Path for the generated SavedModel
- :param clear_devices: Clear TensorFlow devices positioning (True/False)
- """
- tf.compat.v1.reset_default_graph()
- with tf.compat.v1.Session() as sess:
- # Restore variables from disk
- model_saver = tf.compat.v1.train.import_meta_graph(ckpt_path + ".meta", clear_devices=clear_devices)
- model_saver.restore(sess, ckpt_path)
-
- # Create a SavedModel
- create_savedmodel(sess, inputs=inputs, outputs=outputs, directory=savedmodel_path)
-
-
-@deprecated(version="3.0.0", reason="Please use otbtf.read_image_as_np() instead")
-def read_samples(filename):
- """
- Read a patches image.
- @param filename: raster file name
- """
- return read_image_as_np(filename, as_patches=True)
-
-
-# Aliases for backward compatibility
-# pylint: disable=invalid-name
-CreateSavedModel = create_savedmodel
-CheckpointToSavedModel = ckpt_to_savedmodel
diff --git a/tricks/ckpt2savedmodel.py b/tricks/ckpt2savedmodel.py
deleted file mode 100755
index ff22965f985623b318292790717ed32fa4e04536..0000000000000000000000000000000000000000
--- a/tricks/ckpt2savedmodel.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# ==========================================================================
-#
-# Copyright 2018-2019 IRSTEA
-# Copyright 2020-2021 INRAE
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0.txt
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# ==========================================================================*/
-"""
-This application converts a checkpoint into a SavedModel, that can be used in
-TensorflowModelTrain or TensorflowModelServe OTB applications.
-This is intended to work mostly with tf.v1 models, since the models in tf.v2
-can be more conveniently exported as SavedModel (see how to build a model with
-keras in Tensorflow 2).
-"""
-import argparse
-from tricks.tricks import ckpt_to_savedmodel
-
-
-def main():
- """
- Main function
- """
- parser = argparse.ArgumentParser()
- parser.add_argument("--ckpt", help="Checkpoint file (without the \".meta\" extension)", required=True)
- parser.add_argument("--inputs", help="Inputs names (e.g. [\"x_cnn_1:0\", \"x_cnn_2:0\"])", required=True, nargs='+')
- parser.add_argument("--outputs", help="Outputs names (e.g. [\"prediction:0\", \"features:0\"])", required=True,
- nargs='+')
- parser.add_argument("--model", help="Output directory for SavedModel", required=True)
- parser.add_argument('--clear_devices', dest='clear_devices', action='store_true')
- parser.set_defaults(clear_devices=False)
- params = parser.parse_args()
-
- ckpt_to_savedmodel(ckpt_path=params.ckpt,
- inputs=params.inputs,
- outputs=params.outputs,
- savedmodel_path=params.model,
- clear_devices=params.clear_devices)
-
-
-if __name__ == "__main__":
- main()