From 71b36a64e3804997ff03b986b5b134bc05397965 Mon Sep 17 00:00:00 2001
From: Florian de Boissieu <fdeboiss@gmail.com>
Date: Fri, 10 May 2024 00:15:32 +0200
Subject: [PATCH 1/7] fix end_datetime parsing expanding it to the end of the
day (in seconds)
---
simplestac/local.py | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/simplestac/local.py b/simplestac/local.py
index dfe946b..c9c8dfd 100644
--- a/simplestac/local.py
+++ b/simplestac/local.py
@@ -401,7 +401,7 @@ def properties_from_assets(assets, update_assets=True):
-def stac_item_parser(item_dir, fmt, assets=None):
+def stac_item_parser(item_dir, fmt, assets=None, expand_end_date=True):
"""Parse the item information from the scene directory.
Parameters
@@ -414,7 +414,13 @@ def stac_item_parser(item_dir, fmt, assets=None):
assets : dict, optional
The assets information, by default None.
See `stac_asset_info_from_raster`.
-
+ expand_end_date : bool, optional
+ Whether to expand the end_date to the last second of the day, by default True.
+ At the moment, the STAC specs considers end_datetime as inclusive, which means that
+ if end date is 2019-12-31, it should default to 2019-12-31T23:59:59.999999999Z.
+ We simplify it to 2019-12-31T23:59:59Z.
+ See https://github.com/radiantearth/stac-spec/issues/1255.
+
Returns
-------
dict
@@ -459,6 +465,11 @@ def stac_item_parser(item_dir, fmt, assets=None):
dt = to_datetime(dt.group(1), format=v["format"])
dt_dict[k] = dt
+ # have end_datetime inclusive
+ if expand_end_date:
+ if "end_datetime" in dt_dict:
+ dt_dict["end_datetime"] = to_datetime(dt_dict["end_datetime"].date()) + timedelta(days=1, seconds= -1)
+
# parsing id, default is the image directory name
if "id" in fmt and "pattern" in fmt["id"]:
match = re.match(fmt["id"]["pattern"], item_dir.name)
--
GitLab
From 9dd3d1d6c9353669019ed845b0315066dadcbca4 Mon Sep 17 00:00:00 2001
From: Florian de Boissieu <fdeboiss@gmail.com>
Date: Fri, 10 May 2024 00:17:48 +0200
Subject: [PATCH 2/7] update changelog
---
CHANGELOG.md | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4720be6..400d81f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+# v1.1.1
+
+## Fix
+- end_datetime expanding by default to the end of the day in seconds, e.g. 2019-12-31T23:59:59Z.
+
# v1.1.0
## Add
--
GitLab
From 16b0a37e1c83229f41043c5fbba41a128b43a350 Mon Sep 17 00:00:00 2001
From: Florian de Boissieu <fdeboiss@gmail.com>
Date: Fri, 10 May 2024 00:19:23 +0200
Subject: [PATCH 3/7] fix imports in local.py
---
simplestac/local.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/simplestac/local.py b/simplestac/local.py
index c9c8dfd..40576eb 100644
--- a/simplestac/local.py
+++ b/simplestac/local.py
@@ -5,7 +5,7 @@ from a series of scenes/images in local files.
## TODO: see if rio-cogeo would simplify the code and externalise
# the parsing of the metadata from the band files.
-
+from datetime import timedelta
import json
import geopandas as gpd
import logging
--
GitLab
From 7b1e14c7955a06355058b9f28d2cc37e51a55d79 Mon Sep 17 00:00:00 2001
From: Florian de Boissieu <fdeboiss@gmail.com>
Date: Wed, 15 May 2024 00:37:04 +0200
Subject: [PATCH 4/7] add ducntion add_reduced_coords
---
simplestac/utils.py | 62 ++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 61 insertions(+), 1 deletion(-)
diff --git a/simplestac/utils.py b/simplestac/utils.py
index 9291296..6dad8b0 100644
--- a/simplestac/utils.py
+++ b/simplestac/utils.py
@@ -1102,4 +1102,64 @@ def extract_points(x, points, method=None, tolerance=None, drop=False):
coords = points[coords_cols]
points = x.sel(coords.to_xarray(), method=method, tolerance=tolerance, drop=drop)
return points
-#######################################
\ No newline at end of file
+#######################################
+
+################# Some useful functions for xarrays ###############
+def add_reduced_coords(da, da1, dim):
+ """Add reduced coords to xarray
+
+ When reducing xarray along a dimension,
+ it drops all coordinates linked to that dimension.
+ This function adds them back if they have unique
+ values along that dimension.
+
+ Parameters
+ ----------
+ da : xarray.DataArray
+ The reference dataarray.
+ da1 : xarray.DataArray
+ The reduced dataarray.
+ dim : str
+ The reduced dimension.
+
+ Returns
+ -------
+ xarray.DataArray
+ The dataarray with added coords.
+
+ Notes
+ -----
+ See issue https://github.com/pydata/xarray/issues/8317
+
+ Examples
+ --------
+ >>> import xarray as xr
+ >>> import numpy as np
+ >>> N=6
+ >>> da = xr.DataArray(np.arange(1.*N**3).reshape(N,N,N), dims=["x", "y", "time"],
+ ... coords={
+ ... "x": np.arange(N),
+ ... "y": np.arange(N),
+ ... "time": [pd.NaT]*N,
+ ... "start": ("time", [1.]*3 + [3.]*3),
+ ... "end": ("time", ["2."]*2 + ["2."]+ ["4."]*3),
+ ... "tile":("time", ["16PFS", "16PGS", "16PKS"]*2),
+ ... "autre": "autre",})
+ >>> da = da.set_xindex(["start", "tile"]).unstack("time")
+ >>> da1 = da.mean("time")
+ >>> add_reduced_coords(da, da1, "time")
+ """
+ for k,c in da.coords.items():
+ if len(c.dims) > 0 and dim in c.dims:
+ axis = np.where(np.array(c.dims)==dim)[0]
+ # unique coordinate values along axis
+ uc = np.unique(c, axis=axis).squeeze()
+ # new coordinate dims
+ nd = list(c.dims)
+ nd.remove(dim)
+ # try to add it, if it fails we just ignore it
+ try:
+ da1 = da1.assign_coords({k:(nd, uc)})
+ except:
+ pass
+ return da1
\ No newline at end of file
--
GitLab
From 0801c7b86839fec8d895c633d776588936503edb Mon Sep 17 00:00:00 2001
From: Florian de Boissieu <fdeboiss@gmail.com>
Date: Wed, 15 May 2024 00:46:39 +0200
Subject: [PATCH 5/7] add modifier arg to write_assets
---
simplestac/utils.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/simplestac/utils.py b/simplestac/utils.py
index 6dad8b0..410094e 100644
--- a/simplestac/utils.py
+++ b/simplestac/utils.py
@@ -608,6 +608,7 @@ def write_assets(x: Union[ItemCollection, pystac.Item],
progress=True,
writer_args=None,
inplace=False,
+ modifier=None,
**kwargs):
"""
Writes item(s) assets to the specified output directory.
@@ -643,6 +644,11 @@ def write_assets(x: Union[ItemCollection, pystac.Item],
See Notes for an example.
inplace : bool, optional
Whether to modify the input collection in place or clone it. Defaults to False (i.e. clone).
+ modifier : function, optional
+ A callable that modifies the children collection and items
+ returned by this Client. This can be useful for injecting
+ authentication parameters into child assets to access data
+ from non-public sources, see pystac_client.Client for details.
**kwargs
Additional keyword arguments passed to write_raster.
@@ -687,6 +693,8 @@ def write_assets(x: Union[ItemCollection, pystac.Item],
items = []
for item in tqdm(x, disable=not progress):
ic = ItemCollection([item], clone_items=True)
+ if modifier is not None:
+ ic = modifier(ic)
arr = ic.to_xarray(bbox=bbox, geometry=geometry,xy_coords=xy_coords, ).squeeze("time")
item_dir = (output_dir / item.id).mkdir_p()
for b in arr.band.values:
--
GitLab
From 9ec9018dcf193345919d01dc97fa024613764481 Mon Sep 17 00:00:00 2001
From: Florian de Boissieu <fdeboiss@gmail.com>
Date: Wed, 15 May 2024 15:10:30 +0200
Subject: [PATCH 6/7] update changelog
---
CHANGELOG.md | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 400d81f..0a17711 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
# v1.1.1
+## Add
+- add modifier to write_assets
+- add function add_reduced_coords to fix the issue https://github.com/pydata/xarray/issues/8317
+
## Fix
- end_datetime expanding by default to the end of the day in seconds, e.g. 2019-12-31T23:59:59Z.
--
GitLab
From 82986d3b2aca0f738c4d50eb2d6b15b8e168b264 Mon Sep 17 00:00:00 2001
From: Florian de Boissieu <fdeboiss@gmail.com>
Date: Wed, 15 May 2024 15:18:28 +0200
Subject: [PATCH 7/7] update write_assets test with new modifier arg
---
tests/test_remote.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/tests/test_remote.py b/tests/test_remote.py
index 5a293ac..91153a3 100644
--- a/tests/test_remote.py
+++ b/tests/test_remote.py
@@ -67,14 +67,13 @@ def test_write_assets(roi, s2scene_pc_dir):
col = ItemCollection(search.item_collection()).drop_non_raster()
bbox = roi.to_crs(col.to_xarray().rio.crs).total_bounds
- col = pc.sign(col)
encoding=dict(
dtype="int16",
scale_factor=0.001,
add_offset=0.0,
_FillValue=-9999,
)
- new_col = write_assets(col, s2scene_pc_dir, bbox=bbox, encoding=encoding)
+ new_col = write_assets(col, s2scene_pc_dir, bbox=bbox, encoding=encoding, modifier=pc.sign_inplace)
assert len(new_col) == len(col)
assert len(new_col) == len(s2scene_pc_dir.dirs())
item = new_col[0]
@@ -84,7 +83,7 @@ def test_write_assets(roi, s2scene_pc_dir):
assert new_col[0].assets["B08"].extra_fields["raster:bands"][0]["scale"] == 0.001
with TemporaryDirectory(prefix="simplestac-tests_") as tempdir:
- new_col2 = write_assets(col, tempdir, geometry=roi.buffer(5), encoding=encoding)
+ new_col2 = write_assets(col, tempdir, geometry=roi.buffer(5), encoding=encoding, modifier=pc.sign_inplace)
assert len(new_col2) == len(new_col)
assert new_col2[0].bbox == new_col[0].bbox
--
GitLab