From 71b36a64e3804997ff03b986b5b134bc05397965 Mon Sep 17 00:00:00 2001 From: Florian de Boissieu <fdeboiss@gmail.com> Date: Fri, 10 May 2024 00:15:32 +0200 Subject: [PATCH 1/7] fix end_datetime parsing expanding it to the end of the day (in seconds) --- simplestac/local.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/simplestac/local.py b/simplestac/local.py index dfe946b..c9c8dfd 100644 --- a/simplestac/local.py +++ b/simplestac/local.py @@ -401,7 +401,7 @@ def properties_from_assets(assets, update_assets=True): -def stac_item_parser(item_dir, fmt, assets=None): +def stac_item_parser(item_dir, fmt, assets=None, expand_end_date=True): """Parse the item information from the scene directory. Parameters @@ -414,7 +414,13 @@ def stac_item_parser(item_dir, fmt, assets=None): assets : dict, optional The assets information, by default None. See `stac_asset_info_from_raster`. - + expand_end_date : bool, optional + Whether to expand the end_date to the last second of the day, by default True. + At the moment, the STAC specs considers end_datetime as inclusive, which means that + if end date is 2019-12-31, it should default to 2019-12-31T23:59:59.999999999Z. + We simplify it to 2019-12-31T23:59:59Z. + See https://github.com/radiantearth/stac-spec/issues/1255. + Returns ------- dict @@ -459,6 +465,11 @@ def stac_item_parser(item_dir, fmt, assets=None): dt = to_datetime(dt.group(1), format=v["format"]) dt_dict[k] = dt + # have end_datetime inclusive + if expand_end_date: + if "end_datetime" in dt_dict: + dt_dict["end_datetime"] = to_datetime(dt_dict["end_datetime"].date()) + timedelta(days=1, seconds= -1) + # parsing id, default is the image directory name if "id" in fmt and "pattern" in fmt["id"]: match = re.match(fmt["id"]["pattern"], item_dir.name) -- GitLab From 9dd3d1d6c9353669019ed845b0315066dadcbca4 Mon Sep 17 00:00:00 2001 From: Florian de Boissieu <fdeboiss@gmail.com> Date: Fri, 10 May 2024 00:17:48 +0200 Subject: [PATCH 2/7] update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4720be6..400d81f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# v1.1.1 + +## Fix +- end_datetime expanding by default to the end of the day in seconds, e.g. 2019-12-31T23:59:59Z. + # v1.1.0 ## Add -- GitLab From 16b0a37e1c83229f41043c5fbba41a128b43a350 Mon Sep 17 00:00:00 2001 From: Florian de Boissieu <fdeboiss@gmail.com> Date: Fri, 10 May 2024 00:19:23 +0200 Subject: [PATCH 3/7] fix imports in local.py --- simplestac/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplestac/local.py b/simplestac/local.py index c9c8dfd..40576eb 100644 --- a/simplestac/local.py +++ b/simplestac/local.py @@ -5,7 +5,7 @@ from a series of scenes/images in local files. ## TODO: see if rio-cogeo would simplify the code and externalise # the parsing of the metadata from the band files. - +from datetime import timedelta import json import geopandas as gpd import logging -- GitLab From 7b1e14c7955a06355058b9f28d2cc37e51a55d79 Mon Sep 17 00:00:00 2001 From: Florian de Boissieu <fdeboiss@gmail.com> Date: Wed, 15 May 2024 00:37:04 +0200 Subject: [PATCH 4/7] add ducntion add_reduced_coords --- simplestac/utils.py | 62 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/simplestac/utils.py b/simplestac/utils.py index 9291296..6dad8b0 100644 --- a/simplestac/utils.py +++ b/simplestac/utils.py @@ -1102,4 +1102,64 @@ def extract_points(x, points, method=None, tolerance=None, drop=False): coords = points[coords_cols] points = x.sel(coords.to_xarray(), method=method, tolerance=tolerance, drop=drop) return points -####################################### \ No newline at end of file +####################################### + +################# Some useful functions for xarrays ############### +def add_reduced_coords(da, da1, dim): + """Add reduced coords to xarray + + When reducing xarray along a dimension, + it drops all coordinates linked to that dimension. + This function adds them back if they have unique + values along that dimension. + + Parameters + ---------- + da : xarray.DataArray + The reference dataarray. + da1 : xarray.DataArray + The reduced dataarray. + dim : str + The reduced dimension. + + Returns + ------- + xarray.DataArray + The dataarray with added coords. + + Notes + ----- + See issue https://github.com/pydata/xarray/issues/8317 + + Examples + -------- + >>> import xarray as xr + >>> import numpy as np + >>> N=6 + >>> da = xr.DataArray(np.arange(1.*N**3).reshape(N,N,N), dims=["x", "y", "time"], + ... coords={ + ... "x": np.arange(N), + ... "y": np.arange(N), + ... "time": [pd.NaT]*N, + ... "start": ("time", [1.]*3 + [3.]*3), + ... "end": ("time", ["2."]*2 + ["2."]+ ["4."]*3), + ... "tile":("time", ["16PFS", "16PGS", "16PKS"]*2), + ... "autre": "autre",}) + >>> da = da.set_xindex(["start", "tile"]).unstack("time") + >>> da1 = da.mean("time") + >>> add_reduced_coords(da, da1, "time") + """ + for k,c in da.coords.items(): + if len(c.dims) > 0 and dim in c.dims: + axis = np.where(np.array(c.dims)==dim)[0] + # unique coordinate values along axis + uc = np.unique(c, axis=axis).squeeze() + # new coordinate dims + nd = list(c.dims) + nd.remove(dim) + # try to add it, if it fails we just ignore it + try: + da1 = da1.assign_coords({k:(nd, uc)}) + except: + pass + return da1 \ No newline at end of file -- GitLab From 0801c7b86839fec8d895c633d776588936503edb Mon Sep 17 00:00:00 2001 From: Florian de Boissieu <fdeboiss@gmail.com> Date: Wed, 15 May 2024 00:46:39 +0200 Subject: [PATCH 5/7] add modifier arg to write_assets --- simplestac/utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/simplestac/utils.py b/simplestac/utils.py index 6dad8b0..410094e 100644 --- a/simplestac/utils.py +++ b/simplestac/utils.py @@ -608,6 +608,7 @@ def write_assets(x: Union[ItemCollection, pystac.Item], progress=True, writer_args=None, inplace=False, + modifier=None, **kwargs): """ Writes item(s) assets to the specified output directory. @@ -643,6 +644,11 @@ def write_assets(x: Union[ItemCollection, pystac.Item], See Notes for an example. inplace : bool, optional Whether to modify the input collection in place or clone it. Defaults to False (i.e. clone). + modifier : function, optional + A callable that modifies the children collection and items + returned by this Client. This can be useful for injecting + authentication parameters into child assets to access data + from non-public sources, see pystac_client.Client for details. **kwargs Additional keyword arguments passed to write_raster. @@ -687,6 +693,8 @@ def write_assets(x: Union[ItemCollection, pystac.Item], items = [] for item in tqdm(x, disable=not progress): ic = ItemCollection([item], clone_items=True) + if modifier is not None: + ic = modifier(ic) arr = ic.to_xarray(bbox=bbox, geometry=geometry,xy_coords=xy_coords, ).squeeze("time") item_dir = (output_dir / item.id).mkdir_p() for b in arr.band.values: -- GitLab From 9ec9018dcf193345919d01dc97fa024613764481 Mon Sep 17 00:00:00 2001 From: Florian de Boissieu <fdeboiss@gmail.com> Date: Wed, 15 May 2024 15:10:30 +0200 Subject: [PATCH 6/7] update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 400d81f..0a17711 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # v1.1.1 +## Add +- add modifier to write_assets +- add function add_reduced_coords to fix the issue https://github.com/pydata/xarray/issues/8317 + ## Fix - end_datetime expanding by default to the end of the day in seconds, e.g. 2019-12-31T23:59:59Z. -- GitLab From 82986d3b2aca0f738c4d50eb2d6b15b8e168b264 Mon Sep 17 00:00:00 2001 From: Florian de Boissieu <fdeboiss@gmail.com> Date: Wed, 15 May 2024 15:18:28 +0200 Subject: [PATCH 7/7] update write_assets test with new modifier arg --- tests/test_remote.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_remote.py b/tests/test_remote.py index 5a293ac..91153a3 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -67,14 +67,13 @@ def test_write_assets(roi, s2scene_pc_dir): col = ItemCollection(search.item_collection()).drop_non_raster() bbox = roi.to_crs(col.to_xarray().rio.crs).total_bounds - col = pc.sign(col) encoding=dict( dtype="int16", scale_factor=0.001, add_offset=0.0, _FillValue=-9999, ) - new_col = write_assets(col, s2scene_pc_dir, bbox=bbox, encoding=encoding) + new_col = write_assets(col, s2scene_pc_dir, bbox=bbox, encoding=encoding, modifier=pc.sign_inplace) assert len(new_col) == len(col) assert len(new_col) == len(s2scene_pc_dir.dirs()) item = new_col[0] @@ -84,7 +83,7 @@ def test_write_assets(roi, s2scene_pc_dir): assert new_col[0].assets["B08"].extra_fields["raster:bands"][0]["scale"] == 0.001 with TemporaryDirectory(prefix="simplestac-tests_") as tempdir: - new_col2 = write_assets(col, tempdir, geometry=roi.buffer(5), encoding=encoding) + new_col2 = write_assets(col, tempdir, geometry=roi.buffer(5), encoding=encoding, modifier=pc.sign_inplace) assert len(new_col2) == len(new_col) assert new_col2[0].bbox == new_col[0].bbox -- GitLab