diff --git a/CHANGELOG.md b/CHANGELOG.md index 7388cc78..ccb16fa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] -## [v5.0.0a0] - 2025-05-29 +## [v5.0.0] - 2025-06-11 ### Added - Created new `sfeos_helpers` package to improve code organization and maintainability [#376](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/376) - Added introduction section - What is stac-fastapi-elasticsearch-opensearch? - to README [#384](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/384) +- Added support for enum queryables [#390](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/390) ### Changed @@ -22,16 +23,28 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Created `aggregation` package for Elasticsearch/OpenSearch-specific aggregation functionality - Moved shared logic from core module to helper functions for better code reuse - Separated utility functions from constant mappings for clearer code organization -- Updated documentation to reflect recent code refactoring [#376](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/376) +- Updated documentation to reflect recent code refactoring [#376](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/376) - Improved README documentation with consistent formatting and enhanced sections [#381](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/381): - Added sfeos logo and banner - Added a comprehensive Quick Start guide - Reorganized sections for better navigation - Reformatted content with bullet points for improved readability - Added more detailed examples for API interaction - +- Updated mkdocs/ sfeos doucmentation page [#386](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/386) +- Improved datetime query handling to only check start and end datetime values when datetime is None [#396](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/396) +- Optimize data_loader.py script [#395](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/395) +- Refactored test configuration to use shared app config pattern [#399](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/399) +- Make `orjson` usage more consistent [#402](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/402) + ### Fixed +- Added the ability to authenticate with OpenSearch/ElasticSearch with SSL disabled [#388](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/388) + +### Removed + +- Removed `requests` dev dependency [#395](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/395) + + ## [v4.2.0] - 2025-05-15 ### Added @@ -406,8 +419,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Use genexp in execute_search and get_all_collections to return results. - Added db_to_stac serializer to item_collection method in core.py. -[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v5.0.0a0...main -[v5.0.0a0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v4.2.0...v5.0.0a0 +[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v5.0.0...main +[v5.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v4.2.0...v5.0.0 [v4.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v4.1.0...v4.2.0 [v4.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v4.0.0...v4.1.0 [v4.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v3.2.5...v4.0.0 diff --git a/Makefile b/Makefile index 5896e734..c23ca951 100644 --- a/Makefile +++ b/Makefile @@ -3,13 +3,11 @@ APP_HOST ?= 0.0.0.0 EXTERNAL_APP_PORT ?= 8080 ES_APP_PORT ?= 8080 +OS_APP_PORT ?= 8082 + ES_HOST ?= docker.for.mac.localhost ES_PORT ?= 9200 -OS_APP_PORT ?= 8082 -OS_HOST ?= docker.for.mac.localhost -OS_PORT ?= 9202 - run_es = docker compose \ run \ -p ${EXTERNAL_APP_PORT}:${ES_APP_PORT} \ diff --git a/compose.yml b/compose.yml index 240934d6..93da617f 100644 --- a/compose.yml +++ b/compose.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=5.0.0a0 + - STAC_FASTAPI_VERSION=5.0.0 - STAC_FASTAPI_LANDING_PAGE_ID=stac-fastapi-elasticsearch - APP_HOST=0.0.0.0 - APP_PORT=8080 @@ -42,7 +42,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=5.0.0a0 + - STAC_FASTAPI_VERSION=5.0.0 - STAC_FASTAPI_LANDING_PAGE_ID=stac-fastapi-opensearch - APP_HOST=0.0.0.0 - APP_PORT=8082 diff --git a/data_loader.py b/data_loader.py index 7d157e40..dea02dce 100644 --- a/data_loader.py +++ b/data_loader.py @@ -1,106 +1,105 @@ """Data Loader CLI STAC_API Ingestion Tool.""" -import json + import os +from typing import Any import click -import requests +import orjson +from httpx import Client -def load_data(data_dir, filename): +def load_data(filepath: str) -> dict[str, Any]: """Load json data from a file within the specified data directory.""" - filepath = os.path.join(data_dir, filename) - if not os.path.exists(filepath): + try: + with open(filepath, "rb") as file: + return orjson.loads(file.read()) + except FileNotFoundError as e: click.secho(f"File not found: {filepath}", fg="red", err=True) - raise click.Abort() - with open(filepath) as file: - return json.load(file) + raise click.Abort() from e -def load_collection(base_url, collection_id, data_dir): +def load_collection(client: Client, collection_id: str, data_dir: str) -> None: """Load a STAC collection into the database.""" - collection = load_data(data_dir, "collection.json") + collection = load_data(os.path.join(data_dir, "collection.json")) collection["id"] = collection_id - try: - resp = requests.post(f"{base_url}/collections", json=collection) - if resp.status_code == 200 or resp.status_code == 201: - click.echo(f"Status code: {resp.status_code}") - click.echo(f"Added collection: {collection['id']}") - elif resp.status_code == 409: - click.echo(f"Status code: {resp.status_code}") - click.echo(f"Collection: {collection['id']} already exists") - else: - click.echo(f"Status code: {resp.status_code}") - click.echo( - f"Error writing {collection['id']} collection. Message: {resp.text}" - ) - except requests.ConnectionError: - click.secho("Failed to connect", fg="red", err=True) + resp = client.post("/collections", json=collection) + if resp.status_code == 200 or resp.status_code == 201: + click.echo(f"Status code: {resp.status_code}") + click.echo(f"Added collection: {collection['id']}") + elif resp.status_code == 409: + click.echo(f"Status code: {resp.status_code}") + click.echo(f"Collection: {collection['id']} already exists") + else: + click.echo(f"Status code: {resp.status_code}") + click.echo(f"Error writing {collection['id']} collection. Message: {resp.text}") -def load_items(base_url, collection_id, use_bulk, data_dir): +def load_items( + client: Client, collection_id: str, use_bulk: bool, data_dir: str +) -> None: """Load STAC items into the database based on the method selected.""" - # Attempt to dynamically find a suitable feature collection file - feature_files = [ - file - for file in os.listdir(data_dir) - if file.endswith(".json") and file != "collection.json" - ] - if not feature_files: + with os.scandir(data_dir) as entries: + # Attempt to dynamically find a suitable feature collection file + # Use the first found feature collection file + feature_file = next( + ( + entry.path + for entry in entries + if entry.is_file() + and entry.name.endswith(".json") + and entry.name != "collection.json" + ), + None, + ) + + if feature_file is None: click.secho( "No feature collection files found in the specified directory.", fg="red", err=True, ) raise click.Abort() - feature_collection_file = feature_files[ - 0 - ] # Use the first found feature collection file - feature_collection = load_data(data_dir, feature_collection_file) - load_collection(base_url, collection_id, data_dir) + feature_collection = load_data(feature_file) + + load_collection(client, collection_id, data_dir) if use_bulk: - load_items_bulk_insert(base_url, collection_id, feature_collection, data_dir) + load_items_bulk_insert(client, collection_id, feature_collection) else: - load_items_one_by_one(base_url, collection_id, feature_collection, data_dir) + load_items_one_by_one(client, collection_id, feature_collection) -def load_items_one_by_one(base_url, collection_id, feature_collection, data_dir): +def load_items_one_by_one( + client: Client, collection_id: str, feature_collection: dict[str, Any] +) -> None: """Load STAC items into the database one by one.""" for feature in feature_collection["features"]: - try: - feature["collection"] = collection_id - resp = requests.post( - f"{base_url}/collections/{collection_id}/items", json=feature - ) - if resp.status_code == 200: - click.echo(f"Status code: {resp.status_code}") - click.echo(f"Added item: {feature['id']}") - elif resp.status_code == 409: - click.echo(f"Status code: {resp.status_code}") - click.echo(f"Item: {feature['id']} already exists") - except requests.ConnectionError: - click.secho("Failed to connect", fg="red", err=True) - - -def load_items_bulk_insert(base_url, collection_id, feature_collection, data_dir): - """Load STAC items into the database via bulk insert.""" - try: - for i, _ in enumerate(feature_collection["features"]): - feature_collection["features"][i]["collection"] = collection_id - resp = requests.post( - f"{base_url}/collections/{collection_id}/items", json=feature_collection - ) + feature["collection"] = collection_id + resp = client.post(f"/collections/{collection_id}/items", json=feature) if resp.status_code == 200: click.echo(f"Status code: {resp.status_code}") - click.echo("Bulk inserted items successfully.") - elif resp.status_code == 204: - click.echo(f"Status code: {resp.status_code}") - click.echo("Bulk update successful, no content returned.") + click.echo(f"Added item: {feature['id']}") elif resp.status_code == 409: click.echo(f"Status code: {resp.status_code}") - click.echo("Conflict detected, some items might already exist.") - except requests.ConnectionError: - click.secho("Failed to connect", fg="red", err=True) + click.echo(f"Item: {feature['id']} already exists") + + +def load_items_bulk_insert( + client: Client, collection_id: str, feature_collection: dict[str, Any] +) -> None: + """Load STAC items into the database via bulk insert.""" + for feature in feature_collection["features"]: + feature["collection"] = collection_id + resp = client.post(f"/collections/{collection_id}/items", json=feature_collection) + if resp.status_code == 200: + click.echo(f"Status code: {resp.status_code}") + click.echo("Bulk inserted items successfully.") + elif resp.status_code == 204: + click.echo(f"Status code: {resp.status_code}") + click.echo("Bulk update successful, no content returned.") + elif resp.status_code == 409: + click.echo(f"Status code: {resp.status_code}") + click.echo("Conflict detected, some items might already exist.") @click.command() @@ -117,9 +116,10 @@ def load_items_bulk_insert(base_url, collection_id, feature_collection, data_dir default="sample_data/", help="Directory containing collection.json and feature collection file", ) -def main(base_url, collection_id, use_bulk, data_dir): +def main(base_url: str, collection_id: str, use_bulk: bool, data_dir: str) -> None: """Load STAC items into the database.""" - load_items(base_url, collection_id, use_bulk, data_dir) + with Client(base_url=base_url) as client: + load_items(client, collection_id, use_bulk, data_dir) if __name__ == "__main__": diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 67764805..2333b1c1 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -21,33 +21,57 @@ nav: - Tips and Tricks: tips-and-tricks.md - API: - stac_fastapi.elasticsearch: - - app: api/stac_fastapi/elasticsearch/app.md - index: api/stac_fastapi/elasticsearch/index.md + - app: api/stac_fastapi/elasticsearch/app.md - config: api/stac_fastapi/elasticsearch/config.md - database_logic: api/stac_fastapi/elasticsearch/database_logic.md - - index: api/stac_fastapi/elasticsearch/index.md - version: api/stac_fastapi/elasticsearch/version.md - stac_fastapi.opensearch: - - app: api/stac_fastapi/opensearch/app.md - index: api/stac_fastapi/opensearch/index.md + - app: api/stac_fastapi/opensearch/app.md - config: api/stac_fastapi/opensearch/config.md - database_logic: api/stac_fastapi/opensearch/database_logic.md - - index: api/stac_fastapi/opensearch/index.md - version: api/stac_fastapi/opensearch/version.md + - sfeos_helpers: + - index: api/sfeos_helpers/index.md + - aggregation: + - module: api/sfeos_helpers/aggregation/index.md + - client: api/sfeos_helpers/aggregation/client.md + - format: api/sfeos_helpers/aggregation/format.md + - database: + - module: api/sfeos_helpers/database/index.md + - datetime: api/sfeos_helpers/database/datetime.md + - document: api/sfeos_helpers/database/document.md + - index: api/sfeos_helpers/database/index.md + - mapping: api/sfeos_helpers/database/mapping.md + - query: api/sfeos_helpers/database/query.md + - utils: api/sfeos_helpers/database/utils.md + - filter: + - module: api/sfeos_helpers/filter/index.md + - client: api/sfeos_helpers/filter/client.md + - cql2: api/sfeos_helpers/filter/cql2.md + - transform: api/sfeos_helpers/filter/transform.md + - mappings: api/sfeos_helpers/mappings.md + - version: api/sfeos_helpers/version.md - stac_fastapi.core: - index: api/stac_fastapi/core/index.md - base_database_logic: api/stac_fastapi/core/base_database_logic.md - base_settings: api/stac_fastapi/core/base_settings.md + - basic_auth: api/stac_fastapi/core/basic_auth.md - core: api/stac_fastapi/core/core.md - datetime_utils: api/stac_fastapi/core/datetime_utils.md - extensions: - module: api/stac_fastapi/core/extensions/index.md + - aggregation: api/stac_fastapi/core/extensions/aggregation.md + - fields: api/stac_fastapi/core/extensions/fields.md - filter: api/stac_fastapi/core/extensions/filter.md - query: api/stac_fastapi/core/extensions/query.md - models: - module: api/stac_fastapi/core/models/index.md - links: api/stac_fastapi/core/models/links.md - search: api/stac_fastapi/core/models/search.md + - rate_limit: api/stac_fastapi/core/rate_limit.md + - route_dependencies: api/stac_fastapi/core/route_dependencies.md - serializers: api/stac_fastapi/core/serializers.md - session: api/stac_fastapi/core/session.md - utilities: api/stac_fastapi/core/utilities.md diff --git a/docs/src/stylesheets/extra.css b/docs/src/stylesheets/extra.css index 353eb887..c9b906a0 100644 --- a/docs/src/stylesheets/extra.css +++ b/docs/src/stylesheets/extra.css @@ -1,3 +1,35 @@ :root { --md-primary-fg-color: rgb(13, 118, 160); - } \ No newline at end of file +} + +/* Control the size of the main logo */ +img[src*="sfeos.png"] { + max-width: 100%; + height: auto; + width: auto !important; + max-height: 200px; +} + +/* Control the size of sponsor logos */ +img[src*="logo"], img[src*="VITO.png"] { + max-height: 60px !important; + width: auto !important; + height: auto !important; +} + +/* Control the size of technology logos */ +img[src*="STAC-01.png"], +img[src*="python.png"], +img[src*="fastapi.svg"], +img[src*="elasticsearch.png"], +img[src*="opensearch.svg"] { + max-height: 50px !important; + width: auto !important; + height: auto !important; +} + +/* Make sure all images are responsive and don't overflow */ +img { + max-width: 100%; + height: auto; +} \ No newline at end of file diff --git a/examples/auth/README.md b/examples/auth/README.md index 0bd068e2..d1e4d85e 100644 --- a/examples/auth/README.md +++ b/examples/auth/README.md @@ -123,7 +123,7 @@ limited permissions to specific read-only endpoints. {"path": "/collections/{collection_id}", "method": ["GET"]}, {"path": "/collections/{collection_id}/items", "method": ["GET"]}, {"path": "/queryables", "method": ["GET"]}, - {"path": "/queryables/collections/{collection_id}/queryables", "method": ["GET"]}, + {"path": "/collections/{collection_id}/queryables", "method": ["GET"]}, {"path": "/_mgmt/ping", "method": ["GET"]} ], "dependencies": [ diff --git a/examples/auth/compose.basic_auth.yml b/examples/auth/compose.basic_auth.yml index eb7b2d75..866c8c44 100644 --- a/examples/auth/compose.basic_auth.yml +++ b/examples/auth/compose.basic_auth.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=5.0.0a0 + - STAC_FASTAPI_VERSION=5.0.0 - STAC_FASTAPI_LANDING_PAGE_ID=stac-fastapi-elasticsearch - APP_HOST=0.0.0.0 - APP_PORT=8080 @@ -21,7 +21,7 @@ services: - ES_USE_SSL=false - ES_VERIFY_CERTS=false - BACKEND=elasticsearch - - STAC_FASTAPI_ROUTE_DEPENDENCIES=[{"routes":[{"method":"*","path":"*"}],"dependencies":[{"method":"stac_fastapi.core.basic_auth.BasicAuth","kwargs":{"credentials":[{"username":"admin","password":"admin"}]}}]},{"routes":[{"path":"/","method":["GET"]},{"path":"/conformance","method":["GET"]},{"path":"/collections/{collection_id}/items/{item_id}","method":["GET"]},{"path":"/search","method":["GET","POST"]},{"path":"/collections","method":["GET"]},{"path":"/collections/{collection_id}","method":["GET"]},{"path":"/collections/{collection_id}/items","method":["GET"]},{"path":"/queryables","method":["GET"]},{"path":"/queryables/collections/{collection_id}/queryables","method":["GET"]},{"path":"/_mgmt/ping","method":["GET"]}],"dependencies":[{"method":"stac_fastapi.core.basic_auth.BasicAuth","kwargs":{"credentials":[{"username":"reader","password":"reader"}]}}]}] + - STAC_FASTAPI_ROUTE_DEPENDENCIES=[{"routes":[{"method":"*","path":"*"}],"dependencies":[{"method":"stac_fastapi.core.basic_auth.BasicAuth","kwargs":{"credentials":[{"username":"admin","password":"admin"}]}}]},{"routes":[{"path":"/","method":["GET"]},{"path":"/conformance","method":["GET"]},{"path":"/collections/{collection_id}/items/{item_id}","method":["GET"]},{"path":"/search","method":["GET","POST"]},{"path":"/collections","method":["GET"]},{"path":"/collections/{collection_id}","method":["GET"]},{"path":"/collections/{collection_id}/items","method":["GET"]},{"path":"/queryables","method":["GET"]},{"path":"/collections/{collection_id}/queryables","method":["GET"]},{"path":"/_mgmt/ping","method":["GET"]}],"dependencies":[{"method":"stac_fastapi.core.basic_auth.BasicAuth","kwargs":{"credentials":[{"username":"reader","password":"reader"}]}}]}] ports: - "8080:8080" volumes: @@ -43,7 +43,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=5.0.0a0 + - STAC_FASTAPI_VERSION=5.0.0 - STAC_FASTAPI_LANDING_PAGE_ID=stac-fastapi-opensearch - APP_HOST=0.0.0.0 - APP_PORT=8082 @@ -55,7 +55,7 @@ services: - ES_USE_SSL=false - ES_VERIFY_CERTS=false - BACKEND=opensearch - - STAC_FASTAPI_ROUTE_DEPENDENCIES=[{"routes":[{"method":"*","path":"*"}],"dependencies":[{"method":"stac_fastapi.core.basic_auth.BasicAuth","kwargs":{"credentials":[{"username":"admin","password":"admin"}]}}]},{"routes":[{"path":"/","method":["GET"]},{"path":"/conformance","method":["GET"]},{"path":"/collections/{collection_id}/items/{item_id}","method":["GET"]},{"path":"/search","method":["GET","POST"]},{"path":"/collections","method":["GET"]},{"path":"/collections/{collection_id}","method":["GET"]},{"path":"/collections/{collection_id}/items","method":["GET"]},{"path":"/queryables","method":["GET"]},{"path":"/queryables/collections/{collection_id}/queryables","method":["GET"]},{"path":"/_mgmt/ping","method":["GET"]}],"dependencies":[{"method":"stac_fastapi.core.basic_auth.BasicAuth","kwargs":{"credentials":[{"username":"reader","password":"reader"}]}}]}] + - STAC_FASTAPI_ROUTE_DEPENDENCIES=[{"routes":[{"method":"*","path":"*"}],"dependencies":[{"method":"stac_fastapi.core.basic_auth.BasicAuth","kwargs":{"credentials":[{"username":"admin","password":"admin"}]}}]},{"routes":[{"path":"/","method":["GET"]},{"path":"/conformance","method":["GET"]},{"path":"/collections/{collection_id}/items/{item_id}","method":["GET"]},{"path":"/search","method":["GET","POST"]},{"path":"/collections","method":["GET"]},{"path":"/collections/{collection_id}","method":["GET"]},{"path":"/collections/{collection_id}/items","method":["GET"]},{"path":"/queryables","method":["GET"]},{"path":"/collections/{collection_id}/queryables","method":["GET"]},{"path":"/_mgmt/ping","method":["GET"]}],"dependencies":[{"method":"stac_fastapi.core.basic_auth.BasicAuth","kwargs":{"credentials":[{"username":"reader","password":"reader"}]}}]}] ports: - "8082:8082" volumes: diff --git a/examples/auth/compose.oauth2.yml b/examples/auth/compose.oauth2.yml index f739e03a..32490f81 100644 --- a/examples/auth/compose.oauth2.yml +++ b/examples/auth/compose.oauth2.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=5.0.0a0 + - STAC_FASTAPI_VERSION=5.0.0 - STAC_FASTAPI_LANDING_PAGE_ID=stac-fastapi-elasticsearch - APP_HOST=0.0.0.0 - APP_PORT=8080 @@ -44,7 +44,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=5.0.0a0 + - STAC_FASTAPI_VERSION=5.0.0 - STAC_FASTAPI_LANDING_PAGE_ID=stac-fastapi-opensearch - APP_HOST=0.0.0.0 - APP_PORT=8082 diff --git a/examples/auth/compose.route_dependencies.yml b/examples/auth/compose.route_dependencies.yml index 3a11b1ad..b5821b25 100644 --- a/examples/auth/compose.route_dependencies.yml +++ b/examples/auth/compose.route_dependencies.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=5.0.0a0 + - STAC_FASTAPI_VERSION=5.0.0 - STAC_FASTAPI_LANDING_PAGE_ID=stac-fastapi-elasticsearch - APP_HOST=0.0.0.0 - APP_PORT=8080 @@ -43,7 +43,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=5.0.0a0 + - STAC_FASTAPI_VERSION=5.0.0 - STAC_FASTAPI_LANDING_PAGE_ID=stac-fastapi-opensearch - APP_HOST=0.0.0.0 - APP_PORT=8082 diff --git a/examples/rate_limit/compose.rate_limit.yml b/examples/rate_limit/compose.rate_limit.yml index 448c7760..6487bf1d 100644 --- a/examples/rate_limit/compose.rate_limit.yml +++ b/examples/rate_limit/compose.rate_limit.yml @@ -9,7 +9,7 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-elasticsearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Elasticsearch backend - - STAC_FASTAPI_VERSION=5.0.0a0 + - STAC_FASTAPI_VERSION=5.0.0 - STAC_FASTAPI_LANDING_PAGE_ID=stac-fastapi-elasticsearch - APP_HOST=0.0.0.0 - APP_PORT=8080 @@ -43,7 +43,8 @@ services: environment: - STAC_FASTAPI_TITLE=stac-fastapi-opensearch - STAC_FASTAPI_DESCRIPTION=A STAC FastAPI with an Opensearch backend - - STAC_FASTAPI_VERSION=5.0.0a0 + - STAC_FASTAPI_VERSION=5.0.0 + - STAC_FASTAPI_LANDING_PAGE_ID=stac-fastapi-opensearch - APP_HOST=0.0.0.0 - APP_PORT=8082 - RELOAD=true diff --git a/stac_fastapi/core/stac_fastapi/core/base_database_logic.py b/stac_fastapi/core/stac_fastapi/core/base_database_logic.py index 0043cfb8..57ca9437 100644 --- a/stac_fastapi/core/stac_fastapi/core/base_database_logic.py +++ b/stac_fastapi/core/stac_fastapi/core/base_database_logic.py @@ -1,7 +1,7 @@ """Base database logic.""" import abc -from typing import Any, Dict, Iterable, Optional +from typing import Any, Dict, Iterable, List, Optional class BaseDatabaseLogic(abc.ABC): @@ -36,6 +36,18 @@ async def delete_item( """Delete an item from the database.""" pass + @abc.abstractmethod + async def get_items_mapping(self, collection_id: str) -> Dict[str, Dict[str, Any]]: + """Get the mapping for the items in the collection.""" + pass + + @abc.abstractmethod + async def get_items_unique_values( + self, collection_id: str, field_names: Iterable[str], *, limit: int = ... + ) -> Dict[str, List[str]]: + """Get the unique values for the given fields in the collection.""" + pass + @abc.abstractmethod async def create_collection(self, collection: Dict, refresh: bool = False) -> None: """Create a collection in the database.""" diff --git a/stac_fastapi/core/stac_fastapi/core/extensions/filter.py b/stac_fastapi/core/stac_fastapi/core/extensions/filter.py index 08200030..c6859672 100644 --- a/stac_fastapi/core/stac_fastapi/core/extensions/filter.py +++ b/stac_fastapi/core/stac_fastapi/core/extensions/filter.py @@ -60,6 +60,17 @@ "maximum": 100, }, } +"""Queryables that are present in all collections.""" + +OPTIONAL_QUERYABLES: Dict[str, Dict[str, Any]] = { + "platform": { + "$enum": True, + "description": "Satellite platform identifier", + }, +} +"""Queryables that are present in some collections.""" + +ALL_QUERYABLES: Dict[str, Dict[str, Any]] = DEFAULT_QUERYABLES | OPTIONAL_QUERYABLES class LogicalOp(str, Enum): diff --git a/stac_fastapi/core/stac_fastapi/core/route_dependencies.py b/stac_fastapi/core/stac_fastapi/core/route_dependencies.py index 29dcc58b..fa5e4934 100644 --- a/stac_fastapi/core/stac_fastapi/core/route_dependencies.py +++ b/stac_fastapi/core/stac_fastapi/core/route_dependencies.py @@ -2,11 +2,11 @@ import importlib import inspect -import json import logging import os from typing import List +import orjson from fastapi import Depends from jsonschema import validate @@ -84,14 +84,14 @@ def get_route_dependencies_conf(route_dependencies_env: str) -> list: """Get Route dependencies configuration from file or environment variable.""" - if os.path.exists(route_dependencies_env): - with open(route_dependencies_env, encoding="utf-8") as route_dependencies_file: - route_dependencies_conf = json.load(route_dependencies_file) + if os.path.isfile(route_dependencies_env): + with open(route_dependencies_env, "rb") as f: + route_dependencies_conf = orjson.loads(f.read()) else: try: - route_dependencies_conf = json.loads(route_dependencies_env) - except json.JSONDecodeError as exception: + route_dependencies_conf = orjson.loads(route_dependencies_env) + except orjson.JSONDecodeError as exception: _LOGGER.error("Invalid JSON format for route dependencies. %s", exception) raise diff --git a/stac_fastapi/core/stac_fastapi/core/version.py b/stac_fastapi/core/stac_fastapi/core/version.py index e152cdff..4104c952 100644 --- a/stac_fastapi/core/stac_fastapi/core/version.py +++ b/stac_fastapi/core/stac_fastapi/core/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "5.0.0a0" +__version__ = "5.0.0" diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 34a139d6..d9197a44 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -6,8 +6,8 @@ desc = f.read() install_requires = [ - "stac-fastapi-core==5.0.0a0", - "sfeos-helpers==5.0.0a0", + "stac-fastapi-core==5.0.0", + "sfeos-helpers==5.0.0", "elasticsearch[async]~=8.18.0", "uvicorn~=0.23.0", "starlette>=0.35.0,<0.36.0", @@ -19,7 +19,6 @@ "pytest-cov~=4.0.0", "pytest-asyncio~=0.21.0", "pre-commit~=3.0.0", - "requests>=2.32.0,<3.0.0", "ciso8601~=2.3.0", "httpx>=0.24.0,<0.28.0", ], diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index e9a420d3..7e678b02 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -37,6 +37,7 @@ TokenPaginationExtension, TransactionExtension, ) +from stac_fastapi.extensions.core.filter import FilterConformanceClasses from stac_fastapi.extensions.third_party import BulkTransactionExtension from stac_fastapi.sfeos_helpers.aggregation import EsAsyncBaseAggregationClient from stac_fastapi.sfeos_helpers.filter import EsAsyncBaseFiltersClient @@ -56,7 +57,7 @@ client=EsAsyncBaseFiltersClient(database=database_logic) ) filter_extension.conformance_classes.append( - "http://www.opengis.net/spec/cql2/1.0/conf/advanced-comparison-operators" + FilterConformanceClasses.ADVANCED_COMPARISON_OPERATORS ) aggregation_extension = AggregationExtension( @@ -103,22 +104,24 @@ post_request_model = create_post_request_model(search_extensions) -api = StacApi( - title=os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-elasticsearch"), - description=os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-elasticsearch"), - api_version=os.getenv("STAC_FASTAPI_VERSION", "5.0.0a0"), - settings=settings, - extensions=extensions, - client=CoreClient( +app_config = { + "title": os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-elasticsearch"), + "description": os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-elasticsearch"), + "api_version": os.getenv("STAC_FASTAPI_VERSION", "5.0.0"), + "settings": settings, + "extensions": extensions, + "client": CoreClient( database=database_logic, session=session, post_request_model=post_request_model, landing_page_id=os.getenv("STAC_FASTAPI_LANDING_PAGE_ID", "stac-fastapi"), ), - search_get_request_model=create_get_request_model(search_extensions), - search_post_request_model=post_request_model, - route_dependencies=get_route_dependencies(), -) + "search_get_request_model": create_get_request_model(search_extensions), + "search_post_request_model": post_request_model, + "route_dependencies": get_route_dependencies(), +} + +api = StacApi(**app_config) @asynccontextmanager diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index d371c6a5..49495854 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -52,6 +52,10 @@ def _es_config() -> Dict[str, Any]: if http_compress: config["http_compress"] = True + # Handle authentication + if (u := os.getenv("ES_USER")) and (p := os.getenv("ES_PASS")): + config["http_auth"] = (u, p) + # Explicitly exclude SSL settings when not using SSL if not use_ssl: return config @@ -64,10 +68,6 @@ def _es_config() -> Dict[str, Any]: if config["verify_certs"]: config["ca_certs"] = os.getenv("CURL_CA_BUNDLE", certifi.where()) - # Handle authentication - if (u := os.getenv("ES_USER")) and (p := os.getenv("ES_PASS")): - config["http_auth"] = (u, p) - return config diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index d529ce01..94f2530f 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -1,7 +1,6 @@ """Database logic.""" import asyncio -import json import logging from base64 import urlsafe_b64decode, urlsafe_b64encode from copy import deepcopy @@ -9,6 +8,7 @@ import attr import elasticsearch.helpers as helpers +import orjson from elasticsearch.dsl import Q, Search from elasticsearch.exceptions import NotFoundError as ESNotFoundError from starlette.requests import Request @@ -245,121 +245,97 @@ def apply_collections_filter(search: Search, collection_ids: List[str]): @staticmethod def apply_datetime_filter( search: Search, interval: Optional[Union[DateTimeType, str]] - ): + ) -> Search: """Apply a filter to search on datetime, start_datetime, and end_datetime fields. Args: - search (Search): The search object to filter. - interval: Optional[Union[DateTimeType, str]] + search: The search object to filter. + interval: Optional datetime interval to filter by. Can be: + - A single datetime string (e.g., "2023-01-01T12:00:00") + - A datetime range string (e.g., "2023-01-01/2023-12-31") + - A datetime object + - A tuple of (start_datetime, end_datetime) Returns: - Search: The filtered search object. + The filtered search object. """ + if not interval: + return search + should = [] - datetime_search = return_date(interval) + try: + datetime_search = return_date(interval) + except (ValueError, TypeError) as e: + # Handle invalid interval formats if return_date fails + logger.error(f"Invalid interval format: {interval}, error: {e}") + return search - # If the request is a single datetime return - # items with datetimes equal to the requested datetime OR - # the requested datetime is between their start and end datetimes if "eq" in datetime_search: - should.extend( - [ - Q( - "bool", - filter=[ - Q( - "term", - properties__datetime=datetime_search["eq"], - ), - ], - ), - Q( - "bool", - filter=[ - Q( - "range", - properties__start_datetime={ - "lte": datetime_search["eq"], - }, - ), - Q( - "range", - properties__end_datetime={ - "gte": datetime_search["eq"], - }, - ), - ], - ), - ] - ) - - # If the request is a date range return - # items with datetimes within the requested date range OR - # their startdatetime ithin the requested date range OR - # their enddatetime ithin the requested date range OR - # the requested daterange within their start and end datetimes + # For exact matches, include: + # 1. Items with matching exact datetime + # 2. Items with datetime:null where the time falls within their range + should = [ + Q( + "bool", + filter=[ + Q("exists", field="properties.datetime"), + Q("term", **{"properties__datetime": datetime_search["eq"]}), + ], + ), + Q( + "bool", + must_not=[Q("exists", field="properties.datetime")], + filter=[ + Q("exists", field="properties.start_datetime"), + Q("exists", field="properties.end_datetime"), + Q( + "range", + properties__start_datetime={"lte": datetime_search["eq"]}, + ), + Q( + "range", + properties__end_datetime={"gte": datetime_search["eq"]}, + ), + ], + ), + ] else: - should.extend( - [ - Q( - "bool", - filter=[ - Q( - "range", - properties__datetime={ - "gte": datetime_search["gte"], - "lte": datetime_search["lte"], - }, - ), - ], - ), - Q( - "bool", - filter=[ - Q( - "range", - properties__start_datetime={ - "gte": datetime_search["gte"], - "lte": datetime_search["lte"], - }, - ), - ], - ), - Q( - "bool", - filter=[ - Q( - "range", - properties__end_datetime={ - "gte": datetime_search["gte"], - "lte": datetime_search["lte"], - }, - ), - ], - ), - Q( - "bool", - filter=[ - Q( - "range", - properties__start_datetime={ - "lte": datetime_search["gte"] - }, - ), - Q( - "range", - properties__end_datetime={ - "gte": datetime_search["lte"] - }, - ), - ], - ), - ] - ) - - search = search.query(Q("bool", filter=[Q("bool", should=should)])) - - return search + # For date ranges, include: + # 1. Items with datetime in the range + # 2. Items with datetime:null that overlap the search range + should = [ + Q( + "bool", + filter=[ + Q("exists", field="properties.datetime"), + Q( + "range", + properties__datetime={ + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + }, + ), + ], + ), + Q( + "bool", + must_not=[Q("exists", field="properties.datetime")], + filter=[ + Q("exists", field="properties.start_datetime"), + Q("exists", field="properties.end_datetime"), + Q( + "range", + properties__start_datetime={"lte": datetime_search["lte"]}, + ), + Q( + "range", + properties__end_datetime={"gte": datetime_search["gte"]}, + ), + ], + ), + ] + + return search.query(Q("bool", should=should, minimum_should_match=1)) @staticmethod def apply_bbox_filter(search: Search, bbox: List): @@ -527,7 +503,7 @@ async def execute_search( search_after = None if token: - search_after = json.loads(urlsafe_b64decode(token).decode()) + search_after = orjson.loads(urlsafe_b64decode(token)) query = search.query.to_dict() if search.query else None @@ -567,7 +543,7 @@ async def execute_search( next_token = None if len(hits) > limit and limit < max_result_window: if hits and (sort_array := hits[limit - 1].get("sort")): - next_token = urlsafe_b64encode(json.dumps(sort_array).encode()).decode() + next_token = urlsafe_b64encode(orjson.dumps(sort_array)).decode() matched = ( es_response["hits"]["total"]["value"] @@ -895,6 +871,37 @@ async def get_items_mapping(self, collection_id: str) -> Dict[str, Any]: except ESNotFoundError: raise NotFoundError(f"Mapping for index {index_name} not found") + async def get_items_unique_values( + self, collection_id: str, field_names: Iterable[str], *, limit: int = 100 + ) -> Dict[str, List[str]]: + """Get the unique values for the given fields in the collection.""" + limit_plus_one = limit + 1 + index_name = index_alias_by_collection_id(collection_id) + + query = await self.client.search( + index=index_name, + body={ + "size": 0, + "aggs": { + field: {"terms": {"field": field, "size": limit_plus_one}} + for field in field_names + }, + }, + ) + + result: Dict[str, List[str]] = {} + for field, agg in query["aggregations"].items(): + if len(agg["buckets"]) > limit: + logger.warning( + "Skipping enum field %s: exceeds limit of %d unique values. " + "Consider excluding this field from enumeration or increase the limit.", + field, + limit, + ) + continue + result[field] = [bucket["key"] for bucket in agg["buckets"]] + return result + async def create_collection(self, collection: Collection, **kwargs: Any): """Create a single collection in the database. diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index e152cdff..4104c952 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "5.0.0a0" +__version__ = "5.0.0" diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index 68586236..49c58802 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -6,8 +6,8 @@ desc = f.read() install_requires = [ - "stac-fastapi-core==5.0.0a0", - "sfeos-helpers==5.0.0a0", + "stac-fastapi-core==5.0.0", + "sfeos-helpers==5.0.0", "opensearch-py~=2.8.0", "opensearch-py[async]~=2.8.0", "uvicorn~=0.23.0", @@ -20,7 +20,6 @@ "pytest-cov~=4.0.0", "pytest-asyncio~=0.21.0", "pre-commit~=3.0.0", - "requests>=2.32.0,<3.0.0", "ciso8601~=2.3.0", "httpx>=0.24.0,<0.28.0", ], diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py index bd2ec073..3d0cc64c 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/app.py @@ -31,6 +31,7 @@ TokenPaginationExtension, TransactionExtension, ) +from stac_fastapi.extensions.core.filter import FilterConformanceClasses from stac_fastapi.extensions.third_party import BulkTransactionExtension from stac_fastapi.opensearch.config import OpensearchSettings from stac_fastapi.opensearch.database_logic import ( @@ -56,7 +57,7 @@ client=EsAsyncBaseFiltersClient(database=database_logic) ) filter_extension.conformance_classes.append( - "http://www.opengis.net/spec/cql2/1.0/conf/advanced-comparison-operators" + FilterConformanceClasses.ADVANCED_COMPARISON_OPERATORS ) aggregation_extension = AggregationExtension( @@ -104,22 +105,24 @@ post_request_model = create_post_request_model(search_extensions) -api = StacApi( - title=os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-opensearch"), - description=os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-opensearch"), - api_version=os.getenv("STAC_FASTAPI_VERSION", "5.0.0a0"), - settings=settings, - extensions=extensions, - client=CoreClient( +app_config = { + "title": os.getenv("STAC_FASTAPI_TITLE", "stac-fastapi-opensearch"), + "description": os.getenv("STAC_FASTAPI_DESCRIPTION", "stac-fastapi-opensearch"), + "api_version": os.getenv("STAC_FASTAPI_VERSION", "5.0.0"), + "settings": settings, + "extensions": extensions, + "client": CoreClient( database=database_logic, session=session, post_request_model=post_request_model, landing_page_id=os.getenv("STAC_FASTAPI_LANDING_PAGE_ID", "stac-fastapi"), ), - search_get_request_model=create_get_request_model(search_extensions), - search_post_request_model=post_request_model, - route_dependencies=get_route_dependencies(), -) + "search_get_request_model": create_get_request_model(search_extensions), + "search_post_request_model": post_request_model, + "route_dependencies": get_route_dependencies(), +} + +api = StacApi(**app_config) @asynccontextmanager diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py index d3811376..3fe4d71b 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/config.py @@ -40,18 +40,6 @@ def _es_config() -> Dict[str, Any]: if http_compress: config["http_compress"] = True - # Explicitly exclude SSL settings when not using SSL - if not use_ssl: - return config - - # Include SSL settings if using https - config["ssl_version"] = ssl.PROTOCOL_SSLv23 - config["verify_certs"] = get_bool_env("ES_VERIFY_CERTS", default=True) - - # Include CA Certificates if verifying certs - if config["verify_certs"]: - config["ca_certs"] = os.getenv("CURL_CA_BUNDLE", certifi.where()) - # Handle authentication if (u := os.getenv("ES_USER")) and (p := os.getenv("ES_PASS")): config["http_auth"] = (u, p) @@ -65,6 +53,18 @@ def _es_config() -> Dict[str, Any]: config["headers"] = headers + # Explicitly exclude SSL settings when not using SSL + if not use_ssl: + return config + + # Include SSL settings if using https + config["ssl_version"] = ssl.PROTOCOL_SSLv23 + config["verify_certs"] = get_bool_env("ES_VERIFY_CERTS", default=True) + + # Include CA Certificates if verifying certs + if config["verify_certs"]: + config["ca_certs"] = os.getenv("CURL_CA_BUNDLE", certifi.where()) + return config diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index f93311f9..979a0f8f 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -1,13 +1,13 @@ """Database logic.""" import asyncio -import json import logging from base64 import urlsafe_b64decode, urlsafe_b64encode from copy import deepcopy from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Union import attr +import orjson from opensearchpy import exceptions, helpers from opensearchpy.helpers.query import Q from opensearchpy.helpers.search import Search @@ -282,121 +282,97 @@ def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str] @staticmethod def apply_datetime_filter( search: Search, interval: Optional[Union[DateTimeType, str]] - ): - """Apply a filter to search based on datetime field, start_datetime, and end_datetime fields. + ) -> Search: + """Apply a filter to search on datetime, start_datetime, and end_datetime fields. Args: - search (Search): The search object to filter. - interval: Optional[Union[DateTimeType, str]] + search: The search object to filter. + interval: Optional datetime interval to filter by. Can be: + - A single datetime string (e.g., "2023-01-01T12:00:00") + - A datetime range string (e.g., "2023-01-01/2023-12-31") + - A datetime object + - A tuple of (start_datetime, end_datetime) Returns: - Search: The filtered search object. + The filtered search object. """ + if not interval: + return search + should = [] - datetime_search = return_date(interval) + try: + datetime_search = return_date(interval) + except (ValueError, TypeError) as e: + # Handle invalid interval formats if return_date fails + logger.error(f"Invalid interval format: {interval}, error: {e}") + return search - # If the request is a single datetime return - # items with datetimes equal to the requested datetime OR - # the requested datetime is between their start and end datetimes if "eq" in datetime_search: - should.extend( - [ - Q( - "bool", - filter=[ - Q( - "term", - properties__datetime=datetime_search["eq"], - ), - ], - ), - Q( - "bool", - filter=[ - Q( - "range", - properties__start_datetime={ - "lte": datetime_search["eq"], - }, - ), - Q( - "range", - properties__end_datetime={ - "gte": datetime_search["eq"], - }, - ), - ], - ), - ] - ) - - # If the request is a date range return - # items with datetimes within the requested date range OR - # their startdatetime ithin the requested date range OR - # their enddatetime ithin the requested date range OR - # the requested daterange within their start and end datetimes + # For exact matches, include: + # 1. Items with matching exact datetime + # 2. Items with datetime:null where the time falls within their range + should = [ + Q( + "bool", + filter=[ + Q("exists", field="properties.datetime"), + Q("term", **{"properties__datetime": datetime_search["eq"]}), + ], + ), + Q( + "bool", + must_not=[Q("exists", field="properties.datetime")], + filter=[ + Q("exists", field="properties.start_datetime"), + Q("exists", field="properties.end_datetime"), + Q( + "range", + properties__start_datetime={"lte": datetime_search["eq"]}, + ), + Q( + "range", + properties__end_datetime={"gte": datetime_search["eq"]}, + ), + ], + ), + ] else: - should.extend( - [ - Q( - "bool", - filter=[ - Q( - "range", - properties__datetime={ - "gte": datetime_search["gte"], - "lte": datetime_search["lte"], - }, - ), - ], - ), - Q( - "bool", - filter=[ - Q( - "range", - properties__start_datetime={ - "gte": datetime_search["gte"], - "lte": datetime_search["lte"], - }, - ), - ], - ), - Q( - "bool", - filter=[ - Q( - "range", - properties__end_datetime={ - "gte": datetime_search["gte"], - "lte": datetime_search["lte"], - }, - ), - ], - ), - Q( - "bool", - filter=[ - Q( - "range", - properties__start_datetime={ - "lte": datetime_search["gte"] - }, - ), - Q( - "range", - properties__end_datetime={ - "gte": datetime_search["lte"] - }, - ), - ], - ), - ] - ) - - search = search.query(Q("bool", filter=[Q("bool", should=should)])) - - return search + # For date ranges, include: + # 1. Items with datetime in the range + # 2. Items with datetime:null that overlap the search range + should = [ + Q( + "bool", + filter=[ + Q("exists", field="properties.datetime"), + Q( + "range", + properties__datetime={ + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + }, + ), + ], + ), + Q( + "bool", + must_not=[Q("exists", field="properties.datetime")], + filter=[ + Q("exists", field="properties.start_datetime"), + Q("exists", field="properties.end_datetime"), + Q( + "range", + properties__start_datetime={"lte": datetime_search["lte"]}, + ), + Q( + "range", + properties__end_datetime={"gte": datetime_search["gte"]}, + ), + ], + ), + ] + + return search.query(Q("bool", should=should, minimum_should_match=1)) @staticmethod def apply_bbox_filter(search: Search, bbox: List): @@ -551,7 +527,7 @@ async def execute_search( search_after = None if token: - search_after = json.loads(urlsafe_b64decode(token).decode()) + search_after = orjson.loads(urlsafe_b64decode(token)) if search_after: search_body["search_after"] = search_after @@ -591,7 +567,7 @@ async def execute_search( next_token = None if len(hits) > limit and limit < max_result_window: if hits and (sort_array := hits[limit - 1].get("sort")): - next_token = urlsafe_b64encode(json.dumps(sort_array).encode()).decode() + next_token = urlsafe_b64encode(orjson.dumps(sort_array)).decode() matched = ( es_response["hits"]["total"]["value"] @@ -904,6 +880,37 @@ async def get_items_mapping(self, collection_id: str) -> Dict[str, Any]: except exceptions.NotFoundError: raise NotFoundError(f"Mapping for index {index_name} not found") + async def get_items_unique_values( + self, collection_id: str, field_names: Iterable[str], *, limit: int = 100 + ) -> Dict[str, List[str]]: + """Get the unique values for the given fields in the collection.""" + limit_plus_one = limit + 1 + index_name = index_alias_by_collection_id(collection_id) + + query = await self.client.search( + index=index_name, + body={ + "size": 0, + "aggs": { + field: {"terms": {"field": field, "size": limit_plus_one}} + for field in field_names + }, + }, + ) + + result: Dict[str, List[str]] = {} + for field, agg in query["aggregations"].items(): + if len(agg["buckets"]) > limit: + logger.warning( + "Skipping enum field %s: exceeds limit of %d unique values. " + "Consider excluding this field from enumeration or increase the limit.", + field, + limit, + ) + continue + result[field] = [bucket["key"] for bucket in agg["buckets"]] + return result + async def create_collection(self, collection: Collection, **kwargs: Any): """Create a single collection in the database. diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py index e152cdff..4104c952 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "5.0.0a0" +__version__ = "5.0.0" diff --git a/stac_fastapi/sfeos_helpers/setup.py b/stac_fastapi/sfeos_helpers/setup.py index 7228a41c..687dd530 100644 --- a/stac_fastapi/sfeos_helpers/setup.py +++ b/stac_fastapi/sfeos_helpers/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi.core==5.0.0a0", + "stac-fastapi.core==5.0.0", ] setup( diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/client.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/client.py index 4b2a1a71..9d0eb69b 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/client.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/client.py @@ -1,12 +1,12 @@ """Filter client implementation for Elasticsearch/OpenSearch.""" from collections import deque -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Tuple import attr from stac_fastapi.core.base_database_logic import BaseDatabaseLogic -from stac_fastapi.core.extensions.filter import DEFAULT_QUERYABLES +from stac_fastapi.core.extensions.filter import ALL_QUERYABLES, DEFAULT_QUERYABLES from stac_fastapi.extensions.core.filter.client import AsyncBaseFiltersClient from stac_fastapi.sfeos_helpers.mappings import ES_MAPPING_TYPE_TO_JSON @@ -59,31 +59,31 @@ async def get_queryables( mapping_data = await self.database.get_items_mapping(collection_id) mapping_properties = next(iter(mapping_data.values()))["mappings"]["properties"] - stack = deque(mapping_properties.items()) + stack: deque[Tuple[str, Dict[str, Any]]] = deque(mapping_properties.items()) + enum_fields: Dict[str, Dict[str, Any]] = {} while stack: - field_name, field_def = stack.popleft() + field_fqn, field_def = stack.popleft() # Iterate over nested fields field_properties = field_def.get("properties") if field_properties: - # Fields in Item Properties should be exposed with their un-prefixed names, - # and not require expressions to prefix them with properties, - # e.g., eo:cloud_cover instead of properties.eo:cloud_cover. - if field_name == "properties": - stack.extend(field_properties.items()) - else: - stack.extend( - (f"{field_name}.{k}", v) for k, v in field_properties.items() - ) + stack.extend( + (f"{field_fqn}.{k}", v) for k, v in field_properties.items() + ) # Skip non-indexed or disabled fields field_type = field_def.get("type") if not field_type or not field_def.get("enabled", True): continue + # Fields in Item Properties should be exposed with their un-prefixed names, + # and not require expressions to prefix them with properties, + # e.g., eo:cloud_cover instead of properties.eo:cloud_cover. + field_name = field_fqn.removeprefix("properties.") + # Generate field properties - field_result = DEFAULT_QUERYABLES.get(field_name, {}) + field_result = ALL_QUERYABLES.get(field_name, {}) properties[field_name] = field_result field_name_human = field_name.replace("_", " ").title() @@ -95,4 +95,13 @@ async def get_queryables( if field_type in {"date", "date_nanos"}: field_result.setdefault("format", "date-time") + if field_result.pop("$enum", False): + enum_fields[field_fqn] = field_result + + if enum_fields: + for field_fqn, unique_values in ( + await self.database.get_items_unique_values(collection_id, enum_fields) + ).items(): + enum_fields[field_fqn]["enum"] = unique_values + return queryables diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py index e152cdff..4104c952 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "5.0.0a0" +__version__ = "5.0.0" diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 807da5e4..c5cb6415 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -34,6 +34,7 @@ "POST /collections/{collection_id}/items", "PUT /collections/{collection_id}", "PUT /collections/{collection_id}/items/{item_id}", + "POST /collections/{collection_id}/bulk_items", "GET /aggregations", "GET /aggregate", "POST /aggregations", diff --git a/stac_fastapi/tests/conftest.py b/stac_fastapi/tests/conftest.py index afb9ac9b..a1761288 100644 --- a/stac_fastapi/tests/conftest.py +++ b/stac_fastapi/tests/conftest.py @@ -2,7 +2,6 @@ import copy import json import os -import sys from typing import Any, Callable, Dict, Optional import pytest @@ -13,7 +12,7 @@ from stac_pydantic import api from stac_fastapi.api.app import StacApi -from stac_fastapi.api.models import create_get_request_model, create_post_request_model +from stac_fastapi.core.basic_auth import BasicAuth from stac_fastapi.core.core import ( BulkTransactionsClient, CoreClient, @@ -25,11 +24,11 @@ EsAggregationExtensionPostRequest, ) from stac_fastapi.core.rate_limit import setup_rate_limit -from stac_fastapi.core.route_dependencies import get_route_dependencies from stac_fastapi.core.utilities import get_bool_env from stac_fastapi.sfeos_helpers.aggregation import EsAsyncBaseAggregationClient if os.getenv("BACKEND", "elasticsearch").lower() == "opensearch": + from stac_fastapi.opensearch.app import app_config from stac_fastapi.opensearch.config import AsyncOpensearchSettings as AsyncSettings from stac_fastapi.opensearch.config import OpensearchSettings as SearchSettings from stac_fastapi.opensearch.database_logic import ( @@ -38,10 +37,13 @@ create_index_templates, ) else: + from stac_fastapi.elasticsearch.app import app_config from stac_fastapi.elasticsearch.config import ( - ElasticsearchSettings as SearchSettings, AsyncElasticsearchSettings as AsyncSettings, ) + from stac_fastapi.elasticsearch.config import ( + ElasticsearchSettings as SearchSettings, + ) from stac_fastapi.elasticsearch.database_logic import ( DatabaseLogic, create_collection_index, @@ -196,47 +198,7 @@ def bulk_txn_client(): @pytest_asyncio.fixture(scope="session") async def app(): - settings = AsyncSettings() - - aggregation_extension = AggregationExtension( - client=EsAsyncBaseAggregationClient( - database=database, session=None, settings=settings - ) - ) - aggregation_extension.POST = EsAggregationExtensionPostRequest - aggregation_extension.GET = EsAggregationExtensionGetRequest - - search_extensions = [ - TransactionExtension( - client=TransactionsClient( - database=database, session=None, settings=settings - ), - settings=settings, - ), - SortExtension(), - FieldsExtension(), - QueryExtension(), - TokenPaginationExtension(), - FilterExtension(), - FreeTextExtension(), - ] - - extensions = [aggregation_extension] + search_extensions - - post_request_model = create_post_request_model(search_extensions) - - return StacApi( - settings=settings, - client=CoreClient( - database=database, - session=None, - extensions=extensions, - post_request_model=post_request_model, - ), - extensions=extensions, - search_get_request_model=create_get_request_model(search_extensions), - search_post_request_model=post_request_model, - ).app + return StacApi(**app_config).app @pytest_asyncio.fixture(scope="session") @@ -252,49 +214,8 @@ async def app_client(app): @pytest_asyncio.fixture(scope="session") async def app_rate_limit(): - settings = AsyncSettings() - - aggregation_extension = AggregationExtension( - client=EsAsyncBaseAggregationClient( - database=database, session=None, settings=settings - ) - ) - aggregation_extension.POST = EsAggregationExtensionPostRequest - aggregation_extension.GET = EsAggregationExtensionGetRequest - - search_extensions = [ - TransactionExtension( - client=TransactionsClient( - database=database, session=None, settings=settings - ), - settings=settings, - ), - SortExtension(), - FieldsExtension(), - QueryExtension(), - TokenPaginationExtension(), - FilterExtension(), - FreeTextExtension(), - ] - - extensions = [aggregation_extension] + search_extensions - - post_request_model = create_post_request_model(search_extensions) - - app = StacApi( - settings=settings, - client=CoreClient( - database=database, - session=None, - extensions=extensions, - post_request_model=post_request_model, - ), - extensions=extensions, - search_get_request_model=create_get_request_model(search_extensions), - search_post_request_model=post_request_model, - ).app - - # Set up rate limit + """Fixture to get the FastAPI app with test-specific rate limiting.""" + app = StacApi(**app_config).app setup_rate_limit(app, rate_limit="2/minute") return app @@ -313,83 +234,52 @@ async def app_client_rate_limit(app_rate_limit): @pytest_asyncio.fixture(scope="session") async def app_basic_auth(): + """Fixture to get the FastAPI app with basic auth configured.""" - stac_fastapi_route_dependencies = """[ - { - "routes":[{"method":"*","path":"*"}], - "dependencies":[ - { - "method":"stac_fastapi.core.basic_auth.BasicAuth", - "kwargs":{"credentials":[{"username":"admin","password":"admin"}]} - } - ] - }, - { - "routes":[ - {"path":"/","method":["GET"]}, - {"path":"/conformance","method":["GET"]}, - {"path":"/collections/{collection_id}/items/{item_id}","method":["GET"]}, - {"path":"/search","method":["GET","POST"]}, - {"path":"/collections","method":["GET"]}, - {"path":"/collections/{collection_id}","method":["GET"]}, - {"path":"/collections/{collection_id}/items","method":["GET"]}, - {"path":"/queryables","method":["GET"]}, - {"path":"/queryables/collections/{collection_id}/queryables","method":["GET"]}, - {"path":"/_mgmt/ping","method":["GET"]} - ], - "dependencies":[ - { - "method":"stac_fastapi.core.basic_auth.BasicAuth", - "kwargs":{"credentials":[{"username":"reader","password":"reader"}]} - } - ] - } - ]""" + # Create a copy of the app config + test_config = app_config.copy() - settings = AsyncSettings() - - aggregation_extension = AggregationExtension( - client=EsAsyncBaseAggregationClient( - database=database, session=None, settings=settings - ) + # Create basic auth dependency wrapped in Depends + basic_auth = Depends( + BasicAuth(credentials=[{"username": "admin", "password": "admin"}]) ) - aggregation_extension.POST = EsAggregationExtensionPostRequest - aggregation_extension.GET = EsAggregationExtensionGetRequest - search_extensions = [ - TransactionExtension( - client=TransactionsClient( - database=database, session=None, settings=settings - ), - settings=settings, - ), - SortExtension(), - FieldsExtension(), - QueryExtension(), - TokenPaginationExtension(), - FilterExtension(), - FreeTextExtension(), + # Define public routes that don't require auth + public_paths = { + "/": ["GET"], + "/conformance": ["GET"], + "/collections/{collection_id}/items/{item_id}": ["GET"], + "/search": ["GET", "POST"], + "/collections": ["GET"], + "/collections/{collection_id}": ["GET"], + "/collections/{collection_id}/items": ["GET"], + "/queryables": ["GET"], + "/collections/{collection_id}/queryables": ["GET"], + "/_mgmt/ping": ["GET"], + } + + # Initialize route dependencies with public paths + test_config["route_dependencies"] = [ + ( + [{"path": path, "method": method} for method in methods], + [], # No auth for public routes + ) + for path, methods in public_paths.items() ] - extensions = [aggregation_extension] + search_extensions - - post_request_model = create_post_request_model(search_extensions) - - stac_api = StacApi( - settings=settings, - client=CoreClient( - database=database, - session=None, - extensions=extensions, - post_request_model=post_request_model, - ), - extensions=extensions, - search_get_request_model=create_get_request_model(search_extensions), - search_post_request_model=post_request_model, - route_dependencies=get_route_dependencies(stac_fastapi_route_dependencies), + # Add catch-all route with basic auth + test_config["route_dependencies"].extend( + [ + ( + [{"path": "*", "method": "*"}], + [basic_auth], + ) # Require auth for all other routes + ] ) - return stac_api.app + # Create the app with basic auth + api = StacApi(**test_config) + return api.app @pytest_asyncio.fixture(scope="session") @@ -418,56 +308,19 @@ def must_be_bob( @pytest_asyncio.fixture(scope="session") async def route_dependencies_app(): - # Add file to python path to allow get_route_dependencies to import must_be_bob - sys.path.append(os.path.dirname(__file__)) - - stac_fastapi_route_dependencies = """[ - { - "routes": [ - { - "method": "GET", - "path": "/collections" - } - ], - "dependencies": [ - { - "method": "conftest.must_be_bob" - } - ] - } - ]""" + """Fixture to get the FastAPI app with custom route dependencies.""" - settings = AsyncSettings() - extensions = [ - TransactionExtension( - client=TransactionsClient( - database=database, session=None, settings=settings - ), - settings=settings, - ), - SortExtension(), - FieldsExtension(), - QueryExtension(), - TokenPaginationExtension(), - FilterExtension(), - FreeTextExtension(), - ] + # Create a copy of the app config + test_config = app_config.copy() - post_request_model = create_post_request_model(extensions) + # Define route dependencies + test_config["route_dependencies"] = [ + ([{"method": "GET", "path": "/collections"}], [Depends(must_be_bob)]) + ] - return StacApi( - settings=settings, - client=CoreClient( - database=database, - session=None, - extensions=extensions, - post_request_model=post_request_model, - ), - extensions=extensions, - search_get_request_model=create_get_request_model(extensions), - search_post_request_model=post_request_model, - route_dependencies=get_route_dependencies(stac_fastapi_route_dependencies), - ).app + # Create the app with custom route dependencies + api = StacApi(**test_config) + return api.app @pytest_asyncio.fixture(scope="session") @@ -483,9 +336,16 @@ async def route_dependencies_client(route_dependencies_app): def build_test_app(): + """Build a test app with configurable transaction extensions.""" + # Create a copy of the base config + test_config = app_config.copy() + + # Get transaction extensions setting TRANSACTIONS_EXTENSIONS = get_bool_env( "ENABLE_TRANSACTIONS_EXTENSIONS", default=True ) + + # Configure extensions settings = AsyncSettings() aggregation_extension = AggregationExtension( client=EsAsyncBaseAggregationClient( @@ -494,6 +354,7 @@ def build_test_app(): ) aggregation_extension.POST = EsAggregationExtensionPostRequest aggregation_extension.GET = EsAggregationExtensionGetRequest + search_extensions = [ SortExtension(), FieldsExtension(), @@ -502,27 +363,30 @@ def build_test_app(): FilterExtension(), FreeTextExtension(), ] + + # Add transaction extension if enabled if TRANSACTIONS_EXTENSIONS: - search_extensions.insert( - 0, + search_extensions.append( TransactionExtension( client=TransactionsClient( database=database, session=None, settings=settings ), settings=settings, - ), + ) ) + + # Update extensions in config extensions = [aggregation_extension] + search_extensions - post_request_model = create_post_request_model(search_extensions) - return StacApi( - settings=settings, - client=CoreClient( - database=database, - session=None, - extensions=extensions, - post_request_model=post_request_model, - ), + test_config["extensions"] = extensions + + # Update client with new extensions + test_config["client"] = CoreClient( + database=database, + session=None, extensions=extensions, - search_get_request_model=create_get_request_model(search_extensions), - search_post_request_model=post_request_model, - ).app + post_request_model=test_config["search_post_request_model"], + ) + + # Create and return the app + api = StacApi(**test_config) + return api.app diff --git a/stac_fastapi/tests/extensions/test_filter.py b/stac_fastapi/tests/extensions/test_filter.py index fb6bc850..e54d198e 100644 --- a/stac_fastapi/tests/extensions/test_filter.py +++ b/stac_fastapi/tests/extensions/test_filter.py @@ -1,10 +1,13 @@ import json import logging import os +import uuid from os import listdir from os.path import isfile, join +from typing import Callable, Dict import pytest +from httpx import AsyncClient THIS_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -40,7 +43,6 @@ async def test_filter_extension_collection_link(app_client, load_test_data): @pytest.mark.asyncio async def test_search_filters_post(app_client, ctx): - filters = [] pwd = f"{THIS_DIR}/cql2" for fn in [fn for f in listdir(pwd) if isfile(fn := join(pwd, f))]: @@ -625,3 +627,50 @@ async def test_search_filter_extension_cql2text_s_disjoint_property(app_client, assert resp.status_code == 200 resp_json = resp.json() assert len(resp_json["features"]) == 1 + + +@pytest.mark.asyncio +async def test_queryables_enum_platform( + app_client: AsyncClient, + load_test_data: Callable[[str], Dict], + monkeypatch: pytest.MonkeyPatch, +): + # Arrange + # Enforce instant database refresh + # TODO: Is there a better way to do this? + monkeypatch.setenv("DATABASE_REFRESH", "true") + + # Create collection + collection_data = load_test_data("test_collection.json") + collection_id = collection_data["id"] = f"enum-test-collection-{uuid.uuid4()}" + r = await app_client.post("/collections", json=collection_data) + r.raise_for_status() + + # Create items with different platform values + NUM_ITEMS = 3 + for i in range(1, NUM_ITEMS + 1): + item_data = load_test_data("test_item.json") + item_data["id"] = f"enum-test-item-{i}" + item_data["collection"] = collection_id + item_data["properties"]["platform"] = "landsat-8" if i % 2 else "sentinel-2" + r = await app_client.post(f"/collections/{collection_id}/items", json=item_data) + r.raise_for_status() + + # Act + # Test queryables endpoint + queryables = ( + (await app_client.get(f"/collections/{collection_data['id']}/queryables")) + .raise_for_status() + .json() + ) + + # Assert + # Verify distinct values (should only have 2 unique values despite 3 items) + properties = queryables["properties"] + platform_info = properties["platform"] + platform_values = platform_info["enum"] + assert set(platform_values) == {"landsat-8", "sentinel-2"} + + # Clean up + r = await app_client.delete(f"/collections/{collection_id}") + r.raise_for_status() diff --git a/stac_fastapi/tests/resources/test_item.py b/stac_fastapi/tests/resources/test_item.py index 6f344b19..0102bf9b 100644 --- a/stac_fastapi/tests/resources/test_item.py +++ b/stac_fastapi/tests/resources/test_item.py @@ -1,9 +1,11 @@ import json +import logging import os import uuid from copy import deepcopy from datetime import datetime, timedelta from random import randint +from typing import Dict from urllib.parse import parse_qs, urlparse, urlsplit import ciso8601 @@ -15,7 +17,9 @@ from stac_fastapi.core.datetime_utils import datetime_to_str, now_to_rfc3339_str from stac_fastapi.types.core import LandingPageMixin -from ..conftest import create_item, refresh_indices +from ..conftest import create_collection, create_item, refresh_indices + +logger = logging.getLogger(__name__) if os.getenv("BACKEND", "elasticsearch").lower() == "opensearch": from stac_fastapi.opensearch.database_logic import DatabaseLogic @@ -398,8 +402,8 @@ async def test_item_search_temporal_intersecting_window_post(app_client, ctx): test_item = ctx.item item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"]) - item_date_before = item_date - timedelta(days=10) - item_date_after = item_date - timedelta(days=2) + item_date_before = item_date - timedelta(days=2) # Changed from 10 to 2 + item_date_after = item_date + timedelta(days=2) # Changed from -2 to +2 params = { "collections": [test_item["collection"]], @@ -940,36 +944,183 @@ async def test_search_datetime_validation_errors(app_client): assert resp.status_code == 400 -# this test should probably pass but doesn't - stac-pydantic -# https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/issues/247 - -# @pytest.mark.asyncio -# async def test_item_custom_links(app_client, ctx, txn_client): -# item = ctx.item -# item_id = "test-item-custom-links" -# item["id"] = item_id -# item["links"].append( -# { -# "href": "https://maps.example.com/wms", -# "rel": "wms", -# "type": "image/png", -# "title": "RGB composite visualized through a WMS", -# "wms:layers": ["rgb"], -# "wms:transparent": True, -# } -# ) -# await create_item(txn_client, item) - -# resp = await app_client.get("/search", params={"id": item_id}) -# assert resp.status_code == 200 -# resp_json = resp.json() -# links = resp_json["features"][0]["links"] -# for link in links: -# if link["rel"] == "wms": -# assert link["href"] == "https://maps.example.com/wms" -# assert link["type"] == "image/png" -# assert link["title"] == "RGB composite visualized through a WMS" -# assert link["wms:layers"] == ["rgb"] -# assert link["wms:transparent"] -# return True -# assert False, resp_json +@pytest.mark.asyncio +async def test_item_custom_links(app_client, ctx, txn_client): + item = ctx.item + item_id = "test-item-custom-links" + item["id"] = item_id + item["links"].append( + { + "href": "https://maps.example.com/wms", + "rel": "wms", + "type": "image/png", + "title": "RGB composite visualized through a WMS", + "wms:layers": ["rgb"], + "wms:transparent": True, + } + ) + await create_item(txn_client, item) + + resp = await app_client.get("/search", params={"id": item_id}) + assert resp.status_code == 200 + resp_json = resp.json() + links = resp_json["features"][0]["links"] + for link in links: + if link["rel"] == "wms": + assert link["href"] == "https://maps.example.com/wms" + assert link["type"] == "image/png" + assert link["title"] == "RGB composite visualized through a WMS" + assert link["wms:layers"] == ["rgb"] + assert link["wms:transparent"] + return True + assert False, resp_json + + +async def _search_and_get_ids( + app_client, + endpoint: str = "/search", + method: str = "get", + params: Dict = None, + json: Dict = None, +) -> set: + """Helper to send search request and extract feature IDs.""" + if method == "get": + resp = await app_client.get(endpoint, params=params) + else: + resp = await app_client.post(endpoint, json=json) + + assert resp.status_code == 200, f"Search failed: {resp.text}" + data = resp.json() + return {f["id"] for f in data.get("features", [])} + + +@pytest.mark.asyncio +async def test_search_datetime_with_null_datetime( + app_client, txn_client, load_test_data +): + """Test datetime filtering when properties.datetime is null or set, ensuring start_datetime and end_datetime are set when datetime is null.""" + # Setup: Create test collection + test_collection = load_test_data("test_collection.json") + try: + await create_collection(txn_client, collection=test_collection) + except Exception as e: + logger.error(f"Failed to create collection: {e}") + pytest.fail(f"Collection creation failed: {e}") + + base_item = load_test_data("test_item.json") + collection_id = base_item["collection"] + + # Item 1: Null datetime, valid start/end datetimes + null_dt_item = deepcopy(base_item) + null_dt_item["id"] = "null-datetime-item" + null_dt_item["properties"]["datetime"] = None + null_dt_item["properties"]["start_datetime"] = "2020-01-01T00:00:00Z" + null_dt_item["properties"]["end_datetime"] = "2020-01-02T00:00:00Z" + + # Item 2: Valid datetime, no start/end datetimes + valid_dt_item = deepcopy(base_item) + valid_dt_item["id"] = "valid-datetime-item" + valid_dt_item["properties"]["datetime"] = "2020-01-01T11:00:00Z" + valid_dt_item["properties"]["start_datetime"] = None + valid_dt_item["properties"]["end_datetime"] = None + + # Item 3: Valid datetime outside range, valid start/end datetimes + range_item = deepcopy(base_item) + range_item["id"] = "range-item" + range_item["properties"]["datetime"] = "2020-01-03T00:00:00Z" + range_item["properties"]["start_datetime"] = "2020-01-01T00:00:00Z" + range_item["properties"]["end_datetime"] = "2020-01-02T00:00:00Z" + + # Create valid items + items = [null_dt_item, valid_dt_item, range_item] + for item in items: + try: + await create_item(txn_client, item) + except Exception as e: + logger.error(f"Failed to create item {item['id']}: {e}") + pytest.fail(f"Item creation failed: {e}") + + # Refresh indices once + try: + await refresh_indices(txn_client) + except Exception as e: + logger.error(f"Failed to refresh indices: {e}") + pytest.fail(f"Index refresh failed: {e}") + + # Refresh indices once + try: + await refresh_indices(txn_client) + except Exception as e: + logger.error(f"Failed to refresh indices: {e}") + pytest.fail(f"Index refresh failed: {e}") + + # Test 1: Exact datetime matching valid-datetime-item and null-datetime-item + feature_ids = await _search_and_get_ids( + app_client, + params={ + "datetime": "2020-01-01T11:00:00Z", + "collections": [collection_id], + }, + ) + assert feature_ids == { + "valid-datetime-item", # Matches properties__datetime + "null-datetime-item", # Matches start_datetime <= datetime <= end_datetime + }, "Exact datetime search failed" + + # Test 2: Range including valid-datetime-item, null-datetime-item, and range-item + feature_ids = await _search_and_get_ids( + app_client, + params={ + "datetime": "2020-01-01T00:00:00Z/2020-01-03T00:00:00Z", + "collections": [collection_id], + }, + ) + assert feature_ids == { + "valid-datetime-item", # Matches properties__datetime in range + "null-datetime-item", # Matches start_datetime <= lte, end_datetime >= gte + "range-item", # Matches properties__datetime in range + }, "Range search failed" + + # Test 3: POST request for range matching null-datetime-item and valid-datetime-item + feature_ids = await _search_and_get_ids( + app_client, + method="post", + json={ + "datetime": "2020-01-01T00:00:00Z/2020-01-02T00:00:00Z", + "collections": [collection_id], + }, + ) + assert feature_ids == { + "null-datetime-item", # Matches start_datetime <= lte, end_datetime >= gte + "valid-datetime-item", # Matches properties__datetime in range + }, "POST range search failed" + + # Test 4: Exact datetime matching only range-item's datetime + feature_ids = await _search_and_get_ids( + app_client, + params={ + "datetime": "2020-01-03T00:00:00Z", + "collections": [collection_id], + }, + ) + assert feature_ids == { + "range-item", # Matches properties__datetime + }, "Exact datetime for range-item failed" + + # Test 5: Range matching null-datetime-item but not range-item's datetime + feature_ids = await _search_and_get_ids( + app_client, + params={ + "datetime": "2020-01-01T12:00:00Z/2020-01-02T12:00:00Z", + "collections": [collection_id], + }, + ) + assert feature_ids == { + "null-datetime-item", # Overlaps: search range [12:00-01-01 to 12:00-02-01] overlaps item range [00:00-01-01 to 00:00-02-01] + }, "Range search excluding range-item datetime failed" + + # Cleanup + try: + await txn_client.delete_collection(test_collection["id"]) + except Exception as e: + logger.warning(f"Failed to delete collection: {e}")