From f5150811d31358ce0934a046947266baf48af57a Mon Sep 17 00:00:00 2001 From: DPeterK Date: Mon, 24 Jan 2022 14:55:39 +0000 Subject: [PATCH 1/4] Add admin abstract interface, and new concrete implementation --- edr_data_interface/abstract/admin.py | 5 +++++ edr_data_interface/concrete/caf/__init__.py | 1 + edr_data_interface/concrete/caf/admin.py | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+) create mode 100644 edr_data_interface/abstract/admin.py create mode 100644 edr_data_interface/concrete/caf/__init__.py create mode 100644 edr_data_interface/concrete/caf/admin.py diff --git a/edr_data_interface/abstract/admin.py b/edr_data_interface/abstract/admin.py new file mode 100644 index 0000000..ffa102a --- /dev/null +++ b/edr_data_interface/abstract/admin.py @@ -0,0 +1,5 @@ +from .core import Interface + + +class RefreshCollections(Interface): + pass \ No newline at end of file diff --git a/edr_data_interface/concrete/caf/__init__.py b/edr_data_interface/concrete/caf/__init__.py new file mode 100644 index 0000000..1b96992 --- /dev/null +++ b/edr_data_interface/concrete/caf/__init__.py @@ -0,0 +1 @@ +from .admin import RefreshCollections \ No newline at end of file diff --git a/edr_data_interface/concrete/caf/admin.py b/edr_data_interface/concrete/caf/admin.py new file mode 100644 index 0000000..184b7d6 --- /dev/null +++ b/edr_data_interface/concrete/caf/admin.py @@ -0,0 +1,18 @@ +from typing import List + +from clean_air.data.storage import S3FSMetadataStore +from clean_air.models import Metadata + +from ...abstract.admin import RefreshCollections + + +class RefreshCollections(RefreshCollections): + def __init__(self): + self.metadata_store = S3FSMetadataStore + + def data(self): + collections_metadata: List[Metadata] = [] + for collection_name in self.metadata_store.available_datasets(): + collection = self.metadata_store.get(collection_name) + collections_metadata.append(collection) + return collections_metadata \ No newline at end of file From cba95e6dbf0a9de6a994f5e4d56333de28a35264 Mon Sep 17 00:00:00 2001 From: DPeterK Date: Mon, 24 Jan 2022 17:06:49 +0000 Subject: [PATCH 2/4] Add concrete admin interface to dummy --- edr_data_interface/abstract/admin.py | 9 ++++- edr_data_interface/concrete/caf/admin.py | 9 +++-- edr_data_interface/concrete/dummy/__init__.py | 1 + edr_data_interface/concrete/dummy/admin.py | 34 +++++++++++++++++++ 4 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 edr_data_interface/concrete/dummy/admin.py diff --git a/edr_data_interface/abstract/admin.py b/edr_data_interface/abstract/admin.py index ffa102a..e30a643 100644 --- a/edr_data_interface/abstract/admin.py +++ b/edr_data_interface/abstract/admin.py @@ -2,4 +2,11 @@ class RefreshCollections(Interface): - pass \ No newline at end of file + def collection(self, name): + raise NotImplemented + + def collections(self): + raise NotImplemented + + def data(self): + return self.collections() \ No newline at end of file diff --git a/edr_data_interface/concrete/caf/admin.py b/edr_data_interface/concrete/caf/admin.py index 184b7d6..ec17ea3 100644 --- a/edr_data_interface/concrete/caf/admin.py +++ b/edr_data_interface/concrete/caf/admin.py @@ -10,9 +10,12 @@ class RefreshCollections(RefreshCollections): def __init__(self): self.metadata_store = S3FSMetadataStore - def data(self): + def collection(self, name) -> Metadata: + return self.metadata_store.get(name) + + def collections(self) -> List: collections_metadata: List[Metadata] = [] for collection_name in self.metadata_store.available_datasets(): - collection = self.metadata_store.get(collection_name) - collections_metadata.append(collection) + collection_metadata = self.collection(collection_name) + collections_metadata.append(collection_metadata) return collections_metadata \ No newline at end of file diff --git a/edr_data_interface/concrete/dummy/__init__.py b/edr_data_interface/concrete/dummy/__init__.py index 3fdc92d..f5be4c9 100644 --- a/edr_data_interface/concrete/dummy/__init__.py +++ b/edr_data_interface/concrete/dummy/__init__.py @@ -1 +1,2 @@ +from .admin import RefreshCollections from .capabilities import API, Capabilities, Conformance \ No newline at end of file diff --git a/edr_data_interface/concrete/dummy/admin.py b/edr_data_interface/concrete/dummy/admin.py new file mode 100644 index 0000000..9b010e8 --- /dev/null +++ b/edr_data_interface/concrete/dummy/admin.py @@ -0,0 +1,34 @@ +from collections import namedtuple +from dataclasses import dataclass +from collections import namedtuple +from typing import Dict, List + +from ...abstract.admin import RefreshCollections + + +SAMPLES: Dict = { + "one": ["00001", "One", "The first item", "CRS84"], + "two": ["00002", "Two", "The second item", "EPSG4326"], +} + +FIELDS: List[str] = [ + "id", + "name", + "description", + "crs", +] + +collection = namedtuple("collection", FIELDS) + + +class RefreshCollections(RefreshCollections): + def collection(self, name) -> collection: + sample = SAMPLES[name] + return collection(*sample) + + def collections(self) -> List[collection]: + collections: List = [] + for name in SAMPLES.keys(): + collection = self.collection(name) + collections.append(collection) + return collections \ No newline at end of file From 79e515efeeeffedb3794b033609f5cb63307456c Mon Sep 17 00:00:00 2001 From: DPeterK Date: Fri, 28 Jan 2022 14:40:57 +0000 Subject: [PATCH 3/4] Support optional extents --- .gitignore | 3 ++ edr_data_interface/abstract/admin.py | 44 ++++++++++++++++-- edr_data_interface/concrete/dummy/admin.py | 54 ++++++++++++++++++---- 3 files changed, 89 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index b6e4761..1008373 100644 --- a/.gitignore +++ b/.gitignore @@ -114,6 +114,9 @@ venv.bak/ .spyderproject .spyproject +# PyCharm +.idea + # Rope project settings .ropeproject diff --git a/edr_data_interface/abstract/admin.py b/edr_data_interface/abstract/admin.py index e30a643..154010c 100644 --- a/edr_data_interface/abstract/admin.py +++ b/edr_data_interface/abstract/admin.py @@ -2,11 +2,49 @@ class RefreshCollections(Interface): - def collection(self, name): + def __init__(self, supported_data_queries) -> None: + self.supported_data_queries = supported_data_queries + self.temporal_extent = False + self.vertical_extent = False + + self._get_temporal_extent() + self._get_vertical_extent() + + def _get_temporal_extent(self): + """ + Determine if the collection's data has a temporal extent. + + Must set `self.temporal_extent`, and also determine `interval`, `trs` and `name` + for the temporal extent JSON response. The `trs` variable must be provided + as well-known text (WKT). + + """ + self.temporal_extent = False + + def _get_vertical_extent(self): + """ + Determine if the collection's data has a vertical extent. + + Must set `self.vertical_extent`, and also determine `interval`, `vrs` and `name` + for the vertical extent JSON response. The `vrs` variable must be provided + as well-known text (WKT). + + """ + self.vertical_extent = False + + def has_temporal_extent(self): + """Define if the collection has a temporal extent.""" + return self.temporal_extent + + def has_vertical_extent(self): + """Define if the collection has a vertical extent.""" + return self.vertical_extent + + def make_collection(self, name): raise NotImplemented - def collections(self): + def make_collections(self): raise NotImplemented def data(self): - return self.collections() \ No newline at end of file + return self.make_collections() \ No newline at end of file diff --git a/edr_data_interface/concrete/dummy/admin.py b/edr_data_interface/concrete/dummy/admin.py index 9b010e8..1229045 100644 --- a/edr_data_interface/concrete/dummy/admin.py +++ b/edr_data_interface/concrete/dummy/admin.py @@ -1,5 +1,4 @@ from collections import namedtuple -from dataclasses import dataclass from collections import namedtuple from typing import Dict, List @@ -7,8 +6,24 @@ SAMPLES: Dict = { - "one": ["00001", "One", "The first item", "CRS84"], - "two": ["00002", "Two", "The second item", "EPSG4326"], + "one": [ + "00001", + "One", + "The first item", + "CRS84", + "WGS 1984", + [-180, -90, 180, 90], + ["Example", "Dummy"], + ], + "two": [ + "00002", + "Two", + "The second item", + "EPSG4326", + "EPSG4326", + [-180, -90, 180, 90], + ["Example", "Dummy"], + ], } FIELDS: List[str] = [ @@ -16,19 +31,40 @@ "name", "description", "crs", + "crs_name", + "bbox", + "keywords", ] -collection = namedtuple("collection", FIELDS) - class RefreshCollections(RefreshCollections): - def collection(self, name) -> collection: + def __init__(self, supported_data_queries) -> None: + super().__init__(supported_data_queries) + self.collection = namedtuple("collection", FIELDS) + + def _get_temporal_extent(self): + this_extent = True + if this_extent: + FIELDS.extend(["temporal_interval", "trs", "temporal_name"]) + SAMPLES["one"].extend(["today", "TIMECRS", "Dummy temporal extent"]) + SAMPLES["two"].extend(["today/tomorrow", "TIMECRS", "Dummy temporal extent"]) + self.temporal_extent = this_extent + + def _get_vertical_extent(self): + this_extent = True + if this_extent: + FIELDS.extend(["vertical_interval", "vrs", "vertical_name"]) + SAMPLES["one"].extend([[2], "VERTCS", "Dummy vertical extent"]) + SAMPLES["two"].extend([[2, 10], "VERTCS", "Dummy vertical extent"]) + self.vertical_extent = this_extent + + def make_collection(self, name): sample = SAMPLES[name] - return collection(*sample) + return self.collection(*sample) - def collections(self) -> List[collection]: + def make_collections(self) -> List: collections: List = [] for name in SAMPLES.keys(): - collection = self.collection(name) + collection = self.make_collection(name) collections.append(collection) return collections \ No newline at end of file From 2f767b9ae7b1544ef9e176116b81c6b0e060a694 Mon Sep 17 00:00:00 2001 From: DPeterK Date: Fri, 28 Jan 2022 16:58:56 +0000 Subject: [PATCH 4/4] Add per-collection parameters handling --- edr_data_interface/abstract/admin.py | 12 +++- edr_data_interface/concrete/dummy/admin.py | 68 ++++++++++++++++++++-- 2 files changed, 72 insertions(+), 8 deletions(-) diff --git a/edr_data_interface/abstract/admin.py b/edr_data_interface/abstract/admin.py index 154010c..d59f563 100644 --- a/edr_data_interface/abstract/admin.py +++ b/edr_data_interface/abstract/admin.py @@ -40,11 +40,19 @@ def has_vertical_extent(self): """Define if the collection has a vertical extent.""" return self.vertical_extent + def get_parameters(self, collection_id): + """ + Return metadata for all the parameters (~physical quantities) + associated with the collection specified by its ID `collection_id`. + + """ + raise NotImplementedError + def make_collection(self, name): - raise NotImplemented + raise NotImplementedError def make_collections(self): - raise NotImplemented + raise NotImplementedError def data(self): return self.make_collections() \ No newline at end of file diff --git a/edr_data_interface/concrete/dummy/admin.py b/edr_data_interface/concrete/dummy/admin.py index 1229045..f03a6ff 100644 --- a/edr_data_interface/concrete/dummy/admin.py +++ b/edr_data_interface/concrete/dummy/admin.py @@ -5,8 +5,43 @@ from ...abstract.admin import RefreshCollections +PARAMS = { + "a": [ + "xxxxa", + "A dummy parameter A with certain characteristics", + "Dummy ratio", + "0.5", + "http://www.example.com/define/dummy_ratio", + "http://www.example.com/define/property/a", + "A dummy parameter A", + ], + "b": [ + "xxxxb", + "A dummy parameter B with special characteristics", + "Dummy value", + "1000", + "http://www.example.com/define/dummy_value", + "http://www.example.com/define/property/b", + "A dummy parameter B", + ], + "c": [ + "xxxxc", + "A dummy parameter C with specific characteristics", + "Dummy constant", + "1", + "http://www.example.com/define/dummy_constant", + "http://www.example.com/define/property/c", + "A dummy parameter C", + ], +} + +PARAMS_LOOKUP = { + "00001": ["a", "c"], + "00002": ["a", "b", "c"], +} + SAMPLES: Dict = { - "one": [ + "00001": [ "00001", "One", "The first item", @@ -15,7 +50,7 @@ [-180, -90, 180, 90], ["Example", "Dummy"], ], - "two": [ + "00002": [ "00002", "Two", "The second item", @@ -36,6 +71,19 @@ "keywords", ] +PARAM_FIELDS = [ + "name", + "id", + "description", + "unit_label", + "unit_value", + "unit_defn", + "property_id", + "property_label" +] + +param = namedtuple("param", PARAM_FIELDS) + class RefreshCollections(RefreshCollections): def __init__(self, supported_data_queries) -> None: @@ -46,18 +94,26 @@ def _get_temporal_extent(self): this_extent = True if this_extent: FIELDS.extend(["temporal_interval", "trs", "temporal_name"]) - SAMPLES["one"].extend(["today", "TIMECRS", "Dummy temporal extent"]) - SAMPLES["two"].extend(["today/tomorrow", "TIMECRS", "Dummy temporal extent"]) + SAMPLES["00001"].extend(["today", "TIMECRS", "Dummy temporal extent"]) + SAMPLES["00002"].extend(["today/tomorrow", "TIMECRS", "Dummy temporal extent"]) self.temporal_extent = this_extent def _get_vertical_extent(self): this_extent = True if this_extent: FIELDS.extend(["vertical_interval", "vrs", "vertical_name"]) - SAMPLES["one"].extend([[2], "VERTCS", "Dummy vertical extent"]) - SAMPLES["two"].extend([[2, 10], "VERTCS", "Dummy vertical extent"]) + SAMPLES["00001"].extend([[2], "VERTCS", "Dummy vertical extent"]) + SAMPLES["00002"].extend([[2, 10], "VERTCS", "Dummy vertical extent"]) self.vertical_extent = this_extent + def get_parameters(self, collection_id): + param_names = PARAMS_LOOKUP[collection_id] + params = {} + for name in param_names: + param_metadata = PARAMS[name] + params[name] = param(name, *param_metadata) + return params + def make_collection(self, name): sample = SAMPLES[name] return self.collection(*sample)