Skip to content
This repository was archived by the owner on May 12, 2025. It is now read-only.

Improve README #18

Merged
malmans2 merged 11 commits into from
Mar 6, 2025
Merged

Improve README #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
4039b4b
add collection properties
malmans2 Mar 5, 2025
3e5f8fb
improve README
malmans2 Mar 5, 2025
f11a7b7
add more methods to the README
malmans2 Mar 5, 2025
ec0c16b
reorder
malmans2 Mar 5, 2025
791bf21
improve README
malmans2 Mar 6, 2025
f09ff4e
cleanup
malmans2 Mar 6, 2025
f64cfa8
cleanup
malmans2 Mar 6, 2025
b94e602
cleanup
malmans2 Mar 6, 2025
6848095
cleanup
malmans2 Mar 6, 2025
5a67687
cleanup
malmans2 Mar 6, 2025
7e50ea5
fix concurrency
malmans2 Mar 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
158 changes: 138 additions & 20 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,53 +61,171 @@ Instantiate the API client and optionally verify authentication:

```

Explore a collection:
Retrieve data:

```python
>>> collection_id = "reanalysis-era5-pressure-levels"
>>> request = {
... "product_type": ["reanalysis"],
... "variable": ["temperature"],
... "year": ["2022"],
... "month": ["01"],
... "day": ["01"],
... "time": ["00:00"],
... "pressure_level": ["1000"],
... "data_format": "grib",
... "download_format": "unarchived"
... }

>>> client.retrieve(collection_id, request, target="target_1.grib") # blocks
'target_1.grib'

>>> remote = client.submit(collection_id, request) # doesn't block
>>> remote
Remote(...)
>>> remote.download("target_2.grib") # blocks
'target_2.grib'

>>> results = client.submit_and_wait_on_results(collection_id, request) # blocks
>>> results
Results(...)
>>> remote.download("target_3.grib")
'target_3.grib'

>>> client.download_results(remote.request_id, "target_4.grib") # blocks
'target_4.grib'

```

List all collection IDs sorted by last update:

```python
>>> collections = client.get_collections(sortby="update")

>>> collection_ids = []
>>> while collections is not None: # Loop over pages
... collection_ids.extend(collections.collection_ids)
... collections = collections.next # Move to the next page

>>> collection_ids
[...]
>>> collection_id in collection_ids
True

```

Explore a collection:

```python
>>> collection = client.get_collection(collection_id)

>>> collection.id == collection_id
True
>>> collection.title
'...'
>>> collection.description
'...'

>>> collection.published_at
datetime.datetime(...)
>>> collection.updated_at
datetime.datetime(...)

>>> collection.begin_datetime
datetime.datetime(...)
>>> collection.end_datetime
datetime.datetime(...)
>>> collection.bbox
(...)

>>> request = {
... "product_type": "reanalysis",
... "variable": "temperature",
... "year": "2022",
... "month": "01",
... "day": "01",
... "pressure_level": "1000",
... "time": "00:00",
... }
>>> collection.process.apply_constraints(request)
>>> collection.submit(request)
Remote(...)

>>> collection.apply_constraints(request)
{...}

```

Retrieve data:
Interact with results:

```python
>>> client.retrieve(collection_id, request, target="tmp1-era5.grib") # blocks
'tmp1-era5.grib'
>>> results = client.get_results(remote.request_id)

>>> remote = client.submit(collection_id, request) # doesn't block
>>> remote.request_id
'...'
>>> remote.status
>>> results.content_length > 0
True
>>> results.content_type
'application/x-grib'
>>> results.location
'...'
>>> remote.download("tmp2-era5.grib") # blocks
'tmp2-era5.grib'

>>> results.download("target_5.grib")
'target_5.grib'

```

List all successful jobs, sorted by newest first:

```python
>>> jobs = client.get_jobs(sortby="-created", status="successful")

>>> request_ids = []
>>> while jobs is not None: # Loop over pages
... request_ids.extend(jobs.request_ids)
... jobs = jobs.next # Move to the next page

>>> request_ids
[...]
>>> remote.request_id in request_ids
True

```

Interact with a previously submitted job:

```python
>>> remote = client.get_remote(remote.request_id)

>>> remote.collection_id == collection_id
True
>>> remote.request == request
True

>>> remote.status
'successful'
>>> remote.results_ready
True

>>> remote.created_at
datetime.datetime(...)
>>> remote.started_at
datetime.datetime(...)
>>> remote.finished_at
datetime.datetime(...)
>>> remote.updated_at == remote.finished_at
True

>>> remote.download("target_6.grib")
'target_6.grib'

>>> remote.get_results()
Results(...)

>>> remote.delete()
{...}

```

Apply constraints and find the number of available days in a given month:

```python
>>> month = {"year": "2000", "month": "02"}
>>> constrained_request = client.apply_constraints(collection_id, month)

>>> len(constrained_request["day"])
29

```

## Workflow for developers/contributors

For best experience create a new conda environment (e.g. DEVELOP) with Python 3.11:
Expand Down
14 changes: 0 additions & 14 deletions datapi/api_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,20 +199,6 @@ def download_results(self, request_id: str, target: str | None = None) -> str:
return self.get_remote(request_id).download(target)

def estimate_costs(self, collection_id: str, request: Any) -> dict[str, Any]:
"""Estimate costs of the parameters in a request.

Parameters
----------
collection_id: str
Collection ID (e.g., ``"projections-cmip6"``).
request: dict[str,Any]
Request parameters.

Returns
-------
dict[str,Any]
Dictionary of estimated costs.
"""
return self.get_process(collection_id).estimate_costs(request)

def get_accepted_licences(
Expand Down
44 changes: 29 additions & 15 deletions datapi/catalogue.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,34 @@ def end_datetime(self) -> datetime.datetime | None:
return value
return datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))

@property
def published_at(self) -> datetime.datetime:
"""When the collection was first published."""
return datetime.datetime.fromisoformat(
self._json_dict["published"].replace("Z", "+00:00")
)

@property
def updated_at(self) -> datetime.datetime:
"""When the collection was last updated."""
return datetime.datetime.fromisoformat(
self._json_dict["updated"].replace("Z", "+00:00")
)

@property
def title(self) -> str:
"""Title of the collection."""
value = self._json_dict["title"]
assert isinstance(value, str)
return value

@property
def description(self) -> str:
"""Description of the collection."""
value = self._json_dict["description"]
assert isinstance(value, str)
return value

@property
def bbox(self) -> tuple[float, float, float, float]:
"""Bounding box of the collection (W, S, E, N)."""
Expand All @@ -74,23 +102,21 @@ def _process(self) -> datapi.Process:
def process(self) -> datapi.Process:
warnings.warn(
"`process` has been deprecated, and in the future will raise an error."
"Please use `submit`, `apply_constraints`, and `estimate_costs` from now on.",
"Please use `submit` and `apply_constraints` from now on.",
DeprecationWarning,
stacklevel=2,
)
return self._process

@property
def form(self) -> list[dict[str, Any]]:
"""Form JSON."""
url = f"{self.url}/form.json"
return ApiResponse.from_request(
"get", url, log_messages=False, **self._request_kwargs
)._json_list

@property
def constraints(self) -> list[dict[str, Any]]:
"""Constraints JSON."""
url = f"{self.url}/constraints.json"
return ApiResponse.from_request(
"get", url, log_messages=False, **self._request_kwargs
Expand Down Expand Up @@ -126,18 +152,6 @@ def apply_constraints(self, request: dict[str, Any]) -> dict[str, Any]:
return self._process.apply_constraints(request)

def estimate_costs(self, request: dict[str, Any]) -> dict[str, Any]:
"""Estimate costs of the parameters in a request.

Parameters
----------
request: dict[str,Any]
Request parameters.

Returns
-------
dict[str,Any]
Dictionary of estimated costs.
"""
return self._process.estimate_costs(request)


Expand Down
60 changes: 41 additions & 19 deletions datapi/processing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,18 +344,6 @@ def apply_constraints(self, request: dict[str, Any]) -> dict[str, Any]:
return response._json_dict

def estimate_costs(self, request: dict[str, Any]) -> dict[str, Any]:
"""Estimate costs of the parameters in a request.

Parameters
----------
request: dict[str,Any]
Request parameters.

Returns
-------
dict[str,Any]
Dictionary of estimated costs.
"""
response = ApiResponse.from_request(
"post",
f"{self.url}/costing",
Expand Down Expand Up @@ -454,22 +442,57 @@ def status(self) -> str:
return str(status)

@property
def creation_datetime(self) -> datetime.datetime:
"""Creation datetime of the job."""
def updated_at(self) -> datetime.datetime:
"""When the job was last updated."""
return datetime.datetime.fromisoformat(self.json["updated"])

@property
def created_at(self) -> datetime.datetime:
"""When the job was created."""
return datetime.datetime.fromisoformat(self.json["created"])

@property
def start_datetime(self) -> datetime.datetime | None:
"""Start datetime of the job. If None, job has not started."""
def creation_datetime(self) -> datetime.datetime:
warnings.warn(
"`creation_datetime` has been deprecated, and in the future will raise an error."
"Please use `created_at` from now on.",
DeprecationWarning,
stacklevel=2,
)
return self.created_at

@property
def started_at(self) -> datetime.datetime | None:
"""When the job started. If None, the job has not started."""
value = self.json.get("started")
return value if value is None else datetime.datetime.fromisoformat(value)

@property
def end_datetime(self) -> datetime.datetime | None:
"""End datetime of the job. If None, job has not finished."""
def start_datetime(self) -> datetime.datetime | None:
warnings.warn(
"`start_datetime` has been deprecated, and in the future will raise an error."
"Please use `started_at` from now on.",
DeprecationWarning,
stacklevel=2,
)
return self.started_at

@property
def finished_at(self) -> datetime.datetime | None:
"""When the job finished. If None, the job has not finished."""
value = self.json.get("finished")
return value if value is None else datetime.datetime.fromisoformat(value)

@property
def end_datetime(self) -> datetime.datetime | None:
warnings.warn(
"`end_datetime` has been deprecated, and in the future will raise an error."
"Please use `finished_at` from now on.",
DeprecationWarning,
stacklevel=2,
)
return self.finished_at

def _wait_on_results(self) -> None:
sleep = 1.0
while not self.results_ready:
Expand Down Expand Up @@ -661,7 +684,6 @@ def _check_size(self, target: str) -> None:

@property
def asset(self) -> dict[str, Any]:
"""Asset dictionary."""
return dict(self._json_dict["asset"]["value"])

def _download(self, url: str, target: str) -> requests.Response:
Expand Down
Loading