diff --git a/icepyx/core/APIformatting.py b/icepyx/core/APIformatting.py index dc20c6878..a87b2efd6 100644 --- a/icepyx/core/APIformatting.py +++ b/icepyx/core/APIformatting.py @@ -1,13 +1,12 @@ """Generate and format information for submitting to API (CMR and NSIDC).""" import datetime as dt -from typing import Any, Generic, Literal, Optional, TypeVar, Union, overload +from typing import Any, Generic, Literal, Optional, TypeVar, overload from icepyx.core.exceptions import ExhaustiveTypeGuardException, TypeGuardException -from icepyx.core.types import ( +from icepyx.core.harmony import HarmonyTemporal +from icepyx.core.types.api import ( CMRParams, - EGIParamsSubset, - EGIRequiredParams, ) # ---------------------------------------------------------------------- @@ -38,18 +37,17 @@ def _fmt_temporal(start, end, key): assert isinstance(start, dt.datetime) assert isinstance(end, dt.datetime) - if key == "temporal": + if key == "temporal": # search option. fmt_timerange = ( start.strftime("%Y-%m-%dT%H:%M:%SZ") + "," + end.strftime("%Y-%m-%dT%H:%M:%SZ") ) - elif key == "time": - fmt_timerange = ( - start.strftime("%Y-%m-%dT%H:%M:%S") - + "," - + end.strftime("%Y-%m-%dT%H:%M:%S") - ) + elif key == "time": # subsetting option. + # Format for harmony. This will do subsetting. + # TODO: change `key` to something more clear. `temporal` is the key + # passed into Harmony, so this is very confusing! + fmt_timerange: HarmonyTemporal = {"start": start, "stop": end} else: raise ValueError("An invalid time key was submitted for formatting.") @@ -212,20 +210,22 @@ def __get__( self, instance: 'Parameters[Literal["required"]]', owner: Any, - ) -> EGIRequiredParams: ... + ): # -> EGIRequiredParams: ... + ... @overload def __get__( self, instance: 'Parameters[Literal["subset"]]', owner: Any, - ) -> EGIParamsSubset: ... + ): # -> EGIParamsSubset: ... + ... def __get__( self, instance: "Parameters", owner: Any, - ) -> Union[CMRParams, EGIRequiredParams, EGIParamsSubset]: + ) -> CMRParams: """ Returns the dictionary of formatted keys associated with the parameter object. @@ -265,6 +265,9 @@ def __init__( self, partype: T, values: Optional[dict] = None, + # TODO: 'download" reqtype appears to never get used. `None` is most + # common, and `search` is passed in when creating required cmr + # parameters to search for matching granules. reqtype: Optional[Literal["search", "download"]] = None, ): assert partype in [ diff --git a/icepyx/core/granules.py b/icepyx/core/granules.py index d6a519048..52b97265a 100644 --- a/icepyx/core/granules.py +++ b/icepyx/core/granules.py @@ -8,24 +8,24 @@ import re import time from typing import Union -from xml.etree import ElementTree as ET import zipfile import numpy as np import requests -from requests.compat import unquote import icepyx.core.APIformatting as apifmt from icepyx.core.auth import EarthdataAuthMixin -from icepyx.core.cmr import CMR_PROVIDER +from icepyx.core.cmr import CMR_PROVIDER, get_concept_id import icepyx.core.exceptions -from icepyx.core.types import ( +from icepyx.core.harmony import HarmonyApi +from icepyx.core.types.api import ( CMRParams, - EGIRequiredParamsDownload, - EGIRequiredParamsSearch, ) -from icepyx.core.urls import DOWNLOAD_BASE_URL, GRANULE_SEARCH_BASE_URL, ORDER_BASE_URL -from icepyx.uat import EDL_ACCESS_TOKEN +from icepyx.core.urls import DOWNLOAD_BASE_URL, GRANULE_SEARCH_BASE_URL + +# TODO: mix this into existing classes rather than declaring as a global +# variable. +HARMONY_API = HarmonyApi() def info(grans: list[dict]) -> dict[str, Union[int, float]]: @@ -185,13 +185,14 @@ def __init__( # self.files = files # session = session + orderIDs: list[str] = [] + # ---------------------------------------------------------------------- # Methods def get_avail( self, CMRparams: CMRParams, - reqparams: EGIRequiredParamsSearch, cloud: bool = False, ): """ @@ -222,9 +223,7 @@ def get_avail( query.Query.avail_granules """ - assert ( - CMRparams is not None and reqparams is not None - ), "Missing required input parameter dictionaries" + assert CMRparams is not None, "Missing required input parameter dictionary" # if not hasattr(self, 'avail'): self.avail = [] @@ -232,14 +231,12 @@ def get_avail( headers = { "Accept": "application/json", "Client-Id": "icepyx", - "Authorization": f"Bearer {EDL_ACCESS_TOKEN}", } # note we should also check for errors whenever we ping NSIDC-API - # make a function to check for errors params = apifmt.combine_params( CMRparams, - {k: reqparams[k] for k in ["short_name", "version", "page_size"]}, {"provider": CMR_PROVIDER}, ) @@ -286,45 +283,24 @@ def get_avail( len(self.avail) > 0 ), "Your search returned no results; try different search parameters" - # DevNote: currently, default subsetting DOES NOT include variable subsetting, - # only spatial and temporal - # DevGoal: add kwargs to allow subsetting and more control over request options. - def place_order( + def place_harmony_subset_order( self, - CMRparams: CMRParams, - reqparams: EGIRequiredParamsDownload, + # TODO: remove `CMRparams` from this function. This should only take subset params. + CMRparams, subsetparams, - verbose, - subset=True, + # TODO: support passing in a geometry filepath geom_filepath=None, - ): + ) -> list[str]: """ - Place an order for the available granules for the query object. - Adds the list of zipped files (orders) to the granules data object (which is - stored as the `granules` attribute of the query object). - You must be logged in to Earthdata to use this function. + Place a harmony subset order. Parameters ---------- CMRparams : Dictionary of properly formatted CMR search parameters. - reqparams : - Dictionary of properly formatted parameters required for searching, ordering, - or downloading from NSIDC (via their EGI system). subsetparams : dictionary Dictionary of properly formatted subsetting parameters. An empty dictionary is passed as input here when subsetting is set to False in query methods. - verbose : boolean, default False - Print out all feedback available from the order process. - Progress information is automatically printed regardless of the value of verbose. - subset : boolean, default True - Apply subsetting to the data order from the NSIDC, returning only data that meets the - subset parameters. - Spatial and temporal subsetting based on the input parameters happens - by default when subset=True, but additional subsetting options are available. - Spatial subsetting returns all data that are within the area of interest - (but not complete granules. - This eliminates false-positive granules returned by the metadata-level search) geom_filepath : string, default None String of the full filename and path when the spatial input is a file. @@ -337,171 +313,94 @@ def place_order( -------- query.Query.order_granules """ - raise icepyx.core.exceptions.RefactoringException + request_params = apifmt.combine_params(CMRparams, subsetparams) + concept_id = get_concept_id( + product=request_params["short_name"], + version=request_params["version"], + ) - self.get_avail(CMRparams, reqparams) + # Place the order. + # TODO: there are probably other options we want to more generically + # expose here. E.g., instead of just accepting a `bounding_box` of a + # particular flavor, we want to be able to pass in a polygon? + job_id = HARMONY_API.place_order( + concept_id=concept_id, + # TODO: why the double-nested bbox dict here? + bounding_box=subsetparams["bbox"]["bbox"], + temporal=subsetparams["time"], + ) - if subset is False: - request_params = apifmt.combine_params( - CMRparams, reqparams, {"agent": "NO"} - ) - else: - request_params = apifmt.combine_params(CMRparams, reqparams, subsetparams) + # Append this job to the list of order ids. + self.orderIDs.append(job_id) order_fn = ".order_restart" + with open(order_fn, "w") as fid: + json.dump({"orderIDs": self.orderIDs}, fid) - total_pages = int(np.ceil(len(self.avail) / reqparams["page_size"])) - print( - "Total number of data order requests is ", - total_pages, - " for ", - len(self.avail), - " granules.", - ) - - if reqparams["page_num"] > 0: - pagenums = [reqparams["page_num"]] - else: - pagenums = range(1, total_pages + 1) - - for page_num in pagenums: + print("order ID: ", job_id) + status = HARMONY_API.check_order_status(job_id) + print("Initial status of your harmony order request: ", status["status"]) + # TODO: confirm which status responses we might expect. "running", + # "paused", or "canceled" are documented here: + # https://harmony.earthdata.nasa.gov/docs#getting-job-status + # I have also seen `running` and `running_with_errors`. + # The list of possible statues are here: + # https://github.com/nasa/harmony/blob/8b2eb47feab5283d237f3679ac8e09f50e85038f/db/db.sql#L8 + while status["status"].startswith("running"): print( - "Data request ", - page_num, - " of ", - total_pages, - " is submitting to NSIDC", + "Your harmony order status is still ", + status["status"], + ". Please continue waiting... this may take a few moments.", ) - breakpoint() - request_params.update({"page_num": page_num}) - - request = self.session.get(ORDER_BASE_URL, params=request_params) - - # DevGoal: use the request response/number to do some error handling/ - # give the user better messaging for failures - # print(request.content) - # root = ET.fromstring(request.content) - # print([subset_agent.attrib for subset_agent in root.iter('SubsetAgent')]) - - if verbose is True: - print("Request HTTP response: ", request.status_code) - # print('Order request URL: ', request.url) - - # Raise bad request: Loop will stop for bad response code. - request.raise_for_status() - esir_root = ET.fromstring(request.content) - if verbose is True: - print("Order request URL: ", unquote(request.url)) - print( - "Order request response XML content: ", - request.content.decode("utf-8"), - ) - - # Look up order ID - orderlist = [] - for order in esir_root.findall("./order/"): - # if verbose is True: - # print(order) - orderlist.append(order.text) - orderID = orderlist[0] - print("order ID: ", orderID) - - # Create status URL - statusURL = f"{ORDER_BASE_URL}/{orderID}" - if verbose is True: - print("status URL: ", statusURL) - - # Find order status - request_response = self.session.get(statusURL) - if verbose is True: - print( - "HTTP response from order response URL: ", - request_response.status_code, - ) + # Requesting the status too often can result in a 500 error. + time.sleep(5) + status = HARMONY_API.check_order_status(job_id) + + if status["status"] == "complete_with_errors" or status["status"] == "failed": + print("Your order is: ", status["status"]) + print("Harmony provided these error messages:") + pprint.pprint(status["errors"]) + + # TODO: consider always printing the status message. There's no need for + # this check, and the message is relevant regardless of if there are + # errors or not. We could check for a failure status instead. + if status["status"] == "complete" or status["status"] == "complete_with_errors": + print("Your order is:", status["status"]) + print("Harmony returned this message:") + pprint.pprint(status["message"]) + else: + print(f"Request failed with status {status['status']}.") - # Raise bad request: Loop will stop for bad response code. - request_response.raise_for_status() - request_root = ET.fromstring(request_response.content) - statuslist = [] - for status in request_root.findall("./requestStatus/"): - statuslist.append(status.text) - status = statuslist[0] - print("Initial status of your order request at NSIDC is: ", status) - - loop_root = None - # If status is already finished without going into pending/processing - if status.startswith("complete"): - loop_response = self.session.get(statusURL) - loop_root = ET.fromstring(loop_response.content) - - # Continue loop while request is still processing - while status == "pending" or status == "processing": - print( - "Your order status is still ", - status, - " at NSIDC. Please continue waiting... this may take a few moments.", - ) - # print('Status is not complete. Trying again') - time.sleep(10) - loop_response = self.session.get(statusURL) + return self.orderIDs - # Raise bad request: Loop will stop for bad response code. - loop_response.raise_for_status() - loop_root = ET.fromstring(loop_response.content) - - # find status - statuslist = [] - for status in loop_root.findall("./requestStatus/"): - statuslist.append(status.text) - status = statuslist[0] - # print('Retry request status is: ', status) - if status == "pending" or status == "processing": - continue - - if not isinstance(loop_root, ET.Element): - # The typechecker needs help knowing that at this point loop_root is - # set, as it can't tell that the conditionals above are supposed to be - # exhaustive. - raise icepyx.core.exceptions.ExhaustiveTypeGuardException - - # Order can either complete, complete_with_errors, or fail: - # Provide complete_with_errors error message: - if status == "complete_with_errors" or status == "failed": - messagelist = [] - for message in loop_root.findall("./processInfo/"): - messagelist.append(message.text) - print("Your order is: ", status) - print("NSIDC provided these error messages:") - pprint.pprint(messagelist) - - if status == "complete" or status == "complete_with_errors": - print("Your order is:", status) - messagelist = [] - for message in loop_root.findall("./processInfo/info"): - messagelist.append(message.text) - if messagelist != []: - print("NSIDC returned these messages") - pprint.pprint(messagelist) - if not hasattr(self, "orderIDs"): - self.orderIDs = [] - - self.orderIDs.append(orderID) - else: - print("Request failed.") + def place_non_subset_order( + self, + CMRparams: CMRParams, + ): + """ + Place an order for data files with `earthaccess`. - # DevGoal: save orderIDs more frequently than just at the end for large orders - # (e.g. for len(reqparams['page_num']) > 5 or 10 or something) - # Save orderIDs to file to avoid resubmitting order in case kernel breaks down. - # save orderIDs for every 5 orders when more than 10 orders are submitted. - if reqparams["page_num"] >= 10: - with open(order_fn, "w") as fid: - json.dump({"orderIDs": self.orderIDs}, fid) + Parameters + ---------- + CMRparams : + Dictionary of properly formatted CMR search parameters. + subsetparams : dictionary + Dictionary of properly formatted subsetting parameters. An empty dictionary + is passed as input here when subsetting is set to False in query methods. - # --- Output the final orderIDs - with open(order_fn, "w") as fid: - json.dump({"orderIDs": self.orderIDs}, fid) + Notes + ----- + This function is used by query.Query.order_granules(), which automatically + feeds in the required parameters. - return self.orderIDs + See Also + -------- + query.Query.order_granules + """ + # TODO: use e.g., `earthaccess` to download files un-processed by harmony. + raise NotImplementedError("Support for non-subset orders is not implemented.") + self.get_avail(CMRparams) + request_params = apifmt.combine_params(CMRparams, {"agent": "NO"}) def download(self, verbose, path, restart=False): """ diff --git a/icepyx/core/harmony.py b/icepyx/core/harmony.py index 8d03ecc5d..bf0d1cb3e 100644 --- a/icepyx/core/harmony.py +++ b/icepyx/core/harmony.py @@ -1,13 +1,62 @@ -from typing import Any +import datetime as dt +from typing import Any, NotRequired, TypedDict, Union -import requests +import harmony -from icepyx.core.urls import CAPABILITIES_BASE_URL +from icepyx.core.auth import EarthdataAuthMixin -def get_capabilities(concept_id: str) -> dict[str, Any]: - response = requests.get( - CAPABILITIES_BASE_URL, - params={"collectionId": concept_id}, - ) - return response.json() +class HarmonyTemporal(TypedDict): + start: NotRequired[dt.datetime] + stop: NotRequired[dt.datetime] + + +class HarmonyApi(EarthdataAuthMixin): + def __init__(self): + # initialize authentication properties + EarthdataAuthMixin.__init__(self) + self.harmony_client = harmony.Client( + auth=( + self.auth.username, + self.auth.password, + ), + ) + + def get_capabilities(self, concept_id: str) -> dict[str, Any]: + capabilities_request = harmony.CapabilitiesRequest(concept_id=concept_id) + response = self.harmony_client.submit(capabilities_request) + + return response + + def place_order( + self, + concept_id: str, + # These are optional subset parameters + bounding_box: Union[harmony.BBox, None] = None, + temporal: Union[HarmonyTemporal, None] = None, + ) -> str: + """Places a Harmony order with the given parameters. + + Return a string representing a job ID. + + TODO/Question: it looks like this code will always use the provided + parameters to do subsetting. Are there cases where we just want the data + downloaded as whole granules? If so, we may need to use another API to + do so? + """ + collection = harmony.Collection(id=concept_id) + request = harmony.Request( + collection=collection, spatial=bounding_box, temporal=temporal + ) + + if not request.is_valid(): + # TODO: consider more specific error class & message + raise RuntimeError("Failed to create valid request") + + job_id = self.harmony_client.submit(request) + + return job_id + + def check_order_status(self, job_id: str) -> dict[str, Any]: + status = self.harmony_client.status(job_id) + return status diff --git a/icepyx/core/query.py b/icepyx/core/query.py index 573ca8b1b..630efac95 100644 --- a/icepyx/core/query.py +++ b/icepyx/core/query.py @@ -1,7 +1,7 @@ import datetime as dt from functools import cached_property import pprint -from typing import Optional, Union, cast +from typing import Optional, Union import geopandas as gpd import holoviews as hv @@ -17,11 +17,8 @@ import icepyx.core.is2ref as is2ref import icepyx.core.spatial as spat import icepyx.core.temporal as tp -from icepyx.core.types import ( +from icepyx.core.types.api import ( CMRParams, - EGIParamsSubset, - EGIRequiredParams, - EGIRequiredParamsDownload, ) import icepyx.core.validate_inputs as val from icepyx.core.variables import Variables as Variables @@ -574,8 +571,13 @@ def CMRparams(self) -> CMRParams: # print(self._CMRparams) # print(self._CMRparams.fmted_keys) - # dictionary of optional CMR parameters - kwargs = {} + # create a dictionary of required and optional CMR parameters. + # Start by copying the required keys, formatted for CMR requests. + # TODO: This gets set at some point in the code, but later `CMRparams` + # lacks the values in `cmr_reqparams`! Why?? + self.cmr_reqparams + kwargs = self._cmr_reqparams.fmted_keys.copy() + # temporal CMR parameters if hasattr(self, "_temporal") and self.product != "ATL11": kwargs["start"] = self._temporal._start @@ -597,9 +599,9 @@ def CMRparams(self) -> CMRParams: return self._CMRparams.fmted_keys @property - def reqparams(self) -> EGIRequiredParams: + def cmr_reqparams(self): """ - Display the required key:value pairs that will be submitted. + Display the required key:value pairs that will be submitted for a CMR request. It generates the dictionary if it does not already exist. Examples @@ -613,18 +615,49 @@ def reqparams(self) -> EGIRequiredParams: >>> reg_a.reqparams # doctest: +SKIP {'short_name': 'ATL06', 'version': '006', 'page_size': 2000, 'page_num': 1, 'request_mode': 'async', 'include_meta': 'Y', 'client_string': 'icepyx'} """ - raise RefactoringException - if not hasattr(self, "_reqparams"): - self._reqparams = apifmt.Parameters("required", reqtype="search") - self._reqparams.build_params(product=self.product, version=self._version) + self._cmr_reqparams = apifmt.Parameters("required", reqtype="search") + self._cmr_reqparams.build_params( + product=self.product, version=self._version + ) + + return self._cmr_reqparams.fmted_keys + + def get_harmony_subset_order_params(self): + """TODO: this method will return the parameters for a harmony subset order. + + Params are formatted for use with an `HarmonyAPI` instance. This + function will return all of the required and optional parameters. + """ + + def get_cmr_search_params(self): + """TODO: this method will return the parameters for a CMR search query. - return self._reqparams.fmted_keys + Params are formatted for a CMR search URL. This function will return all + of the required and optional parameters. + """ + # This call ensures that `self._cmr_reqparams` is set. + self.cmr_reqparams + + # This combines the CMRparams with the reqparams. + # CMR params are just the user-provided params. The cmr_reqparams + # contains the required CMR parameters that any request needs, + # regardless of user input. + cmr_params = {**self.CMRparams, **self.cmr_reqparams} + + return cmr_params + + def get_non_subset_order_params(self): + """TODO: this method will return the parameters for a non-subset order via earthaccess + + Params are formatted for input into `earthaccess`. This function will + return all of the required and optional parameters. + """ # @property # DevQuestion: if I make this a property, I get a "dict" object is not callable # when I try to give input kwargs... what approach should I be taking? - def subsetparams(self, **kwargs) -> Union[EGIParamsSubset, dict[Never, Never]]: + def subsetparams(self, **kwargs): # -> Union[EGIParamsSubset, dict[Never, Never]]: """ Display the subsetting key:value pairs that will be submitted. It generates the dictionary if it does not already exist @@ -650,7 +683,7 @@ def subsetparams(self, **kwargs) -> Union[EGIParamsSubset, dict[Never, Never]]: {'time': '2019-02-20T00:00:00,2019-02-28T23:59:59', 'bbox': '-55.0,68.0,-48.0,71.0'} """ - raise RefactoringException + # raise RefactoringException if not hasattr(self, "_subsetparams"): self._subsetparams = apifmt.Parameters("subset") @@ -665,6 +698,7 @@ def subsetparams(self, **kwargs) -> Union[EGIParamsSubset, dict[Never, Never]]: else: # If the user has supplied a subset list of variables, append the # icepyx required variables to the Coverage dict + # TODO: this is not supported in Harmony. if "Coverage" in kwargs: var_list = [ "orbit_info/sc_orient", @@ -690,13 +724,13 @@ def subsetparams(self, **kwargs) -> Union[EGIParamsSubset, dict[Never, Never]]: self._subsetparams.build_params( geom_filepath=self._spatial._geom_file, extent_type=self._spatial._ext_type, - spatial_extent=self._spatial.fmt_for_EGI(), + spatial_extent=self._spatial.fmt_for_harmony(), **kwargs, ) else: self._subsetparams.build_params( extent_type=self._spatial._ext_type, - spatial_extent=self._spatial.fmt_for_EGI(), + spatial_extent=self._spatial.fmt_for_harmony(), **kwargs, ) @@ -959,7 +993,7 @@ def avail_granules( try: self.granules.avail except AttributeError: - self.granules.get_avail(self.CMRparams, self.reqparams) + self.granules.get_avail(self.CMRparams) if ids or cycles or tracks or cloud: # list of outputs in order of ids, cycles, tracks, cloud @@ -973,6 +1007,57 @@ def avail_granules( else: return granules.info(self.granules.avail) + def _order_subset_granules(self, **kwargs): + # This call ensures that `self._cmr_reqparams` is set. + self.cmr_reqparams + if self._cmr_reqparams._reqtype == "search": + self._cmr_reqparams._reqtype = "download" + + # TODO: is it necessary to delete the `self._subsetparams` attr? We are + # performing a subset so this seems the opposite of what is expected. + if hasattr(self, "_subsetparams") and self._subsetparams is None: + del self._subsetparams + + cmr_search_params = self.get_cmr_search_params() + + # REFACTOR: add checks here to see if the granules object has been created, + # and also if it already has a list of avail granules (if not, need to create one and add session) + + # Place multiple orders, one per granule, if readable_granule_name is used. + # TODO/Question: is "readable granule name" a thing in harmony? This may + # only be relevant for non-subset requests that will be handled by e.g., + # `earthaccess`. + # Answer: there appears to be a `granuleName` parameter that harmony can + # take for its own internal queries to CMR, but it is unclear how that + # works. See https://harmony.earthdata.nasa.gov/docs#query-parameters + # `harmony-py` accepts `granule_name` as an input. + if "readable_granule_name[]" in cmr_params: + gran_name_list = cmr_params["readable_granule_name[]"] + tempCMRparams = cmr_params.copy() + if len(gran_name_list) > 1: + print( + "Harmony only allows ordering of one granule by name at a time;" + " your orders will be placed accordingly." + ) + for gran in gran_name_list: + tempCMRparams["readable_granule_name[]"] = gran + self.granules.place_harmony_subset_order( + tempCMRparams, + self.subsetparams(granule_name=gran, **kwargs), + geom_filepath=self._spatial._geom_file, + ) + + else: + self.granules.place_harmony_subset_order( + cmr_params, + self.subsetparams(**kwargs), + geom_filepath=self._spatial._geom_file, + ) + + def _order_whole_granules(self): + raise NotImplementedError + self._subsetparams = None + # DevGoal: display output to indicate number of granules successfully ordered (and number of errors) # DevGoal: deal with subset=True for variables now, and make sure that if a variable subset # Coverage kwarg is input it's successfully passed through all other functions even if this is the only one run. @@ -980,7 +1065,6 @@ def order_granules( self, verbose: bool = False, subset: bool = True, - email: bool = False, **kwargs, ) -> None: """ @@ -992,20 +1076,15 @@ def order_granules( Print out all feedback available from the order process. Progress information is automatically printed regardless of the value of verbose. subset : - Apply subsetting to the data order from the NSIDC, returning only data that meets the + Apply subsetting to the data order from Harmony, returning only data that meets the subset parameters. Spatial and temporal subsetting based on the input parameters happens by default when subset=True, but additional subsetting options are available. Spatial subsetting returns all data that are within the area of interest (but not complete granules. This eliminates false-positive granules returned by the metadata-level search) - email : - Have NSIDC auto-send order status email updates to indicate order status as pending/completed. - The emails are sent to the account associated with your Earthdata account. **kwargs : key-value pairs Additional parameters to be passed to the subsetter. By default temporal and spatial subset keys are passed. - Acceptable key values are ['format','projection','projection_parameters','Coverage']. - The variable 'Coverage' list should be constructed using the `order_vars.wanted` attribute of the object. - At this time (2020-05), only variable ('Coverage') parameters will be automatically formatted. + Acceptable key values are [TODO]. See Also -------- @@ -1024,63 +1103,12 @@ def order_granules( . Retry request status is: complete """ - breakpoint() - raise RefactoringException - - if not hasattr(self, "reqparams"): - self.reqparams - - if self._reqparams._reqtype == "search": - self._reqparams._reqtype = "download" - - if "email" in self._reqparams.fmted_keys or email is False: - self._reqparams.build_params(**self._reqparams.fmted_keys) - elif email is True: - user_profile = self.auth.get_user_profile() # pyright: ignore[reportAttributeAccessIssue] - self._reqparams.build_params( - **self._reqparams.fmted_keys, email=user_profile["email_address"] + if subset: + self._order_subset_granules( + **kwargs, ) - - if subset is False: - self._subsetparams = None - elif ( - subset is True - and hasattr(self, "_subsetparams") - and self._subsetparams is None - ): - del self._subsetparams - - # REFACTOR: add checks here to see if the granules object has been created, - # and also if it already has a list of avail granules (if not, need to create one and add session) - - # Place multiple orders, one per granule, if readable_granule_name is used. - if "readable_granule_name[]" in self.CMRparams: - gran_name_list = self.CMRparams["readable_granule_name[]"] - tempCMRparams = self.CMRparams - if len(gran_name_list) > 1: - print( - "NSIDC only allows ordering of one granule by name at a time; your orders will be placed accordingly." - ) - for gran in gran_name_list: - tempCMRparams["readable_granule_name[]"] = gran - self.granules.place_order( - tempCMRparams, - cast(EGIRequiredParamsDownload, self.reqparams), - self.subsetparams(**kwargs), - verbose, - subset, - geom_filepath=self._spatial._geom_file, - ) - else: - self.granules.place_order( - self.CMRparams, - cast(EGIRequiredParamsDownload, self.reqparams), - self.subsetparams(**kwargs), - verbose, - subset, - geom_filepath=self._spatial._geom_file, - ) + self._order_whole_granules() # DevGoal: put back in the kwargs here so that people can just call download granules with subset=False! def download_granules( diff --git a/icepyx/core/spatial.py b/icepyx/core/spatial.py index fef61846f..b282b559b 100644 --- a/icepyx/core/spatial.py +++ b/icepyx/core/spatial.py @@ -4,6 +4,7 @@ import warnings import geopandas as gpd +import harmony import numpy as np from numpy.typing import NDArray from shapely.geometry import Polygon, box @@ -821,3 +822,60 @@ def fmt_for_EGI(self) -> str: else: raise icepyx.core.exceptions.ExhaustiveTypeGuardException + + def fmt_for_harmony(self) -> dict[str, harmony.BBox]: + """ + Format the spatial extent input into format expected by `harmony-py`. + + Returns a dictionary with keys mapping to `harmony.Request` kwargs, with + values appropriately formated for the harmony request. + + `harmony-py` can take two different spatial parameters: + + * `spatial`: "Bounding box spatial constraints on the data or Well Known + Text (WKT) string describing the spatial constraints." The "Bounding + box" is expected to be a `harmony.BBox`. + * `shape`: "a file path to an ESRI Shapefile zip, GeoJSON file, or KML + file to use for spatial subsetting. Note: not all collections support + shapefile subsetting" + + Question: is `spatial` the same as `shape`, in terms of performance? If + so, we could be consistent and always turn the input into geojson and + pass that along to harmony. Otherwise we should choose `spatial` if the + extent_type is bounding, otherwise `shape`. + Answer: No! They're not the same. They map to different harmony + parameters and each is a different service. E.g., some collections may + have bounding box subsetting while others have shape subsetting (or + both). + TODO: think more about how we verify if certain inputs are valid for + harmony. E.g., do we need to check the capabilities of each and + cross-check that with user inputs to determine which action to take? + Also: Does `icepyx` always perform subsetting based on user input? If + not, how do we determine which parameters are for finding granules vs + performing subetting? + + Question: is there any way to pass in a geojson string directly, so that + we do not have to mock out a file just for harmony? Answer: no, not + direcly. `harmony-py` wants a path to a file on disk. We may want to + have the function that submits the request to harmony with `harmony-py` + accept something that's easily-serializable to a geojson file so that it + can manage the lifespan of the file. It would be best (I think) to avoid + writing tmp files to disk in this function, because it doesn't know when + the request gets made/when to cleanup the file. That means that we may + leave stray files on the user's computer. Ideally, we would be able to + pass `harmony-py` a bytes object (or a shapely Polygon!) + """ + # Begin with bounding box because this is the simplest case. + if self.extent_type == "bounding_box": + harmony_kwargs = { + "bbox": harmony.BBox( + w=self.extent[0], + s=self.extent[1], + e=self.extent[2], + n=self.extent[3], + ) + } + + return harmony_kwargs + else: + raise NotImplementedError diff --git a/icepyx/core/types/api.py b/icepyx/core/types/api.py index b29ba8fb4..9af9bb9d6 100644 --- a/icepyx/core/types/api.py +++ b/icepyx/core/types/api.py @@ -1,11 +1,14 @@ -from typing import Literal, TypedDict, Union +from typing import TypedDict, Union -from typing_extensions import NotRequired from pydantic import BaseModel +from typing_extensions import NotRequired CMRParamsBase = TypedDict( "CMRParamsBase", { + "short_name": str, + "version": str, + "page_size": int, "temporal": NotRequired[str], "options[readable_granule_name][pattern]": NotRequired[str], "options[spatial][or]": NotRequired[str], @@ -25,4 +28,4 @@ class CMRParamsWithPolygon(CMRParamsBase): CMRParams = Union[CMRParamsWithBbox, CMRParamsWithPolygon] -class HarmonyCoverageAPIParamsBase(BaseModel): +class HarmonyCoverageAPIParamsBase(BaseModel): ... diff --git a/requirements.txt b/requirements.txt index 1ff3a8824..fc5e71b16 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ fiona geopandas h5netcdf h5py +harmony-py holoviews hvplot matplotlib