diff --git a/src/fibad/download.py b/src/fibad/download.py index 11a3529..68338f1 100644 --- a/src/fibad/download.py +++ b/src/fibad/download.py @@ -71,9 +71,10 @@ def run(config): return # Create thread objects for each of our worker threads - num_threads = config.get("concurrent_connections", 2) - if num_threads > 5: - raise RuntimeError("This client only opens 5 connections or fewer.") + num_threads = config.get("concurrent_connections", 10) + + if num_threads > 10: + RuntimeError("More than 10 concurrent connections to HSC is disallowed on a per-user basis") # If we are using more than one connection, cut the list of rectangles into # batches, one batch for each thread. @@ -413,6 +414,9 @@ def rect_from_config(config: dict) -> dC.Rect: filter=config["filter"], rerun=config["rerun"], type=config["type"], + image=config.get("image"), + mask=config.get("mask"), + variance=config.get("variance"), ) @staticmethod @@ -461,7 +465,10 @@ def create_rects( for index, location in enumerate(locations): args = {field: location[field] for field in fields} args["lineno"] = index + offset + + # tracts are ints in the fits files and dC.rect constructor wants them as str args["tract"] = str(args["tract"]) + # Sets the file name on the rect to be the object_id, also includes other rect fields # which are interpolated at save time, and are native fields of dc.Rect. args["name"] = str( @@ -471,9 +478,9 @@ def create_rects( rects.append(rect) # We sort rects here so they end up tract,ra,dec ordered across all requests made in all threads - # Threads do their own sorting prior to each chunked request in downloadCutout.py; however + # Threads do their own sorting prior to all chunked request in downloadCutout.py; however # sorting at this stage will allow a greater number of rects that are co-located in the sky - # to end up in the same thread and same chunk. + # to end up in the same thread and therefore be sorted into the same request. rects.sort() return rects diff --git a/src/fibad/downloadCutout/downloadCutout.py b/src/fibad/downloadCutout/downloadCutout.py index a0c1706..6da1cfa 100644 --- a/src/fibad/downloadCutout/downloadCutout.py +++ b/src/fibad/downloadCutout/downloadCutout.py @@ -328,8 +328,8 @@ class Rect: rerun: str = default_rerun type: str = default_type - filter: Union[str, list[str]] = ALLFILTERS tract: int = ANYTRACT + filter: Union[str, list[str]] = ALLFILTERS ra: float = math.nan dec: float = math.nan sw: float = math.nan @@ -344,8 +344,8 @@ class Rect: def create( rerun: Union[str, None] = None, type: Union[str, None] = None, - filter: Union[Union[str, list[str]], None] = None, tract: Union[str, int, None] = None, + filter: Union[Union[str, list[str]], None] = None, ra: Union[str, float, None] = None, dec: Union[str, float, None] = None, sw: Union[str, float, None] = None, diff --git a/src/fibad/fibad_default_config.toml b/src/fibad/fibad_default_config.toml index ec68156..eb202d1 100644 --- a/src/fibad/fibad_default_config.toml +++ b/src/fibad/fibad_default_config.toml @@ -35,7 +35,7 @@ retries = 3 # `timepout` How long should we wait to get a full HTTP response from the server. Default 3600s (1hr) timeout = 3600 # `chunksize` How many sky location rectangles should we request in a single request. Default is 990 -chunksize = 990 +chunk_size = 990 [model] name = "ExampleAutoencoder"