SYMBOL INDEX (1312 symbols across 96 files) FILE: weather_dl/download_pipeline/__init__.py function cli (line 18) | def cli(extra=[]): FILE: weather_dl/download_pipeline/clients.py class Client (line 41) | class Client(abc.ABC): method __init__ (line 52) | def __init__(self, config: Config, level: int = logging.INFO) -> None: method retrieve (line 59) | def retrieve(self, dataset: str, selection: t.Dict, output: str, manif... method num_requests_per_key (line 65) | def num_requests_per_key(cls, dataset: str) -> int: method license_url (line 71) | def license_url(self): class SplitCDSRequest (line 76) | class SplitCDSRequest(): method __init__ (line 78) | def __init__(self, *args, **kwargs): method _download (line 82) | def _download(self, url, path: str, size: int) -> None: method fetch (line 92) | def fetch(self, request: t.Dict, dataset: str) -> t.Dict: method download (line 96) | def download(self, result: cds_api.Result, target: t.Optional[str] = N... class CdsClient (line 107) | class CdsClient(Client): method retrieve (line 129) | def retrieve(self, dataset: str, selection: t.Dict, output: str, manif... method license_url (line 158) | def license_url(self): method num_requests_per_key (line 162) | def num_requests_per_key(cls, dataset: str) -> int: class StdoutLogger (line 183) | class StdoutLogger(io.StringIO): method __init__ (line 186) | def __init__(self, logger_: logging.Logger, level: int = logging.INFO): method log (line 192) | def log(self, msg) -> None: method write (line 195) | def write(self, msg): method __enter__ (line 199) | def __enter__(self): method __exit__ (line 203) | def __exit__(self, exc_type, exc_value, traceback): class SplitMARSRequest (line 208) | class SplitMARSRequest(api.APIRequest): method _download (line 211) | def _download(self, url, path: str, size: int) -> None: method fetch (line 219) | def fetch(self, request: t.Dict, dataset: str) -> t.Dict: method download (line 242) | def download(self, result: t.Dict, target: t.Optional[str] = None) -> ... class SplitRequestMixin (line 254) | class SplitRequestMixin: method fetch (line 257) | def fetch(self, req: t.Dict, dataset: t.Optional[str] = None) -> t.Dict: method download (line 260) | def download(self, res: t.Dict, target: str) -> None: class CDSClientExtended (line 264) | class CDSClientExtended(SplitRequestMixin): method __init__ (line 266) | def __init__(self, *args, **kwargs): class MARSECMWFServiceExtended (line 270) | class MARSECMWFServiceExtended(api.ECMWFService, SplitRequestMixin): method __init__ (line 272) | def __init__(self, *args, **kwargs): class PublicECMWFServerExtended (line 285) | class PublicECMWFServerExtended(api.ECMWFDataServer, SplitRequestMixin): method __init__ (line 286) | def __init__(self, *args, dataset='', **kwargs): class MarsClient (line 298) | class MarsClient(Client): method retrieve (line 317) | def retrieve(self, dataset: str, selection: t.Dict, output: str, manif... method license_url (line 346) | def license_url(self): method num_requests_per_key (line 350) | def num_requests_per_key(cls, dataset: str) -> int: class ECMWFPublicClient (line 365) | class ECMWFPublicClient(Client): method retrieve (line 367) | def retrieve(self, dataset: str, selection: t.Dict, output: str, manif... method num_requests_per_key (line 396) | def num_requests_per_key(cls, dataset: str) -> int: method license_url (line 401) | def license_url(self): class FakeClient (line 407) | class FakeClient(Client): method retrieve (line 410) | def retrieve(self, dataset: str, selection: t.Dict, output: str, manif... method license_url (line 423) | def license_url(self): method num_requests_per_key (line 427) | def num_requests_per_key(cls, dataset: str) -> int: FILE: weather_dl/download_pipeline/clients_test.py class MaxWorkersTest (line 20) | class MaxWorkersTest(unittest.TestCase): method test_cdsclient_internal (line 21) | def test_cdsclient_internal(self): method test_cdsclient_mars_hosted (line 24) | def test_cdsclient_mars_hosted(self): method test_marsclient (line 27) | def test_marsclient(self): method test_fakeclient (line 30) | def test_fakeclient(self): FILE: weather_dl/download_pipeline/config.py class Config (line 24) | class Config: method from_dict (line 66) | def from_dict(cls, config: t.Dict) -> 'Config': function optimize_selection_partition (line 80) | def optimize_selection_partition(selection: t.Dict) -> t.Dict: FILE: weather_dl/download_pipeline/config_test.py class ConfigTest (line 24) | class ConfigTest(unittest.TestCase): method setUp (line 26) | def setUp(self): method test_process_config_files (line 29) | def test_process_config_files(self): method test_process_multi_config_files (line 43) | def test_process_multi_config_files(self): class SelectionSyntaxTest (line 63) | class SelectionSyntaxTest(unittest.TestCase): method test_all_days__invalid_year (line 65) | def test_all_days__invalid_year(self): method test_all_days__invalid_month (line 74) | def test_all_days__invalid_month(self): method test_date_range (line 83) | def test_date_range(self): method test_with_year_month_as_string (line 88) | def test_with_year_month_as_string(self): method test_with_multiple_months (line 101) | def test_with_multiple_months(self): method test_with_multiple_years (line 117) | def test_with_multiple_years(self): FILE: weather_dl/download_pipeline/fetcher.py class Fetcher (line 34) | class Fetcher(beam.DoFn): method __post_init__ (line 53) | def __post_init__(self): method retrieve (line 58) | def retrieve(self, client: Client, dataset: str, selection: t.Dict, de... method fetch_data (line 62) | def fetch_data(self, config: Config, *, worker_name: str = 'default') ... method process (line 93) | def process(self, element) -> None: FILE: weather_dl/download_pipeline/fetcher_test.py class FetchDataTest (line 28) | class FetchDataTest(unittest.TestCase): method setUp (line 30) | def setUp(self) -> None: method test_fetch_data (line 35) | def test_fetch_data(self, mock_fetch, mock_download): method test_fetch_data__manifest__returns_success (line 64) | def test_fetch_data__manifest__returns_success(self, mock_fetch, mock_... method test_fetch_data__manifest__records_retrieve_failure (line 94) | def test_fetch_data__manifest__records_retrieve_failure(self, mock_fet... method test_fetch_data__manifest__records_gcs_failure (line 132) | def test_fetch_data__manifest__records_gcs_failure(self, mock_fetch): method test_fetch_data__skips_existing_download (line 171) | def test_fetch_data__skips_existing_download(self, mock_fetch, mock_gc... FILE: weather_dl/download_pipeline/manifest.py class ManifestException (line 53) | class ManifestException(Exception): class Stage (line 58) | class Stage(enum.Enum): class Status (line 76) | class Status(enum.Enum): class DownloadStatus (line 93) | class DownloadStatus(): method from_dict (line 154) | def from_dict(cls, download_status: t.Dict) -> 'DownloadStatus': method to_dict (line 167) | def to_dict(cls, instance) -> t.Dict: class Manifest (line 186) | class Manifest(abc.ABC): method __post_init__ (line 227) | def __post_init__(self): method schedule (line 231) | def schedule(self, config_name: str, dataset: str, selection: t.Dict, ... method skip (line 260) | def skip(self, config_name: str, dataset: str, selection: t.Dict, loca... method _set_for_transaction (line 300) | def _set_for_transaction(self, config_name: str, dataset: str, selecti... method __enter__ (line 309) | def __enter__(self) -> None: method __exit__ (line 312) | def __exit__(self, exc_type, exc_inst, exc_tb) -> None: method transact (line 352) | def transact(self, config_name: str, dataset: str, selection: t.Dict, ... method set_stage (line 357) | def set_stage(self, stage: Stage) -> None: method _read (line 393) | def _read(self, location: str) -> DownloadStatus: method _update (line 397) | def _update(self, download_status: DownloadStatus) -> None: class ConsoleManifest (line 401) | class ConsoleManifest(Manifest): method __post_init__ (line 403) | def __post_init__(self): method _read (line 406) | def _read(self, location: str) -> DownloadStatus: method _update (line 409) | def _update(self, download_status: DownloadStatus) -> None: class LocalManifest (line 413) | class LocalManifest(Manifest): method __init__ (line 418) | def __init__(self, location: Location) -> None: method _read (line 428) | def _read(self, location: str) -> DownloadStatus: method _update (line 436) | def _update(self, download_status: DownloadStatus) -> None: class BQManifest (line 452) | class BQManifest(Manifest): method __init__ (line 458) | def __init__(self, location: Location) -> None: method _read (line 509) | def _read(self, location: str) -> DownloadStatus: method _update (line 533) | def _update(self, download_status: DownloadStatus) -> None: class FirestoreManifest (line 580) | class FirestoreManifest(Manifest): method _get_db (line 590) | def _get_db(self) -> firestore.firestore.Client: method _read (line 616) | def _read(self, location: str) -> DownloadStatus: method _update (line 633) | def _update(self, download_status: DownloadStatus) -> None: method root_document_for_store (line 651) | def root_document_for_store(self, store_scheme: str) -> DocumentRefere... method get_firestore_config (line 657) | def get_firestore_config(self) -> t.Dict: class MockManifest (line 683) | class MockManifest(Manifest): method __init__ (line 686) | def __init__(self, location: Location) -> None: method _read (line 690) | def _read(self, location: str) -> DownloadStatus: method _update (line 694) | def _update(self, download_status: DownloadStatus) -> None: class NoOpManifest (line 701) | class NoOpManifest(Manifest): method _read (line 704) | def _read(self, location: str) -> DownloadStatus: method _update (line 707) | def _update(self, download_status: DownloadStatus) -> None: FILE: weather_dl/download_pipeline/manifest_test.py function rand_str (line 25) | def rand_str(max_len=32): function make_download_status (line 29) | def make_download_status(location: t.Optional[str] = None) -> DownloadSt... class LocalManifestTest (line 52) | class LocalManifestTest(unittest.TestCase): method test_empty_manifest_is_valid_json (line 55) | def test_empty_manifest_is_valid_json(self): method does_not_overwrite_existing_manifest (line 61) | def does_not_overwrite_existing_manifest(self): method test_writes_valid_json (line 73) | def test_writes_valid_json(self): method test_overwrites_existing_statuses (line 82) | def test_overwrites_existing_statuses(self): method is_valid_json (line 96) | def is_valid_json(self, file: t.IO) -> None: FILE: weather_dl/download_pipeline/parsers.py function date (line 34) | def date(candidate: str) -> datetime.date: function time (line 73) | def time(candidate: str) -> datetime.time: function day_month_year (line 107) | def day_month_year(candidate: t.Any) -> int: function date_range_converter (line 119) | def date_range_converter(candidate: str) -> str: function parse_literal (line 124) | def parse_literal(candidate: t.Any) -> t.Any: function validate (line 134) | def validate(key: str, value: int) -> None: function typecast (line 142) | def typecast(key: str, value: t.Any) -> t.Any: function _read_config_file (line 157) | def _read_config_file(file: t.IO) -> t.Dict: function parse_config (line 175) | def parse_config(file: t.IO) -> t.Dict: function parse_manifest (line 183) | def parse_manifest(location: Location, pipeline_opts: t.Dict) -> Manifest: function _splitlines (line 200) | def _splitlines(block: str) -> t.List[str]: function mars_range_value (line 205) | def mars_range_value(token: str, key: str) -> t.Union[datetime.date, int... function mars_increment_value (line 225) | def mars_increment_value(token: str) -> t.Union[int, float]: function parse_mars_syntax (line 238) | def parse_mars_syntax(block: str, key: str) -> t.List[str]: function date_range (line 322) | def date_range(start: datetime.date, end: datetime.date, increment: int ... function _parse_lists (line 328) | def _parse_lists(config: dict, section: str = '') -> t.Dict: function _number_of_replacements (line 343) | def _number_of_replacements(s: t.Text): function parse_subsections (line 351) | def parse_subsections(config: t.Dict) -> t.Dict: function require (line 372) | def require(condition: bool, message: str, error_type: t.Type[Exception]... function process_config (line 378) | def process_config(file: t.IO, config_name: str) -> Config: function prepare_target_name (line 482) | def prepare_target_name(config: Config) -> str: function get_subsections (line 490) | def get_subsections(config: Config) -> t.List[t.Tuple[str, t.Dict]]: function all_equal (line 515) | def all_equal(iterator): function validate_all_configs (line 524) | def validate_all_configs(configs: t.List[Config]) -> None: FILE: weather_dl/download_pipeline/parsers_test.py class DateTest (line 30) | class DateTest(unittest.TestCase): method test_parses_relative_date (line 32) | def test_parses_relative_date(self): method test_parses_kebob_date (line 36) | def test_parses_kebob_date(self): method test_parses_smooshed_date (line 40) | def test_parses_smooshed_date(self): method test_parses_year_and_day_of_year (line 44) | def test_parses_year_and_day_of_year(self): method test_throws_error (line 49) | def test_throws_error(self): class ParseConfigTest (line 63) | class ParseConfigTest(unittest.TestCase): method _assert_no_newlines_in_section (line 65) | def _assert_no_newlines_in_section(self, dictionary) -> None: method test_json (line 69) | def test_json(self): method test_bad_json (line 74) | def test_bad_json(self): method test_json_produces_lists (line 79) | def test_json_produces_lists(self): method test_json_parses_mars_list (line 85) | def test_json_parses_mars_list(self): method test_json_parses_mars_int_range (line 91) | def test_json_parses_mars_int_range(self): method test_json_parses_mars_int_range_padded (line 97) | def test_json_parses_mars_int_range_padded(self): method test_json_parses_mars_int_range_incremented (line 103) | def test_json_parses_mars_int_range_incremented(self): method test_json_parses_mars_float_range (line 109) | def test_json_parses_mars_float_range(self): method test_json_parses_mars_float_range_incremented (line 115) | def test_json_parses_mars_float_range_incremented(self): method test_json_parses_mars_float_range_incremented_by_float (line 121) | def test_json_parses_mars_float_range_incremented_by_float(self): method test_json_parses_mars_date_range (line 127) | def test_json_parses_mars_date_range(self): method test_json_parses_mars_relative_date_range (line 133) | def test_json_parses_mars_relative_date_range(self): method test_json_parses_mars_date_range_incremented (line 146) | def test_json_parses_mars_date_range_incremented(self): method test_json_raises_syntax_error_missing_right (line 152) | def test_json_raises_syntax_error_missing_right(self): method test_json_raises_syntax_error_missing_left (line 157) | def test_json_raises_syntax_error_missing_left(self): method test_json_raises_syntax_error_missing_increment (line 162) | def test_json_raises_syntax_error_missing_increment(self): method test_json_raises_syntax_error_no_range (line 167) | def test_json_raises_syntax_error_no_range(self): method test_json_raises_value_error_date_types (line 172) | def test_json_raises_value_error_date_types(self): method test_json_raises_value_error_float_types (line 180) | def test_json_raises_value_error_float_types(self): method test_json_raises_value_error_int_types (line 185) | def test_json_raises_value_error_int_types(self): method test_json_parses_accidental_extra_whitespace (line 190) | def test_json_parses_accidental_extra_whitespace(self): method test_json_parses_parameter_subsections (line 196) | def test_json_parses_parameter_subsections(self): method test_cfg (line 209) | def test_cfg(self): method test_bad_cfg (line 219) | def test_bad_cfg(self): method test_cfg_produces_lists (line 228) | def test_cfg_produces_lists(self): method test_cfg_parses_mars_list (line 244) | def test_cfg_parses_mars_list(self): method test_cfg_parses_mars_int_range (line 256) | def test_cfg_parses_mars_int_range(self): method test_cfg_parses_mars_int_range_padded (line 268) | def test_cfg_parses_mars_int_range_padded(self): method test_cfg_parses_mars_int_range_incremented (line 280) | def test_cfg_parses_mars_int_range_incremented(self): method test_cfg_parses_mars_int_reverse_range_incremented (line 292) | def test_cfg_parses_mars_int_reverse_range_incremented(self): method test_cfg_parses_mars_float_range (line 304) | def test_cfg_parses_mars_float_range(self): method test_cfg_parses_mars_float_range_incremented (line 316) | def test_cfg_parses_mars_float_range_incremented(self): method test_cfg_parses_mars_float_range_incremented_by_float (line 328) | def test_cfg_parses_mars_float_range_incremented_by_float(self): method test_cfg_parses_mars_float_reverse_range_incremented_by_float (line 340) | def test_cfg_parses_mars_float_reverse_range_incremented_by_float(self): method test_cfg_parses_mars_date_range (line 352) | def test_cfg_parses_mars_date_range(self): method test_cfg_parses_mars_relative_date_range (line 364) | def test_cfg_parses_mars_relative_date_range(self): method test_cfg_parses_mars_relative_date_reverse_range (line 383) | def test_cfg_parses_mars_relative_date_reverse_range(self): method test_cfg_parses_mars_date_range_incremented (line 403) | def test_cfg_parses_mars_date_range_incremented(self): method test_cfg_parses_mars_date_reverse_range_incremented (line 415) | def test_cfg_parses_mars_date_reverse_range_incremented(self): method test_cfg_raises_syntax_error_missing_right (line 428) | def test_cfg_raises_syntax_error_missing_right(self): method test_cfg_raises_syntax_error_missing_left (line 439) | def test_cfg_raises_syntax_error_missing_left(self): method test_cfg_raises_syntax_error_missing_increment (line 450) | def test_cfg_raises_syntax_error_missing_increment(self): method test_cfg_raises_syntax_error_no_range (line 461) | def test_cfg_raises_syntax_error_no_range(self): method test_cfg_raises_value_error_date_types (line 472) | def test_cfg_raises_value_error_date_types(self): method test_cfg_raises_value_error_float_types (line 486) | def test_cfg_raises_value_error_float_types(self): method test_cfg_raises_value_error_int_types (line 497) | def test_cfg_raises_value_error_int_types(self): method test_cfg_parses_accidental_extra_whitespace (line 508) | def test_cfg_parses_accidental_extra_whitespace(self): method test_json_parse_year_mon_one (line 521) | def test_json_parse_year_mon_one(self): method test_json_parse_year_mon_one_by_one (line 530) | def test_json_parse_year_mon_one_by_one(self): method test_json_parse_year_mon_one_by_two (line 539) | def test_json_parse_year_mon_one_by_two(self): method test_json_parse_year_mon_six (line 548) | def test_json_parse_year_mon_six(self): method test_json_parse_year_mon_six_by_one (line 557) | def test_json_parse_year_mon_six_by_one(self): method test_json_parse_year_mon_six_by_three (line 566) | def test_json_parse_year_mon_six_by_three(self): method test_json_parse_year_mon_six_rev (line 575) | def test_json_parse_year_mon_six_rev(self): method test_json_parse_year_mon_six_by_one_rev (line 584) | def test_json_parse_year_mon_six_by_one_rev(self): method test_json_parse_year_mon_six_by_three_rev (line 593) | def test_json_parse_year_mon_six_by_three_rev(self): method test_json_parse_year_mon_eighteen (line 602) | def test_json_parse_year_mon_eighteen(self): method test_json_parse_year_mon_eighteen_by_two (line 613) | def test_json_parse_year_mon_eighteen_by_two(self): method test_json_parse_year_mon_eighteen_by_six (line 622) | def test_json_parse_year_mon_eighteen_by_six(self): method test_cfg_parses_parameter_subsections (line 631) | def test_cfg_parses_parameter_subsections(self): class HelpersTest (line 652) | class HelpersTest(unittest.TestCase): method test_number_of_replacements (line 659) | def test_number_of_replacements(self): class SubsectionsTest (line 666) | class SubsectionsTest(unittest.TestCase): method test_parses_config_subsections (line 667) | def test_parses_config_subsections(self): class ApiKeyCountingTest (line 675) | class ApiKeyCountingTest(unittest.TestCase): method test_no_keys (line 676) | def test_no_keys(self): method test_api_keys (line 683) | def test_api_keys(self): class ProcessConfigTest (line 696) | class ProcessConfigTest(unittest.TestCase): method test_parse_config (line 698) | def test_parse_config(self): method test_require_params_section (line 709) | def test_require_params_section(self): method test_accepts_parameters_section (line 723) | def test_accepts_parameters_section(self): method test_requires_target_path_param (line 737) | def test_requires_target_path_param(self): method test_requires_target_template_param_not_present (line 753) | def test_requires_target_template_param_not_present(self): method test_accepts_target_path_param (line 769) | def test_accepts_target_path_param(self): method test_requires_partition_keys_to_match_sections (line 785) | def test_requires_partition_keys_to_match_sections(self): method test_accepts_partition_keys_matching_sections (line 818) | def test_accepts_partition_keys_matching_sections(self): method test_accepts_partition_keys_not_present (line 847) | def test_accepts_partition_keys_not_present(self): method test_treats_partition_keys_as_list (line 873) | def test_treats_partition_keys_as_list(self): method test_mismatched_template_partition_keys (line 891) | def test_mismatched_template_partition_keys(self): method test_append_date_dirs_raise_error (line 924) | def test_append_date_dirs_raise_error(self): method test_target_filename_raise_error (line 948) | def test_target_filename_raise_error(self): method test_client_not_set (line 971) | def test_client_not_set(self): method test_client_invalid (line 991) | def test_client_invalid(self): method test_partition_cannot_include_all (line 1012) | def test_partition_cannot_include_all(self): method test_partition_key_contains_date_range (line 1034) | def test_partition_key_contains_date_range(self): method test_partition_key_contains_date__in_case_of_hdate (line 1052) | def test_partition_key_contains_date__in_case_of_hdate(self): method test_singleton_partitions_are_converted_to_lists (line 1070) | def test_singleton_partitions_are_converted_to_lists(self): class PrepareTargetNameTest (line 1090) | class PrepareTargetNameTest(unittest.TestCase): method setUp (line 1191) | def setUp(self) -> None: method test_target_name (line 1194) | def test_target_name(self): FILE: weather_dl/download_pipeline/partition.py class PartitionConfig (line 36) | class PartitionConfig(beam.PTransform): method expand (line 66) | def expand(self, configs): function _create_partition_config (line 118) | def _create_partition_config(option: t.Tuple, config: Config) -> Config: function skip_partition (line 149) | def skip_partition(config: Config, store: Store, manifest: Manifest) -> ... function prepare_partition_index (line 164) | def prepare_partition_index(config: Config, function prepare_partitions_from_index (line 194) | def prepare_partitions_from_index(config: Config, indexes: t.List[Index]... function new_downloads_only (line 210) | def new_downloads_only(candidate: Config, store: t.Optional[Store] = None, function assemble_config (line 221) | def assemble_config(partition: Partition, manifest: Manifest) -> Config: function cycle_iters (line 254) | def cycle_iters(iters: t.List[t.Iterator], take: int = 1) -> t.Iterator: function prepare_fair_partition_index (line 275) | def prepare_fair_partition_index(configs: t.List[Config], FILE: weather_dl/download_pipeline/partition_test.py class OddFilesDoNotExistStore (line 31) | class OddFilesDoNotExistStore(InMemoryStore): method __init__ (line 32) | def __init__(self): method exists (line 36) | def exists(self, filename: str) -> bool: class PreparePartitionTest (line 42) | class PreparePartitionTest(unittest.TestCase): method setUp (line 44) | def setUp(self) -> None: method create_partition_configs (line 47) | def create_partition_configs(self, configs, method test_partition_single_key (line 59) | def test_partition_single_key(self): method test_partition_multi_key (line 80) | def test_partition_multi_key(self): method test_partition_multi_key_single_values (line 103) | def test_partition_multi_key_single_values(self): method test_partition_multi_params_multi_key (line 123) | def test_partition_multi_params_multi_key(self): method test_prepare_partition_records_download_status_to_manifest (line 168) | def test_prepare_partition_records_download_status_to_manifest(self): method test_prepare_partition_records_download_status_to_manifest_for_already_downloaded_shard (line 201) | def test_prepare_partition_records_download_status_to_manifest_for_alr... method test_prepare_partition_update_download_status_for_downloaded_shard_missing_upload_entry (line 238) | def test_prepare_partition_update_download_status_for_downloaded_shard... method test_prepare_partition_update_manifest_for_failed_upload_status_of_downloaded_shard (line 280) | def test_prepare_partition_update_manifest_for_failed_upload_status_of... method test_skip_partitions__never_unbalances_licenses (line 323) | def test_skip_partitions__never_unbalances_licenses(self): method test_multi_config_partition_single_key (line 352) | def test_multi_config_partition_single_key(self): method test_multi_config_partition_single_key_fair_schedule (line 376) | def test_multi_config_partition_single_key_fair_schedule(self): method test_multi_config_partition_multi_params_multi_key (line 400) | def test_multi_config_partition_multi_params_multi_key(self): method test_multi_config_partition_multi_params_fair_schedule (line 449) | def test_multi_config_partition_multi_params_fair_schedule(self): method test_multi_config_partition_multi_params_keys_and_requests_with_fair_schedule (line 499) | def test_multi_config_partition_multi_params_keys_and_requests_with_fa... method test_hdate_partition_single_key (line 551) | def test_hdate_partition_single_key(self): method test_partition_with_no_keys (line 572) | def test_partition_with_no_keys(self): class SkipPartitionsTest (line 589) | class SkipPartitionsTest(unittest.TestCase): method setUp (line 591) | def setUp(self) -> None: method test_skip_partition_missing_force_download (line 595) | def test_skip_partition_missing_force_download(self): method test_skip_partition_force_download_true (line 613) | def test_skip_partition_force_download_true(self): method test_skip_partition_force_download_false (line 632) | def test_skip_partition_force_download_false(self): FILE: weather_dl/download_pipeline/pipeline.py function configure_logger (line 51) | def configure_logger(verbosity: int) -> None: class PipelineArgs (line 65) | class PipelineArgs: function pipeline (line 86) | def pipeline(args: PipelineArgs) -> None: function run (line 135) | def run(argv: t.List[str], save_main_session: bool = True) -> PipelineArgs: FILE: weather_dl/download_pipeline/pipeline_test.py function default_args (line 72) | def default_args(parameters: t.Optional[t.Dict] = None, selection: t.Opt... class ParsePipelineArgs (line 94) | class ParsePipelineArgs(unittest.TestCase): method assert_pipeline (line 97) | def assert_pipeline(self, args, expected): method test_happy_path (line 111) | def test_happy_path(self): method test_force_download (line 114) | def test_force_download(self): method test_dry_run (line 120) | def test_dry_run(self): method test_local_run (line 130) | def test_local_run(self): method test_update_manifest (line 140) | def test_update_manifest(self): method test_user_specified_num_requests_per_key (line 146) | def test_user_specified_num_requests_per_key(self): method test_check_skip_in_dry_run_raise_error_if_dry_run_flag_is_absent (line 154) | def test_check_skip_in_dry_run_raise_error_if_dry_run_flag_is_absent(s... FILE: weather_dl/download_pipeline/stores.py class Store (line 27) | class Store(abc.ABC): method open (line 34) | def open(self, filename: str, mode: str = 'r') -> t.IO: method exists (line 38) | def exists(self, filename: str) -> bool: class InMemoryStore (line 42) | class InMemoryStore(Store): method __init__ (line 45) | def __init__(self): method open (line 48) | def open(self, filename: str, mode: str = 'r') -> t.IO: method exists (line 57) | def exists(self, filename: str) -> bool: class TempFileStore (line 62) | class TempFileStore(Store): method __init__ (line 65) | def __init__(self, directory: t.Optional[str] = None) -> None: method open (line 71) | def open(self, filename: str, mode: str = 'r') -> t.IO: method exists (line 75) | def exists(self, filename: str) -> bool: class LocalFileStore (line 80) | class LocalFileStore(Store): method __init__ (line 83) | def __init__(self, directory: t.Optional[str] = None) -> None: method open (line 89) | def open(self, filename: str, mode: str = 'r') -> t.IO: method exists (line 93) | def exists(self, filename: str) -> bool: class FSStore (line 98) | class FSStore(Store): method open (line 102) | def open(self, filename: str, mode: str = 'r') -> t.IO: method exists (line 121) | def exists(self, filename: str) -> bool: FILE: weather_dl/download_pipeline/stores_test.py class FSStoreTest (line 21) | class FSStoreTest(unittest.TestCase): method test_writes (line 23) | def test_writes(self): method test_reads (line 31) | def test_reads(self): method test_reads__default_argument (line 39) | def test_reads__default_argument(self): method test_asserts_bad_mode__both (line 47) | def test_asserts_bad_mode__both(self): method test_asserts_bad_mode__neither (line 54) | def test_asserts_bad_mode__neither(self): FILE: weather_dl/download_pipeline/util.py function _retry_if_valid_input_but_server_or_socket_error_and_timeout_filter (line 42) | def _retry_if_valid_input_but_server_or_socket_error_and_timeout_filter(... class _FakeClock (line 53) | class _FakeClock: method sleep (line 54) | def sleep(self, value): function retry_with_exponential_backoff (line 58) | def retry_with_exponential_backoff(fun): function ichunked (line 73) | def ichunked(iterable: t.Iterable, n: int) -> t.Iterator[t.Iterable]: function copy (line 87) | def copy(src: str, dst: str) -> None: function to_json_serializable_type (line 105) | def to_json_serializable_type(value: t.Any) -> t.Any: function fetch_geo_polygon (line 148) | def fetch_geo_polygon(area: t.Union[list, str]) -> str: function get_file_size (line 179) | def get_file_size(path: str) -> float: function get_wait_interval (line 187) | def get_wait_interval(num_retries: int = 0) -> float: function generate_md5_hash (line 194) | def generate_md5_hash(input: str) -> str: function download_with_aria2 (line 199) | def download_with_aria2(url: str, path: str) -> None: function generate_hdate (line 213) | def generate_hdate(date: str, subtract_year: str) -> str: FILE: weather_dl/download_pipeline/util_test.py class IChunksTests (line 7) | class IChunksTests(unittest.TestCase): method setUp (line 8) | def setUp(self) -> None: method test_even_chunks (line 11) | def test_even_chunks(self): method test_odd_chunks (line 24) | def test_odd_chunks(self): class TestFetchGeoPolygon (line 36) | class TestFetchGeoPolygon(unittest.TestCase): method test_valid_area (line 37) | def test_valid_area(self): method test_valid_string_area_value (line 46) | def test_valid_string_area_value(self): method test_invalid_string_area_value (line 55) | def test_invalid_string_area_value(self): method test_invalid_latitude_south (line 61) | def test_invalid_latitude_south(self): method test_invalid_latitude_north (line 67) | def test_invalid_latitude_north(self): method test_invalid_longitude_west (line 73) | def test_invalid_longitude_west(self): method test_invalid_longitude_east (line 79) | def test_invalid_longitude_east(self): class TestGenerateHdate (line 86) | class TestGenerateHdate(unittest.TestCase): method test_valid_hdate (line 87) | def test_valid_hdate(self): FILE: weather_dl_v2/cli/app/cli_config.py class CliConfig (line 25) | class CliConfig: method BASE_URI (line 30) | def BASE_URI(self) -> str: method from_dict (line 39) | def from_dict(cls, config: t.Dict): function get_config (line 54) | def get_config(): FILE: weather_dl_v2/cli/app/main.py function ping (line 38) | def ping(): FILE: weather_dl_v2/cli/app/services/download_service.py class DownloadService (line 27) | class DownloadService(abc.ABC): method _list_all_downloads (line 30) | def _list_all_downloads(self): method _list_all_downloads_by_filter (line 34) | def _list_all_downloads_by_filter(self, filter_dict: dict): method _get_download_by_config (line 38) | def _get_download_by_config(self, config_name: str): method _show_config_content (line 42) | def _show_config_content(self, config_name: str): method _add_new_download (line 46) | def _add_new_download( method _remove_download (line 52) | def _remove_download(self, config_name: str): method _refetch_config_partitions (line 56) | def _refetch_config_partitions(self, config_name: str, licenses: t.Lis... class DownloadServiceNetwork (line 60) | class DownloadServiceNetwork(DownloadService): method __init__ (line 62) | def __init__(self): method _list_all_downloads (line 65) | def _list_all_downloads(self): method _list_all_downloads_by_filter (line 70) | def _list_all_downloads_by_filter(self, filter_dict: dict): method _get_download_by_config (line 77) | def _get_download_by_config(self, config_name: str): method _show_config_content (line 83) | def _show_config_content(self, config_name: str): method _add_new_download (line 89) | def _add_new_download( method _remove_download (line 105) | def _remove_download(self, config_name: str): method _refetch_config_partitions (line 110) | def _refetch_config_partitions(self, config_name: str, licenses: t.Lis... class DownloadServiceMock (line 119) | class DownloadServiceMock(DownloadService): function get_download_service (line 123) | def get_download_service(test: bool = False): FILE: weather_dl_v2/cli/app/services/license_service.py class LicenseService (line 26) | class LicenseService(abc.ABC): method _get_all_license (line 29) | def _get_all_license(self): method _get_all_license_by_client_name (line 33) | def _get_all_license_by_client_name(self, client_name: str): method _get_license_by_license_id (line 37) | def _get_license_by_license_id(self, license_id: str): method _add_license (line 41) | def _add_license(self, license_dict: dict): method _remove_license (line 45) | def _remove_license(self, license_id: str): method _update_license (line 49) | def _update_license(self, license_id: str, license_dict: dict): method _redeploy_license_by_license_id (line 53) | def _redeploy_license_by_license_id(self, license_id: str): method _redeploy_licenses_by_client (line 57) | def _redeploy_licenses_by_client(self, client_name: str): class LicenseServiceNetwork (line 61) | class LicenseServiceNetwork(LicenseService): method __init__ (line 63) | def __init__(self): method _get_all_license (line 66) | def _get_all_license(self): method _get_all_license_by_client_name (line 71) | def _get_all_license_by_client_name(self, client_name: str): method _get_license_by_license_id (line 78) | def _get_license_by_license_id(self, license_id: str): method _add_license (line 84) | def _add_license(self, license_dict: dict): method _remove_license (line 91) | def _remove_license(self, license_id: str): method _update_license (line 97) | def _update_license(self, license_id: str, license_dict: dict): method _redeploy_license_by_license_id (line 104) | def _redeploy_license_by_license_id(self, license_id: str): method _redeploy_licenses_by_client (line 111) | def _redeploy_licenses_by_client(self, client_name: str): class LicenseServiceMock (line 119) | class LicenseServiceMock(LicenseService): function get_license_service (line 123) | def get_license_service(test: bool = False): FILE: weather_dl_v2/cli/app/services/network_service.py class NetworkService (line 26) | class NetworkService: method parse_response (line 28) | def parse_response(self, response: requests.Response): method get (line 43) | def get(self, uri, header, query=None, payload=None): method post (line 53) | def post(self, uri, header, query=None, payload=None, file=None): method put (line 65) | def put(self, uri, header, query=None, payload=None, file=None): method delete (line 77) | def delete(self, uri, header, query=None): method patch (line 87) | def patch(self, uri, header, query=None): FILE: weather_dl_v2/cli/app/services/queue_service.py class QueueService (line 28) | class QueueService(abc.ABC): method _get_all_license_queues (line 31) | def _get_all_license_queues(self): method _get_license_queue_by_client_name (line 35) | def _get_license_queue_by_client_name(self, client_name: str): method _get_queue_by_license (line 39) | def _get_queue_by_license(self, license_id: str): method _edit_license_queue (line 43) | def _edit_license_queue(self, license_id: str, priority_list: t.List[s... method _edit_config_absolute_priority (line 47) | def _edit_config_absolute_priority( method _save_queue_to_file (line 53) | def _save_queue_to_file(self, license_id: str, dir: str): class QueueServiceNetwork (line 57) | class QueueServiceNetwork(QueueService): method __init__ (line 59) | def __init__(self): method _get_all_license_queues (line 62) | def _get_all_license_queues(self): method _get_license_queue_by_client_name (line 67) | def _get_license_queue_by_client_name(self, client_name: str): method _get_queue_by_license (line 74) | def _get_queue_by_license(self, license_id: str): method _edit_license_queue (line 79) | def _edit_license_queue(self, license_id: str, priority_list: t.List[s... method _edit_config_absolute_priority (line 86) | def _edit_config_absolute_priority( method _save_queue_to_file (line 95) | def _save_queue_to_file(self, license_id: str, dir_path: str) -> str: class QueueServiceMock (line 116) | class QueueServiceMock(QueueService): function get_queue_service (line 120) | def get_queue_service(test: bool = False): FILE: weather_dl_v2/cli/app/subcommands/config.py class ConfigValidator (line 29) | class ConfigValidator(Validator): function update_cli (line 34) | def update_cli(): function show_server_ip (line 48) | def show_server_ip(): function update_server_ip (line 53) | def update_server_ip( FILE: weather_dl_v2/cli/app/subcommands/download.py class DowloadFilterValidator (line 27) | class DowloadFilterValidator(Validator): function get_downloads (line 38) | def get_downloads( function submit_download (line 65) | def submit_download( function get_download_by_config (line 92) | def get_download_by_config( function show_config (line 99) | def show_config( function remove_download (line 106) | def remove_download( function refetch_config (line 118) | def refetch_config( FILE: weather_dl_v2/cli/app/subcommands/license.py class LicenseValidator (line 25) | class LicenseValidator(Validator): function get_all_license (line 35) | def get_all_license( function get_license (line 57) | def get_license(license: Annotated[str, typer.Argument(help="License ID.... function add_license (line 62) | def add_license( function remove_license (line 86) | def remove_license( function update_license (line 96) | def update_license( function redeploy_license (line 125) | def redeploy_license( FILE: weather_dl_v2/cli/app/subcommands/queue.py class QueueValidator (line 25) | class QueueValidator(Validator): function get_all_license_queue (line 33) | def get_all_license_queue( function get_license_queue (line 55) | def get_license_queue(license: Annotated[str, typer.Argument(help="Licen... function modify_license_queue (line 64) | def modify_license_queue( FILE: weather_dl_v2/cli/app/utils.py function timeit (line 32) | def timeit(func): function confirm_action (line 43) | def confirm_action(message: str = "Are you sure you want to continue?"): function order_dict_fields (line 47) | def order_dict_fields(dictionary, key_order): function as_table (line 51) | def as_table(response: str, key_order=None): class Loader (line 82) | class Loader: method __init__ (line 84) | def __init__(self, desc="Loading...", end="", timeout=0.1): method start (line 101) | def start(self): method _animate (line 105) | def _animate(self): method __enter__ (line 112) | def __enter__(self): method stop (line 115) | def stop(self): method __exit__ (line 120) | def __exit__(self, exc_type, exc_value, tb): class Validator (line 126) | class Validator(abc.ABC): method validate (line 129) | def validate( method validate_json (line 151) | def validate_json(self, file_path, allow_missing: bool = False): method _validate_keys (line 167) | def _validate_keys(self, data_set: set, valid_set: set, allow_missing:... FILE: weather_dl_v2/downloader_kubernetes/downloader.py function download (line 28) | def download(url: str, path: str) -> None: function main (line 62) | def main( FILE: weather_dl_v2/downloader_kubernetes/downloader_config.py class DownloaderConfig (line 28) | class DownloaderConfig: method from_dict (line 33) | def from_dict(cls, config: t.Dict): function get_config (line 48) | def get_config(): FILE: weather_dl_v2/downloader_kubernetes/manifest.py class ManifestException (line 48) | class ManifestException(Exception): class Stage (line 54) | class Stage(enum.Enum): class Status (line 73) | class Status(enum.Enum): class DownloadStatus (line 91) | class DownloadStatus: method from_dict (line 152) | def from_dict(cls, download_status: t.Dict) -> "DownloadStatus": method to_dict (line 165) | def to_dict(cls, instance) -> t.Dict: class Manifest (line 184) | class Manifest(abc.ABC): method __post_init__ (line 224) | def __post_init__(self): method schedule (line 228) | def schedule( method skip (line 268) | def skip( method _set_for_transaction (line 321) | def _set_for_transaction( method __enter__ (line 337) | def __enter__(self) -> None: method __exit__ (line 340) | def __exit__(self, exc_type, exc_inst, exc_tb) -> None: method transact (line 381) | def transact( method set_stage (line 393) | def set_stage(self, stage: Stage) -> None: method _read (line 415) | def _read(self, location: str) -> DownloadStatus: method _update (line 419) | def _update(self, download_status: DownloadStatus) -> None: class FirestoreManifest (line 423) | class FirestoreManifest(Manifest): method _get_db (line 433) | def _get_db(self) -> firestore.firestore.Client: method _read (line 464) | def _read(self, location: str) -> DownloadStatus: method _update (line 479) | def _update(self, download_status: DownloadStatus) -> None: method root_document_for_store (line 497) | def root_document_for_store(self, store_scheme: str) -> DocumentRefere... FILE: weather_dl_v2/downloader_kubernetes/util.py function _retry_if_valid_input_but_server_or_socket_error_and_timeout_filter (line 40) | def _retry_if_valid_input_but_server_or_socket_error_and_timeout_filter( class _FakeClock (line 53) | class _FakeClock: method sleep (line 55) | def sleep(self, value): function retry_with_exponential_backoff (line 59) | def retry_with_exponential_backoff(fun): function ichunked (line 74) | def ichunked(iterable: t.Iterable, n: int) -> t.Iterator[t.Iterable]: function copy (line 88) | def copy(src: str, dst: str) -> None: function to_json_serializable_type (line 100) | def to_json_serializable_type(value: t.Any) -> t.Any: function fetch_geo_polygon (line 147) | def fetch_geo_polygon(area: t.Union[list, str]) -> str: function get_file_size (line 178) | def get_file_size(path: str) -> float: function get_wait_interval (line 188) | def get_wait_interval(num_retries: int = 0) -> float: function generate_md5_hash (line 195) | def generate_md5_hash(input: str) -> str: function download_with_aria2 (line 200) | def download_with_aria2(url: str, path: str) -> None: function download_with_wget (line 229) | def download_with_wget(url: str, path: str) -> None: FILE: weather_dl_v2/fastapi-server/config_processing/config.py class Config (line 26) | class Config: method from_dict (line 68) | def from_dict(cls, config: t.Dict) -> "Config": function optimize_selection_partition (line 82) | def optimize_selection_partition(selection: t.Dict) -> t.Dict: FILE: weather_dl_v2/fastapi-server/config_processing/manifest.py class ManifestException (line 52) | class ManifestException(Exception): class Stage (line 58) | class Stage(enum.Enum): class Status (line 77) | class Status(enum.Enum): class DownloadStatus (line 95) | class DownloadStatus: method from_dict (line 156) | def from_dict(cls, download_status: t.Dict) -> "DownloadStatus": method to_dict (line 169) | def to_dict(cls, instance) -> t.Dict: class Manifest (line 188) | class Manifest(abc.ABC): method __post_init__ (line 227) | def __post_init__(self): method schedule (line 231) | def schedule( method skip (line 271) | def skip( method _set_for_transaction (line 324) | def _set_for_transaction( method __enter__ (line 340) | def __enter__(self) -> None: method __exit__ (line 343) | def __exit__(self, exc_type, exc_inst, exc_tb) -> None: method transact (line 383) | def transact( method set_stage (line 395) | def set_stage(self, stage: Stage) -> None: method _read (line 428) | def _read(self, location: str) -> DownloadStatus: method _update (line 432) | def _update(self, download_status: DownloadStatus) -> None: class FirestoreManifest (line 436) | class FirestoreManifest(Manifest, Database): method _get_db (line 446) | def _get_db(self) -> firestore.firestore.Client: method _read (line 477) | def _read(self, location: str) -> DownloadStatus: method _update (line 492) | def _update(self, download_status: DownloadStatus) -> None: method root_document_for_store (line 510) | def root_document_for_store(self, store_scheme: str) -> DocumentRefere... FILE: weather_dl_v2/fastapi-server/config_processing/parsers.py function date (line 34) | def date(candidate: str) -> datetime.date: function time (line 73) | def time(candidate: str) -> datetime.time: function day_month_year (line 105) | def day_month_year(candidate: t.Any) -> int: function date_range_converter (line 117) | def date_range_converter(candidate: str) -> str: function parse_literal (line 122) | def parse_literal(candidate: t.Any) -> t.Any: function validate (line 132) | def validate(key: str, value: int) -> None: function typecast (line 140) | def typecast(key: str, value: t.Any) -> t.Any: function _read_config_file (line 155) | def _read_config_file(file: t.IO) -> t.Dict: function parse_config (line 173) | def parse_config(file: t.IO) -> t.Dict: function _splitlines (line 181) | def _splitlines(block: str) -> t.List[str]: function mars_range_value (line 186) | def mars_range_value(token: str, key: str) -> t.Union[datetime.date, int... function mars_increment_value (line 207) | def mars_increment_value(token: str) -> t.Union[int, float]: function parse_mars_syntax (line 220) | def parse_mars_syntax(block: str, key: str) -> t.List[str]: function date_range (line 311) | def date_range( function _parse_lists (line 322) | def _parse_lists(config: dict, section: str = "") -> t.Dict: function _number_of_replacements (line 337) | def _number_of_replacements(s: t.Text): function parse_subsections (line 345) | def parse_subsections(config: t.Dict) -> t.Dict: function require (line 366) | def require( function process_config (line 374) | def process_config(file: t.IO, config_name: str) -> Config: function prepare_target_name (line 504) | def prepare_target_name(config: Config) -> str: function get_subsections (line 514) | def get_subsections(config: Config) -> t.List[t.Tuple[str, t.Dict]]: FILE: weather_dl_v2/fastapi-server/config_processing/partition.py class PartitionConfig (line 32) | class PartitionConfig: method _create_partition_config (line 49) | def _create_partition_config(self, option: t.Tuple) -> Config: method skip_partition (line 79) | def skip_partition(self, config: Config) -> bool: method prepare_partitions (line 99) | def prepare_partitions(self) -> t.Iterator[Config]: method new_downloads_only (line 116) | def new_downloads_only(self, candidate: Config) -> bool: method update_manifest_collection (line 124) | def update_manifest_collection(self, partition: Config) -> Config: FILE: weather_dl_v2/fastapi-server/config_processing/pipeline.py function _do_partitions (line 32) | def _do_partitions(partition_obj: PartitionConfig): function start_processing_config (line 40) | async def start_processing_config(config_file, licenses, force_download,... FILE: weather_dl_v2/fastapi-server/config_processing/stores.py class Store (line 27) | class Store(abc.ABC): method open (line 34) | def open(self, filename: str, mode: str = "r") -> t.IO: method exists (line 38) | def exists(self, filename: str) -> bool: class InMemoryStore (line 42) | class InMemoryStore(Store): method __init__ (line 45) | def __init__(self): method open (line 48) | def open(self, filename: str, mode: str = "r") -> t.IO: method exists (line 57) | def exists(self, filename: str) -> bool: class TempFileStore (line 62) | class TempFileStore(Store): method __init__ (line 65) | def __init__(self, directory: t.Optional[str] = None) -> None: method open (line 71) | def open(self, filename: str, mode: str = "r") -> t.IO: method exists (line 75) | def exists(self, filename: str) -> bool: class LocalFileStore (line 80) | class LocalFileStore(Store): method __init__ (line 83) | def __init__(self, directory: t.Optional[str] = None) -> None: method open (line 89) | def open(self, filename: str, mode: str = "r") -> t.IO: method exists (line 93) | def exists(self, filename: str) -> bool: class FSStore (line 98) | class FSStore(Store): method open (line 101) | def open(self, filename: str, mode: str = "r") -> t.IO: method exists (line 120) | def exists(self, filename: str) -> bool: FILE: weather_dl_v2/fastapi-server/config_processing/util.py function _retry_if_valid_input_but_server_or_socket_error_and_timeout_filter (line 43) | def _retry_if_valid_input_but_server_or_socket_error_and_timeout_filter( class _FakeClock (line 56) | class _FakeClock: method sleep (line 58) | def sleep(self, value): function retry_with_exponential_backoff (line 62) | def retry_with_exponential_backoff(fun): function ichunked (line 77) | def ichunked(iterable: t.Iterable, n: int) -> t.Iterator[t.Iterable]: function copy (line 91) | def copy(src: str, dst: str) -> None: function to_json_serializable_type (line 103) | def to_json_serializable_type(value: t.Any) -> t.Any: function fetch_geo_polygon (line 150) | def fetch_geo_polygon(area: t.Union[list, str]) -> str: function get_file_size (line 181) | def get_file_size(path: str) -> float: function get_wait_interval (line 191) | def get_wait_interval(num_retries: int = 0) -> float: function generate_md5_hash (line 198) | def generate_md5_hash(input: str) -> str: function download_with_aria2 (line 203) | def download_with_aria2(url: str, path: str) -> None: function generate_hdate (line 231) | def generate_hdate(date: str, subtract_year: str) -> str: FILE: weather_dl_v2/fastapi-server/database/download_handler.py function get_download_handler (line 27) | def get_download_handler(): function get_mock_download_handler (line 31) | def get_mock_download_handler(): class DownloadHandler (line 35) | class DownloadHandler(abc.ABC): method _start_download (line 38) | async def _start_download(self, config_name: str, client_name: str) ->... method _stop_download (line 42) | async def _stop_download(self, config_name: str) -> None: method _mark_partitioning_status (line 46) | async def _mark_partitioning_status(self, config_name: str, status: st... method _check_download_exists (line 50) | async def _check_download_exists(self, config_name: str) -> bool: method _get_downloads (line 54) | async def _get_downloads(self, client_name: str) -> list: method _get_download_by_config_name (line 58) | async def _get_download_by_config_name(self, config_name: str): class DownloadHandlerMock (line 62) | class DownloadHandlerMock(DownloadHandler): method __init__ (line 64) | def __init__(self): method _start_download (line 67) | async def _start_download(self, config_name: str, client_name: str) ->... method _stop_download (line 72) | async def _stop_download(self, config_name: str) -> None: method _mark_partitioning_status (line 77) | async def _mark_partitioning_status(self, config_name: str, status: st... method _check_download_exists (line 82) | async def _check_download_exists(self, config_name: str) -> bool: method _get_downloads (line 90) | async def _get_downloads(self, client_name: str) -> list: method _get_download_by_config_name (line 93) | async def _get_download_by_config_name(self, config_name: str): class DownloadHandlerFirestore (line 99) | class DownloadHandlerFirestore(DownloadHandler): method __init__ (line 101) | def __init__(self, db: firestore.firestore.Client): method _start_download (line 105) | async def _start_download(self, config_name: str, client_name: str) ->... method _stop_download (line 116) | async def _stop_download(self, config_name: str) -> None: method _mark_partitioning_status (line 124) | async def _mark_partitioning_status(self, config_name: str, status: st... method _check_download_exists (line 134) | async def _check_download_exists(self, config_name: str) -> bool: method _get_downloads (line 140) | async def _get_downloads(self, client_name: str) -> list: method _get_download_by_config_name (line 153) | async def _get_download_by_config_name(self, config_name: str): FILE: weather_dl_v2/fastapi-server/database/license_handler.py function get_license_handler (line 28) | def get_license_handler(): function get_mock_license_handler (line 32) | def get_mock_license_handler(): class LicenseHandler (line 36) | class LicenseHandler(abc.ABC): method _add_license (line 39) | async def _add_license(self, license_dict: dict) -> str: method _delete_license (line 43) | async def _delete_license(self, license_id: str) -> None: method _check_license_exists (line 47) | async def _check_license_exists(self, license_id: str) -> bool: method _get_license_by_license_id (line 51) | async def _get_license_by_license_id(self, license_id: str) -> dict: method _get_license_by_client_name (line 55) | async def _get_license_by_client_name(self, client_name: str) -> list: method _get_licenses (line 59) | async def _get_licenses(self) -> list: method _update_license (line 63) | async def _update_license(self, license_id: str, license_dict: dict) -... method _get_license_without_deployment (line 67) | async def _get_license_without_deployment(self) -> list: method _mark_license_status (line 71) | async def _mark_license_status(self, license_id: str, status: str) -> ... class LicenseHandlerMock (line 74) | class LicenseHandlerMock(LicenseHandler): method __init__ (line 76) | def __init__(self): method _add_license (line 79) | async def _add_license(self, license_dict: dict) -> str: method _delete_license (line 84) | async def _delete_license(self, license_id: str) -> None: method _update_license (line 89) | async def _update_license(self, license_id: str, license_dict: dict) -... method _check_license_exists (line 94) | async def _check_license_exists(self, license_id: str) -> bool: method _get_license_by_license_id (line 102) | async def _get_license_by_license_id(self, license_id: str) -> dict: method _get_license_by_client_name (line 113) | async def _get_license_by_client_name(self, client_name: str) -> list: method _get_licenses (line 122) | async def _get_licenses(self) -> list: method _get_license_without_deployment (line 131) | async def _get_license_without_deployment(self) -> list: class LicenseHandlerFirestore (line 135) | class LicenseHandlerFirestore(LicenseHandler): method __init__ (line 137) | def __init__(self, db: firestore.firestore.AsyncClient): method _add_license (line 141) | async def _add_license(self, license_dict: dict) -> str: method _delete_license (line 155) | async def _delete_license(self, license_id: str) -> None: method _update_license (line 163) | async def _update_license(self, license_id: str, license_dict: dict) -... method _check_license_exists (line 173) | async def _check_license_exists(self, license_id: str) -> bool: method _get_license_by_license_id (line 179) | async def _get_license_by_license_id(self, license_id: str) -> dict: method _get_license_by_client_name (line 185) | async def _get_license_by_client_name(self, client_name: str) -> list: method _get_licenses (line 193) | async def _get_licenses(self) -> list: method _get_license_without_deployment (line 197) | async def _get_license_without_deployment(self) -> list: method _mark_license_status (line 205) | async def _mark_license_status(self, license_id: str, status: str) -> ... FILE: weather_dl_v2/fastapi-server/database/manifest_handler.py function get_manifest_handler (line 26) | def get_manifest_handler(): function get_mock_manifest_handler (line 30) | def get_mock_manifest_handler(): class ManifestHandler (line 34) | class ManifestHandler(abc.ABC): method _get_download_success_count (line 37) | async def _get_download_success_count(self, config_name: str) -> int: method _get_download_failure_count (line 41) | async def _get_download_failure_count(self, config_name: str) -> int: method _get_download_scheduled_count (line 45) | async def _get_download_scheduled_count(self, config_name: str) -> int: method _get_download_inprogress_count (line 49) | async def _get_download_inprogress_count(self, config_name: str) -> int: method _get_download_total_count (line 53) | async def _get_download_total_count(self, config_name: str) -> int: method _get_non_successfull_downloads (line 57) | async def _get_non_successfull_downloads(self, config_name: str) -> list: method _get_failed_downloads (line 61) | async def _get_failed_downloads(self, config_name: str) -> list: class ManifestHandlerMock (line 65) | class ManifestHandlerMock(ManifestHandler): method _get_download_failure_count (line 67) | async def _get_download_failure_count(self, config_name: str) -> int: method _get_download_inprogress_count (line 70) | async def _get_download_inprogress_count(self, config_name: str) -> int: method _get_download_scheduled_count (line 73) | async def _get_download_scheduled_count(self, config_name: str) -> int: method _get_download_success_count (line 76) | async def _get_download_success_count(self, config_name: str) -> int: method _get_download_total_count (line 79) | async def _get_download_total_count(self, config_name: str) -> int: method _get_non_successfull_downloads (line 82) | async def _get_non_successfull_downloads(self, config_name: str) -> list: method _get_failed_downloads (line 85) | async def _get_failed_downloads(self, config_name: str) -> list: class ManifestHandlerFirestore (line 89) | class ManifestHandlerFirestore(ManifestHandler): method __init__ (line 91) | def __init__(self, db: firestore.firestore.Client): method _get_download_success_count (line 95) | async def _get_download_success_count(self, config_name: str) -> int: method _get_download_failure_count (line 109) | async def _get_download_failure_count(self, config_name: str) -> int: method _get_download_scheduled_count (line 122) | async def _get_download_scheduled_count(self, config_name: str) -> int: method _get_download_inprogress_count (line 135) | async def _get_download_inprogress_count(self, config_name: str) -> int: method _get_download_total_count (line 160) | async def _get_download_total_count(self, config_name: str) -> int: method _get_non_successfull_downloads (line 172) | async def _get_non_successfull_downloads(self, config_name: str) -> list: method _get_failed_downloads (line 195) | async def _get_failed_downloads(self, config_name: str) -> list: FILE: weather_dl_v2/fastapi-server/database/queue_handler.py function get_queue_handler (line 27) | def get_queue_handler(): function get_mock_queue_handler (line 31) | def get_mock_queue_handler(): class QueueHandler (line 35) | class QueueHandler(abc.ABC): method _create_license_queue (line 38) | async def _create_license_queue(self, license_id: str, client_name: st... method _remove_license_queue (line 42) | async def _remove_license_queue(self, license_id: str) -> None: method _get_queues (line 46) | async def _get_queues(self) -> list: method _get_queue_by_license_id (line 50) | async def _get_queue_by_license_id(self, license_id: str) -> dict: method _get_queue_by_client_name (line 54) | async def _get_queue_by_client_name(self, client_name: str) -> list: method _update_license_queue (line 58) | async def _update_license_queue(self, license_id: str, priority_list: ... method _update_queues_on_start_download (line 62) | async def _update_queues_on_start_download( method _update_queues_on_stop_download (line 68) | async def _update_queues_on_stop_download(self, config_name: str) -> N... method _update_config_priority_in_license (line 72) | async def _update_config_priority_in_license( method _update_client_name_in_license_queue (line 78) | async def _update_client_name_in_license_queue( class QueueHandlerMock (line 84) | class QueueHandlerMock(QueueHandler): method __init__ (line 86) | def __init__(self): method _create_license_queue (line 89) | async def _create_license_queue(self, license_id: str, client_name: st... method _remove_license_queue (line 94) | async def _remove_license_queue(self, license_id: str) -> None: method _get_queues (line 99) | async def _get_queues(self) -> list: method _get_queue_by_license_id (line 102) | async def _get_queue_by_license_id(self, license_id: str) -> dict: method _get_queue_by_client_name (line 107) | async def _get_queue_by_client_name(self, client_name: str) -> list: method _update_license_queue (line 110) | async def _update_license_queue(self, license_id: str, priority_list: ... method _update_queues_on_start_download (line 115) | async def _update_queues_on_start_download( method _update_queues_on_stop_download (line 122) | async def _update_queues_on_stop_download(self, config_name: str) -> N... method _update_config_priority_in_license (line 127) | async def _update_config_priority_in_license( method _update_client_name_in_license_queue (line 134) | async def _update_client_name_in_license_queue( class QueueHandlerFirestore (line 142) | class QueueHandlerFirestore(QueueHandler): method __init__ (line 144) | def __init__(self, db: firestore.firestore.Client): method _create_license_queue (line 148) | async def _create_license_queue(self, license_id: str, client_name: st... method _remove_license_queue (line 158) | async def _remove_license_queue(self, license_id: str) -> None: method _get_queues (line 166) | async def _get_queues(self) -> list: method _get_queue_by_license_id (line 170) | async def _get_queue_by_license_id(self, license_id: str) -> dict: method _get_queue_by_client_name (line 176) | async def _get_queue_by_client_name(self, client_name: str) -> list: method _update_license_queue (line 184) | async def _update_license_queue(self, license_id: str, priority_list: ... method _update_queues_on_start_download (line 194) | async def _update_queues_on_start_download( method _update_queues_on_stop_download (line 207) | async def _update_queues_on_stop_download(self, config_name: str) -> N... method _update_config_priority_in_license (line 219) | async def _update_config_priority_in_license( method _update_client_name_in_license_queue (line 240) | async def _update_client_name_in_license_queue( FILE: weather_dl_v2/fastapi-server/database/session.py class Database (line 29) | class Database(abc.ABC): method _get_db (line 32) | def _get_db(self): function get_async_client (line 38) | def get_async_client() -> firestore.AsyncClient: function get_gcs_client (line 66) | def get_gcs_client() -> storage.Client: FILE: weather_dl_v2/fastapi-server/database/storage_handler.py function get_storage_handler (line 30) | def get_storage_handler(): class StorageHandler (line 34) | class StorageHandler(abc.ABC): method _upload_file (line 37) | def _upload_file(self, file_path) -> str: method _open_local (line 41) | def _open_local(self, file_name) -> t.Iterator[str]: class StorageHandlerMock (line 45) | class StorageHandlerMock(StorageHandler): method __init__ (line 47) | def __init__(self) -> None: method _upload_file (line 50) | def _upload_file(self, file_path) -> None: method _open_local (line 53) | def _open_local(self, file_name) -> t.Iterator[str]: class StorageHandlerGCS (line 57) | class StorageHandlerGCS(StorageHandler): method __init__ (line 59) | def __init__(self, client: storage.Client) -> None: method _upload_file (line 63) | def _upload_file(self, file_path) -> str: method _open_local (line 73) | def _open_local(self, file_name) -> t.Iterator[str]: FILE: weather_dl_v2/fastapi-server/license_dep/deployment_creator.py function create_license_deployment (line 25) | def create_license_deployment(license_id: str) -> str: function terminate_license_deployment (line 54) | def terminate_license_deployment(license_id: str) -> None: FILE: weather_dl_v2/fastapi-server/main.py function create_pending_license_deployments (line 33) | async def create_pending_license_deployments(): function lifespan (line 49) | async def lifespan(app: FastAPI): function main (line 69) | async def main(): FILE: weather_dl_v2/fastapi-server/routers/download.py function fetch_config_stats (line 45) | async def fetch_config_stats( function get_fetch_config_stats (line 82) | def get_fetch_config_stats(): function get_fetch_config_stats_mock (line 86) | def get_fetch_config_stats_mock(): function get_upload (line 103) | def get_upload(): function get_upload_mock (line 117) | def get_upload_mock(): function get_reschedule_partitions (line 124) | def get_reschedule_partitions(): function get_reschedule_partitions_mock (line 187) | def get_reschedule_partitions_mock(): function submit_download (line 196) | async def submit_download( class DownloadStatus (line 244) | class DownloadStatus(str, Enum): function show_download_config (line 251) | async def show_download_config( function get_downloads (line 276) | async def get_downloads( function get_download_by_config_name (line 323) | async def get_download_by_config_name( function delete_download (line 348) | async def delete_download( function retry_config (line 369) | async def retry_config( FILE: weather_dl_v2/fastapi-server/routers/license.py class License (line 27) | class License(BaseModel): class LicenseInternal (line 34) | class LicenseInternal(License): function mark_license_active (line 45) | async def mark_license_active(license_id: str, license_handler: LicenseH... function update_license_internal (line 50) | async def update_license_internal( function get_create_deployment (line 66) | def get_create_deployment(): function get_create_deployment_mock (line 74) | def get_create_deployment_mock(): function get_terminate_license_deployment (line 81) | def get_terminate_license_deployment(): function get_terminate_license_deployment_mock (line 85) | def get_terminate_license_deployment_mock(): function get_licenses (line 94) | async def get_licenses( function get_license_by_license_id (line 107) | async def get_license_by_license_id( function update_license (line 119) | async def update_license( function add_license (line 147) | async def add_license( function delete_license (line 189) | async def delete_license( function redeploy_licenses (line 208) | async def redeploy_licenses( FILE: weather_dl_v2/fastapi-server/routers/queues.py function get_all_license_queue (line 35) | async def get_all_license_queue( function get_license_queue (line 48) | async def get_license_queue( function modify_license_queue (line 62) | async def modify_license_queue( function modify_config_priority_in_license (line 93) | async def modify_config_priority_in_license( FILE: weather_dl_v2/fastapi-server/server_config.py class ServerConfig (line 28) | class ServerConfig: method from_dict (line 40) | def from_dict(cls, config: t.Dict): function get_config (line 55) | def get_config(): FILE: weather_dl_v2/fastapi-server/tests/integration/test_download.py function _get_download (line 36) | def _get_download(headers, query, code, expected): function test_get_downloads_basic (line 43) | def test_get_downloads_basic(): function _submit_download (line 60) | def _submit_download(headers, file_path, licenses, code, expected): function test_submit_download_basic (line 77) | def test_submit_download_basic(): function test_submit_download_file_not_uploaded (line 92) | def test_submit_download_file_not_uploaded(): function test_submit_download_file_alreadys_exist (line 104) | def test_submit_download_file_alreadys_exist(): function _get_download_by_config (line 118) | def _get_download_by_config(headers, config_name, code, expected): function test_get_download_by_config_basic (line 125) | def test_get_download_by_config_basic(): function test_get_download_by_config_wrong_config (line 142) | def test_get_download_by_config_wrong_config(): function _delete_download_by_config (line 151) | def _delete_download_by_config(headers, config_name, code, expected): function test_delete_download_by_config_basic (line 157) | def test_delete_download_by_config_basic(): function test_delete_download_by_config_wrong_config (line 169) | def test_delete_download_by_config_wrong_config(): FILE: weather_dl_v2/fastapi-server/tests/integration/test_license.py function _get_license (line 43) | def _get_license(headers, query, code, expected): function test_get_license_basic (line 50) | def test_get_license_basic(): function test_get_license_client_name (line 65) | def test_get_license_client_name(): function _add_license (line 81) | def _add_license(headers, payload, code, expected): function test_add_license_basic (line 95) | def test_add_license_basic(): function _get_license_by_license_id (line 110) | def _get_license_by_license_id(headers, license_id, code, expected): function test_get_license_by_license_id (line 118) | def test_get_license_by_license_id(): function test_get_license_wrong_license (line 133) | def test_get_license_wrong_license(): function _update_license (line 144) | def _update_license(headers, license_id, license, code, expected): function test_update_license_basic (line 155) | def test_update_license_basic(): function test_update_license_wrong_license_id (line 170) | def test_update_license_wrong_license_id(): function _delete_license (line 185) | def _delete_license(headers, license_id, code, expected): function test_delete_license_basic (line 192) | def test_delete_license_basic(): function test_delete_license_wrong_license (line 201) | def test_delete_license_wrong_license(): FILE: weather_dl_v2/fastapi-server/tests/integration/test_queues.py function _get_all_queue (line 32) | def _get_all_queue(headers, query, code, expected): function test_get_all_queues (line 39) | def test_get_all_queues(): function test_get_client_queues (line 48) | def test_get_client_queues(): function _get_queue_by_license (line 58) | def _get_queue_by_license(headers, license_id, code, expected): function test_get_queue_by_license_basic (line 65) | def test_get_queue_by_license_basic(): function test_get_queue_by_license_wrong_license (line 74) | def test_get_queue_by_license_wrong_license(): function _modify_license_queue (line 83) | def _modify_license_queue(headers, license_id, priority_list, code, expe... function test_modify_license_queue_basic (line 90) | def test_modify_license_queue_basic(): function test_modify_license_queue_wrong_license_id (line 100) | def test_modify_license_queue_wrong_license_id(): function _modify_config_priority_in_license (line 110) | def _modify_config_priority_in_license(headers, license_id, query, code,... function test_modify_config_priority_in_license_basic (line 119) | def test_modify_config_priority_in_license_basic(): function test_modify_config_priority_in_license_wrong_license (line 131) | def test_modify_config_priority_in_license_wrong_license(): function test_modify_config_priority_in_license_wrong_config (line 141) | def test_modify_config_priority_in_license_wrong_config(): FILE: weather_dl_v2/license_deployment/clients.py class Client (line 41) | class Client(abc.ABC): method __init__ (line 52) | def __init__(self, dataset: str, level: int = logging.INFO) -> None: method retrieve (line 59) | def retrieve( method num_requests_per_key (line 67) | def num_requests_per_key(cls, dataset: str) -> int: method license_url (line 73) | def license_url(self): class SplitCDSRequest (line 78) | class SplitCDSRequest(): method __init__ (line 80) | def __init__(self, *args, **kwargs): method _download (line 84) | def _download(self, url, path: str, size: int) -> None: method fetch (line 94) | def fetch(self, request: t.Dict, dataset: str) -> t.Dict: method download (line 98) | def download(self, result: cds_api.Result, target: t.Optional[str] = N... class CdsClient (line 109) | class CdsClient(Client): method retrieve (line 131) | def retrieve(self, dataset: str, selection: t.Dict, manifest: Manifest... method license_url (line 153) | def license_url(self): method num_requests_per_key (line 157) | def num_requests_per_key(cls, dataset: str) -> int: class StdoutLogger (line 178) | class StdoutLogger(io.StringIO): method __init__ (line 181) | def __init__(self, logger_: logging.Logger, level: int = logging.INFO): method log (line 187) | def log(self, msg) -> None: method write (line 190) | def write(self, msg): method __enter__ (line 194) | def __enter__(self): method __exit__ (line 198) | def __exit__(self, exc_type, exc_value, traceback): class SplitMARSRequest (line 203) | class SplitMARSRequest(api.APIRequest): method _download (line 207) | def _download(self, url, path: str, size: int) -> None: method fetch (line 213) | def fetch(self, request: t.Dict, dataset: str) -> t.Dict: method download (line 236) | def download(self, result: t.Dict, target: t.Optional[str] = None) -> ... class SplitRequestMixin (line 248) | class SplitRequestMixin: method fetch (line 251) | def fetch(self, req: t.Dict, dataset: t.Optional[str] = None) -> t.Dict: method download (line 254) | def download(self, res: t.Dict, target: str) -> None: class CDSClientExtended (line 258) | class CDSClientExtended(SplitRequestMixin): method __init__ (line 261) | def __init__(self, *args, **kwargs): class MARSECMWFServiceExtended (line 265) | class MARSECMWFServiceExtended(api.ECMWFService, SplitRequestMixin): method __init__ (line 268) | def __init__(self, *args, **kwargs): class PublicECMWFServerExtended (line 281) | class PublicECMWFServerExtended(api.ECMWFDataServer, SplitRequestMixin): method __init__ (line 283) | def __init__(self, *args, dataset="", **kwargs): class MarsClient (line 295) | class MarsClient(Client): method retrieve (line 315) | def retrieve(self, dataset: str, selection: t.Dict, manifest: Manifest... method license_url (line 337) | def license_url(self): method num_requests_per_key (line 341) | def num_requests_per_key(cls, dataset: str) -> int: class ECMWFPublicClient (line 356) | class ECMWFPublicClient(Client): method retrieve (line 359) | def retrieve(self, dataset: str, selection: t.Dict, manifest: Manifest... method num_requests_per_key (line 381) | def num_requests_per_key(cls, dataset: str) -> int: method license_url (line 386) | def license_url(self): class FakeClient (line 392) | class FakeClient(Client): method retrieve (line 395) | def retrieve(self, dataset: str, selection: t.Dict, manifest: Manifest... method license_url (line 406) | def license_url(self): method num_requests_per_key (line 410) | def num_requests_per_key(cls, dataset: str) -> int: FILE: weather_dl_v2/license_deployment/config.py class Config (line 26) | class Config: method from_dict (line 68) | def from_dict(cls, config: t.Dict) -> "Config": function optimize_selection_partition (line 82) | def optimize_selection_partition(selection: t.Dict) -> t.Dict: FILE: weather_dl_v2/license_deployment/database.py class Database (line 31) | class Database(abc.ABC): method _get_db (line 34) | def _get_db(self): class CRUDOperations (line 38) | class CRUDOperations(abc.ABC): method _initialize_license_deployment (line 41) | def _initialize_license_deployment(self, license_id: str) -> dict: method _get_config_from_queue_by_license_id (line 45) | def _get_config_from_queue_by_license_id(self, license_id: str) -> dict: method _remove_config_from_license_queue (line 49) | def _remove_config_from_license_queue( method _empty_license_queue (line 55) | def _empty_license_queue(self, license_id: str) -> None: method _get_partition_from_manifest (line 59) | def _get_partition_from_manifest(self, config_name: str) -> str: method _mark_license_status (line 63) | def _mark_license_status(self, license_id: str, status: str) -> None: class FirestoreClient (line 66) | class FirestoreClient(Database, CRUDOperations): method _get_db (line 68) | def _get_db(self) -> firestore.firestore.Client: method _initialize_license_deployment (line 99) | def _initialize_license_deployment(self, license_id: str) -> dict: method _get_config_from_queue_by_license_id (line 108) | def _get_config_from_queue_by_license_id(self, license_id: str) -> str... method _get_partition_from_manifest (line 121) | def _get_partition_from_manifest(self, config_name: str) -> str | None: method _remove_config_from_license_queue (line 125) | def _remove_config_from_license_queue( method _empty_license_queue (line 138) | def _empty_license_queue(self, license_id: str) -> None: method _mark_license_status (line 149) | def _mark_license_status(self, license_id: str, status: str) -> None: function get_partition_from_manifest (line 164) | def get_partition_from_manifest(transaction, config_name: str) -> str | ... FILE: weather_dl_v2/license_deployment/deployment_config.py class DeploymentConfig (line 28) | class DeploymentConfig: method from_dict (line 37) | def from_dict(cls, config: t.Dict): function get_config (line 52) | def get_config(): FILE: weather_dl_v2/license_deployment/fetch.py function create_job (line 39) | def create_job(request, result): function make_fetch_request (line 56) | def make_fetch_request(request, error_map: ThreadSafeDict): function fetch_request_from_db (line 132) | def fetch_request_from_db(): function main (line 148) | def main(): function boot_up (line 198) | def boot_up(license: str) -> None: FILE: weather_dl_v2/license_deployment/job_creator.py function create_download_job (line 24) | def create_download_job(message): FILE: weather_dl_v2/license_deployment/manifest.py class ManifestException (line 52) | class ManifestException(Exception): class Stage (line 58) | class Stage(enum.Enum): class Status (line 77) | class Status(enum.Enum): class DownloadStatus (line 97) | class DownloadStatus: method from_dict (line 158) | def from_dict(cls, download_status: t.Dict) -> "DownloadStatus": method to_dict (line 171) | def to_dict(cls, instance) -> t.Dict: class Manifest (line 190) | class Manifest(abc.ABC): method __post_init__ (line 230) | def __post_init__(self): method schedule (line 234) | def schedule( method skip (line 274) | def skip( method _set_for_transaction (line 327) | def _set_for_transaction( method __enter__ (line 343) | def __enter__(self) -> None: method __exit__ (line 346) | def __exit__(self, exc_type, exc_inst, exc_tb) -> None: method transact (line 387) | def transact( method set_stage (line 399) | def set_stage(self, stage: Stage) -> None: method _read (line 432) | def _read(self, location: str) -> DownloadStatus: method _update (line 436) | def _update(self, download_status: DownloadStatus) -> None: class FirestoreManifest (line 440) | class FirestoreManifest(Manifest, Database): method _get_db (line 450) | def _get_db(self) -> firestore.firestore.Client: method _read (line 481) | def _read(self, location: str) -> DownloadStatus: method _update (line 496) | def _update(self, download_status: DownloadStatus) -> None: method root_document_for_store (line 516) | def root_document_for_store(self, store_scheme: str) -> DocumentRefere... FILE: weather_dl_v2/license_deployment/util.py function exceptionit (line 45) | def exceptionit(func): function _retry_if_valid_input_but_server_or_socket_error_and_timeout_filter (line 55) | def _retry_if_valid_input_but_server_or_socket_error_and_timeout_filter( class GracefulKiller (line 67) | class GracefulKiller: method __init__ (line 73) | def __init__(self): method exit_gracefully (line 77) | def exit_gracefully(self, signum, frame): class _FakeClock (line 82) | class _FakeClock: method sleep (line 84) | def sleep(self, value): function retry_with_exponential_backoff (line 88) | def retry_with_exponential_backoff(fun): function ichunked (line 103) | def ichunked(iterable: t.Iterable, n: int) -> t.Iterator[t.Iterable]: function copy (line 117) | def copy(src: str, dst: str) -> None: function to_json_serializable_type (line 129) | def to_json_serializable_type(value: t.Any) -> t.Any: function fetch_geo_polygon (line 176) | def fetch_geo_polygon(area: t.Union[list, str]) -> str: function get_file_size (line 207) | def get_file_size(path: str) -> float: function get_wait_interval (line 217) | def get_wait_interval(num_retries: int = 0) -> float: function generate_md5_hash (line 224) | def generate_md5_hash(input: str) -> str: function download_with_aria2 (line 229) | def download_with_aria2(url: str, path: str) -> None: class ThreadSafeDict (line 257) | class ThreadSafeDict: method __init__ (line 261) | def __init__(self) -> None: method __getitem__ (line 268) | def __getitem__(self, key): method __setitem__ (line 275) | def __setitem__(self, key, value): method remove (line 280) | def remove(self, key): method has_key (line 285) | def has_key(self, key): method increment (line 292) | def increment(self, key, delta=1): method decrement (line 298) | def decrement(self, key, delta=1): method find_exponential_delay (line 304) | def find_exponential_delay(self, n: int) -> int: method exponential_time (line 311) | def exponential_time(self, key): FILE: weather_mv/loader_pipeline/__init__.py function cli (line 21) | def cli(extra=[]): FILE: weather_mv/loader_pipeline/bq.py class ToBigQuery (line 60) | class ToBigQuery(ToDataSink): method add_parser_arguments (line 120) | def add_parser_arguments(cls, subparser: argparse.ArgumentParser): method validate_arguments (line 163) | def validate_arguments(cls, known_args: argparse.Namespace, pipeline_a... method generate_parquet (line 197) | def generate_parquet( method __post_init__ (line 240) | def __post_init__(self): method prepare_coordinates (line 311) | def prepare_coordinates(self, uri: str) -> t.Iterator[t.Tuple[str, t.D... method extract_rows (line 321) | def extract_rows(self, uri: str, coordinate: t.Dict) -> t.Iterator[t.D... method to_rows (line 338) | def to_rows(self, coordinate: t.Dict, ds: xr.Dataset, uri: str) -> t.I... method chunks_to_rows (line 388) | def chunks_to_rows(self, _, ds: xr.Dataset) -> t.Iterator[t.Dict]: method expand (line 397) | def expand(self, paths): function map_dtype_to_sql_type (line 438) | def map_dtype_to_sql_type(var_type: np.dtype) -> str: function dataset_to_table_schema (line 449) | def dataset_to_table_schema(ds: xr.Dataset) -> t.List[bigquery.SchemaFie... function to_table_schema (line 460) | def to_table_schema(columns: t.List[t.Tuple[str, str]]) -> t.List[bigque... function timestamp_row (line 477) | def timestamp_row(it: t.Dict) -> window.TimestampedValue: function fetch_geo_point (line 483) | def fetch_geo_point(lat: float, long: float) -> str: function fetch_geo_polygon (line 493) | def fetch_geo_polygon(latitude: float, longitude: float, lat_grid_resolu... function bound_point (line 517) | def bound_point(latitude: float, longitude: float, lat_grid_resolution: ... function get_lat_lon_range (line 542) | def get_lat_lon_range(value: float, lat_lon: str, is_point_out_of_bound:... FILE: weather_mv/loader_pipeline/bq_test.py class SchemaCreationTests (line 44) | class SchemaCreationTests(TestDataBase): method setUp (line 46) | def setUp(self) -> None: method test_schema_generation (line 69) | def test_schema_generation(self): method test_schema_generation__with_schema_normalization (line 85) | def test_schema_generation__with_schema_normalization(self): method test_schema_generation__with_target_columns (line 101) | def test_schema_generation__with_target_columns(self): method test_schema_generation__with_target_columns__with_schema_normalization (line 117) | def test_schema_generation__with_target_columns__with_schema_normaliza... method test_schema_generation__no_targets_specified (line 133) | def test_schema_generation__no_targets_specified(self): method test_schema_generation__no_targets_specified__with_schema_normalization (line 150) | def test_schema_generation__no_targets_specified__with_schema_normaliz... method test_schema_generation__missing_target (line 167) | def test_schema_generation__missing_target(self): method test_schema_generation__missing_target__with_schema_normalization (line 172) | def test_schema_generation__missing_target__with_schema_normalization(... method test_schema_generation__non_index_coords (line 178) | def test_schema_generation__non_index_coords(self): class ExtractRowsTestBase (line 203) | class ExtractRowsTestBase(TestDataBase): method extract (line 205) | def extract(self, data_path, *, variables=None, area=None, open_datase... method assertGeopointEqual (line 226) | def assertGeopointEqual(self, actual: str, expected: str) -> None: method assertRowsEqual (line 231) | def assertRowsEqual(self, actual: t.Dict, expected: t.Dict): class ExtractRowsTest (line 247) | class ExtractRowsTest(ExtractRowsTestBase): method setUp (line 249) | def setUp(self) -> None: method test_01_extract_rows (line 254) | def test_01_extract_rows(self): method test_02_extract_rows__with_subset_variables (line 281) | def test_02_extract_rows__with_subset_variables(self): method test_03_extract_rows__specific_area (line 304) | def test_03_extract_rows__specific_area(self): method test_04_extract_rows__specific_area_float_points (line 328) | def test_04_extract_rows__specific_area_float_points(self): method test_05_extract_rows_raises_error_when_geo_data_parquet_dimensions_mismatch (line 354) | def test_05_extract_rows_raises_error_when_geo_data_parquet_dimensions... method test_06_extract_rows__specify_import_time (line 366) | def test_06_extract_rows__specify_import_time(self): method test_07_extract_rows_single_point (line 394) | def test_07_extract_rows_single_point(self): method test_08_extract_rows_nan (line 419) | def test_08_extract_rows_nan(self): method test_09_extract_rows__with_valid_lat_long_with_point (line 447) | def test_09_extract_rows__with_valid_lat_long_with_point(self): method test_10_extract_rows__with_valid_lat_long_with_polygon (line 466) | def test_10_extract_rows__with_valid_lat_long_with_polygon(self): method test_11_extract_rows__with_invalid_lat_lon (line 490) | def test_11_extract_rows__with_invalid_lat_lon(self): method test_12_extract_rows_zarr (line 497) | def test_12_extract_rows_zarr(self): method test_13_droping_variable_while_opening_zarr (line 525) | def test_13_droping_variable_while_opening_zarr(self): class ExtractRowsTifSupportTest (line 554) | class ExtractRowsTifSupportTest(ExtractRowsTestBase): method setUp (line 556) | def setUp(self) -> None: method test_01_extract_rows_with_end_time (line 561) | def test_01_extract_rows_with_end_time(self): method test_02_extract_rows_without_end_time (line 589) | def test_02_extract_rows_without_end_time(self): class ExtractRowsGribSupportTest (line 616) | class ExtractRowsGribSupportTest(ExtractRowsTestBase): method setUp (line 618) | def setUp(self) -> None: method test_01_extract_rows (line 624) | def test_01_extract_rows(self): method test_02_extract_rows__with_vars__excludes_non_index_coords__without_schema_normalization (line 654) | def test_02_extract_rows__with_vars__excludes_non_index_coords__withou... method test_03_extract_rows__with_vars__includes_coordinates_in_vars__without_schema_normalization (line 677) | def test_03_extract_rows__with_vars__includes_coordinates_in_vars__wit... method test_04_extract_rows__with_vars__excludes_non_index_coords__with_schema_normalization (line 704) | def test_04_extract_rows__with_vars__excludes_non_index_coords__with_s... method test_05_extract_rows__with_vars__includes_coordinates_in_vars__with_schema_normalization (line 729) | def test_05_extract_rows__with_vars__includes_coordinates_in_vars__wit... method test_06_multiple_editions__without_schema_normalization (line 755) | def test_06_multiple_editions__without_schema_normalization(self): method test_07_multiple_editions__with_schema_normalization (line 822) | def test_07_multiple_editions__with_schema_normalization(self): method test_08_multiple_editions__with_vars__includes_coordinates_in_vars__with_schema_normalization (line 891) | def test_08_multiple_editions__with_vars__includes_coordinates_in_vars... class ExtractRowsFromZarrTest (line 925) | class ExtractRowsFromZarrTest(ExtractRowsTestBase): method setUp (line 927) | def setUp(self) -> None: method tearDown (line 931) | def tearDown(self) -> None: method test_01_extracts_rows (line 935) | def test_01_extracts_rows(self): FILE: weather_mv/loader_pipeline/ee.py function is_compute_engine (line 61) | def is_compute_engine() -> bool: function get_creds (line 69) | def get_creds(use_personal_account: bool, service_account: str, private_... function ee_initialize (line 104) | def ee_initialize(use_personal_account: bool = False, class SetupEarthEngine (line 140) | class SetupEarthEngine(RateLimit): method __init__ (line 143) | def __init__(self, method setup (line 161) | def setup(self, project_id): method check_setup (line 169) | def check_setup(self, project_id: t.Optional[str] = None): method process (line 179) | def process(self, *args, **kwargs): function get_ee_safe_name (line 184) | def get_ee_safe_name(uri: str) -> str: class AssetData (line 196) | class AssetData: class ToEarthEngine (line 216) | class ToEarthEngine(ToDataSink): method add_parser_arguments (line 269) | def add_parser_arguments(cls, subparser: argparse.ArgumentParser): method validate_arguments (line 317) | def validate_arguments(cls, known_args: argparse.Namespace, pipeline_a... method expand (line 372) | def expand(self, paths): class FilterFilesTransform (line 402) | class FilterFilesTransform(SetupEarthEngine, KwargsFactoryMixin): method __init__ (line 416) | def __init__(self, method process (line 443) | def process(self, uri: str) -> t.Iterator[str]: class ConvertToAsset (line 465) | class ConvertToAsset(beam.DoFn, KwargsFactoryMixin): method add_to_queue (line 486) | def add_to_queue(self, queue: Queue, item: t.Any): method convert_to_asset (line 495) | def convert_to_asset(self, queue: Queue, uri: str): method process (line 627) | def process(self, uri: str) -> t.Iterator[AssetData]: class IngestIntoEETransform (line 659) | class IngestIntoEETransform(SetupEarthEngine, KwargsFactoryMixin): method __init__ (line 673) | def __init__(self, method get_project_id (line 697) | def get_project_id(self) -> str: method ee_tasks_remaining (line 700) | def ee_tasks_remaining(self) -> int: method wait_for_task_queue (line 705) | def wait_for_task_queue(self) -> None: method start_ingestion (line 720) | def start_ingestion(self, asset_data: AssetData) -> t.Optional[str]: method process (line 823) | def process(self, asset_data: AssetData) -> t.Iterator[t.Tuple[str, fl... FILE: weather_mv/loader_pipeline/ee_test.py class AssetNameCreationTests (line 28) | class AssetNameCreationTests(unittest.TestCase): method test_asset_name_creation (line 30) | def test_asset_name_creation(self): method test_asset_name_creation__with_special_chars (line 38) | def test_asset_name_creation__with_special_chars(self): method test_asset_name_creation__with_missing_filename (line 46) | def test_asset_name_creation__with_missing_filename(self): method test_asset_name_creation__with_only_filename (line 54) | def test_asset_name_creation__with_only_filename(self): class ConvertToAssetTests (line 63) | class ConvertToAssetTests(TestDataBase): method setUp (line 65) | def setUp(self) -> None: method tearDown (line 71) | def tearDown(self): method test_convert_to_image_asset (line 74) | def test_convert_to_image_asset(self): method test_convert_to_image_asset__with_multiple_grib_edition (line 83) | def test_convert_to_image_asset__with_multiple_grib_edition(self): method test_convert_to_table_asset (line 92) | def test_convert_to_table_asset(self): method test_convert_to_table_asset__with_multiple_grib_edition (line 101) | def test_convert_to_table_asset__with_multiple_grib_edition(self): FILE: weather_mv/loader_pipeline/execution_test.py class ExecutionTest (line 20) | class ExecutionTest(unittest.TestCase): method test_run (line 30) | def test_run(self): FILE: weather_mv/loader_pipeline/metrics.py function timeit (line 44) | def timeit(func_name: str, keyed_fn: bool = False): class AddTimer (line 109) | class AddTimer(beam.DoFn, KwargsFactoryMixin): method process (line 116) | def process(self, element) -> t.Iterator[t.Any]: class AddBeamMetrics (line 127) | class AddBeamMetrics(beam.DoFn): method __init__ (line 131) | def __init__(self, asset_start_time_format: str): method process (line 141) | def process(self, element): class CreateTimeSeries (line 192) | class CreateTimeSeries(beam.DoFn): method create_time_series_object (line 199) | def create_time_series_object(self, metric_name: str, metric_value: fl... method process (line 222) | def process(self, element: t.Any): class AddMetrics (line 260) | class AddMetrics(beam.PTransform, KwargsFactoryMixin): method expand (line 269) | def expand(self, pcoll: beam.PCollection): FILE: weather_mv/loader_pipeline/pipeline.py function configure_logger (line 35) | def configure_logger(verbosity: int) -> None: function pattern_to_uris (line 42) | def pattern_to_uris(match_pattern: str, is_zarr: bool = False) -> t.Iter... function pipeline (line 51) | def pipeline(known_args: argparse.Namespace, pipeline_args: t.List[str])... function run (line 92) | def run(argv: t.List[str]) -> t.Tuple[argparse.Namespace, t.List[str]]: FILE: weather_mv/loader_pipeline/pipeline_test.py class CLITests (line 21) | class CLITests(unittest.TestCase): method setUp (line 22) | def setUp(self) -> None: class TestCLI (line 85) | class TestCLI(CLITests): method test_dry_runs_are_allowed (line 87) | def test_dry_runs_are_allowed(self): method test_log_level_arg (line 91) | def test_log_level_arg(self): method test_tif_metadata_for_datetime_raise_error_for_non_tif_file (line 95) | def test_tif_metadata_for_datetime_raise_error_for_non_tif_file(self): method test_tif_metadata_for_datetime_raise_error_if_flag_is_absent (line 100) | def test_tif_metadata_for_datetime_raise_error_if_flag_is_absent(self): method test_area_only_allows_four (line 104) | def test_area_only_allows_four(self): method test_topic_creates_a_streaming_pipeline (line 118) | def test_topic_creates_a_streaming_pipeline(self): method test_subscription_creates_a_streaming_pipeline (line 122) | def test_subscription_creates_a_streaming_pipeline(self): method test_accepts_json_string_for_xarray_open (line 126) | def test_accepts_json_string_for_xarray_open(self): method test_ee_does_not_yet_support_zarr (line 134) | def test_ee_does_not_yet_support_zarr(self): method test_rg_zarr_cant_output_netcdf (line 138) | def test_rg_zarr_cant_output_netcdf(self): method test_rg_happy_path (line 142) | def test_rg_happy_path(self): method test_zarr_kwargs_must_come_with_zarr (line 145) | def test_zarr_kwargs_must_come_with_zarr(self): method test_topic_and_subscription__mutually_exclusive (line 149) | def test_topic_and_subscription__mutually_exclusive(self): method test_geo_data_parquet_path_must_be_parquet (line 153) | def test_geo_data_parquet_path_must_be_parquet(self): class IntegrationTest (line 157) | class IntegrationTest(CLITests): method test_dry_runs_are_allowed (line 158) | def test_dry_runs_are_allowed(self): FILE: weather_mv/loader_pipeline/regrid.py function _clear_metview (line 46) | def _clear_metview(): function _metview_op (line 60) | def _metview_op() -> t.Iterator[None]: class MapChunkAsFieldset (line 71) | class MapChunkAsFieldset(beam.PTransform): method apply (line 81) | def apply(self, key: xbeam.Key, fs: Fieldset) -> t.Tuple[xbeam.Key, Fi... method _apply (line 84) | def _apply(self, key: xbeam.Key, ds: xr.Dataset) -> t.Tuple[xbeam.Key,... method expand (line 109) | def expand(self, pcoll): class RegridChunk (line 114) | class RegridChunk(MapChunkAsFieldset): method template (line 126) | def template(self, source_ds: xr.Dataset) -> xr.Dataset: method apply (line 162) | def apply(self, key: xbeam.Key, fs: Fieldset) -> t.Tuple[xbeam.Key, Fi... class Regrid (line 167) | class Regrid(ToDataSink): method add_parser_arguments (line 191) | def add_parser_arguments(cls, subparser: argparse.ArgumentParser) -> N... method validate_arguments (line 209) | def validate_arguments(cls, known_args: argparse.Namespace, pipeline_o... method target_from (line 222) | def target_from(self, uri: str) -> str: method is_grib_file_corrupt (line 241) | def is_grib_file_corrupt(self, local_grib: str) -> bool: method path_exists (line 250) | def path_exists(self, path: str, force_regrid: bool = False) -> bool: method apply (line 258) | def apply(self, uri: str) -> None: method expand (line 311) | def expand(self, paths): FILE: weather_mv/loader_pipeline/regrid_test.py function make_skin_temperature_dataset (line 34) | def make_skin_temperature_dataset() -> xr.Dataset: function metview_cache_exists (line 49) | def metview_cache_exists() -> bool: class RegridTest (line 54) | class RegridTest(TestDataBase): method setUp (line 58) | def setUp(self) -> None: method tearDown (line 73) | def tearDown(self) -> None: method test_target_name (line 76) | def test_target_name(self): method test_target_name__to_netCDF__changes_ext (line 80) | def test_target_name__to_netCDF__changes_ext(self): method test_apply__creates_a_file (line 85) | def test_apply__creates_a_file(self): method test_apply__works_when_called_twice (line 95) | def test_apply__works_when_called_twice(self): method test_apply__to_netCDF__creates_a_netCDF_file (line 99) | def test_apply__to_netCDF__creates_a_netCDF_file(self): method test_zarr__coarsen (line 114) | def test_zarr__coarsen(self): method test_corrupt_grib_file (line 138) | def test_corrupt_grib_file(self): FILE: weather_mv/loader_pipeline/sinks.py class KwargsFactoryMixin (line 54) | class KwargsFactoryMixin: method from_kwargs (line 57) | def from_kwargs(cls, **kwargs): class ToDataSink (line 67) | class ToDataSink(abc.ABC, beam.PTransform, KwargsFactoryMixin): method add_parser_arguments (line 75) | def add_parser_arguments(cls, subparser: argparse.ArgumentParser) -> N... method validate_arguments (line 80) | def validate_arguments(cls, known_args: argparse.Namespace, pipeline_o... function _make_grib_dataset_inmem (line 84) | def _make_grib_dataset_inmem(grib_ds: xr.Dataset) -> xr.Dataset: function match_datetime (line 96) | def match_datetime(file_name: str, regex_expression: str) -> datetime.da... function _preprocess_tif (line 149) | def _preprocess_tif( function _to_utc_timestring (line 230) | def _to_utc_timestring(np_time: np.datetime64) -> str: function _add_is_normalized_attr (line 236) | def _add_is_normalized_attr(ds: xr.Dataset, value: bool) -> xr.Dataset: function _is_3d_da (line 246) | def _is_3d_da(da): function __normalize_grib_dataset (line 251) | def __normalize_grib_dataset(filename: str, function __open_dataset_file (line 345) | def __open_dataset_file(filename: str, function copy (line 398) | def copy(src: str, dst: str) -> None: function open_local (line 416) | def open_local(uri: str) -> t.Iterator[str]: function open_dataset (line 438) | def open_dataset(uri: str, function get_file_time (line 522) | def get_file_time(element: t.Any) -> int: FILE: weather_mv/loader_pipeline/sinks_test.py class TestDataBase (line 28) | class TestDataBase(unittest.TestCase): method setUp (line 29) | def setUp(self) -> None: function _handle_missing_grib_be (line 33) | def _handle_missing_grib_be(f): function limit_memory (line 48) | def limit_memory(max_memory=30): function write_netcdf (line 62) | def write_netcdf(): class OpenDatasetTest (line 81) | class OpenDatasetTest(TestDataBase): method setUp (line 83) | def setUp(self) -> None: method test_opens_grib_files (line 90) | def test_opens_grib_files(self): method test_accepts_xarray_kwargs (line 98) | def test_accepts_xarray_kwargs(self): method test_opens_tif_files (line 106) | def test_opens_tif_files(self): method test_opens_zarr (line 112) | def test_opens_zarr(self): method test_open_dataset__fits_memory_bounds (line 117) | def test_open_dataset__fits_memory_bounds(self): method test_group_common_hypercubes (line 123) | def test_group_common_hypercubes(self): class DatetimeTest (line 129) | class DatetimeTest(unittest.TestCase): method test_datetime_regex_string (line 131) | def test_datetime_regex_string(self): method test_datetime_regex_string_with_missing_parameters (line 141) | def test_datetime_regex_string_with_missing_parameters(self): method test_datetime_regex_string_with_different_order (line 151) | def test_datetime_regex_string_with_different_order(self): FILE: weather_mv/loader_pipeline/streaming.py class GroupMessagesByFixedWindows (line 34) | class GroupMessagesByFixedWindows(beam.PTransform): method __init__ (line 39) | def __init__(self, window_size: int, num_shards: int = 5): method expand (line 44) | def expand(self, pcoll): class AddTimestamp (line 58) | class AddTimestamp(beam.DoFn): method process (line 63) | def process(self, element, publish_time=beam.DoFn.TimestampParam) -> t... class ParsePaths (line 72) | class ParsePaths(beam.DoFn): method __init__ (line 75) | def __init__(self, uri_pattern: str): method try_parse_message (line 81) | def try_parse_message(cls, message_body: t.Union[str, t.Dict]) -> t.Dict: method to_object_path (line 91) | def to_object_path(self, payload: t.Dict) -> str: method should_skip (line 95) | def should_skip(self, message_body: t.Dict) -> bool: method process (line 102) | def process(self, key_value, window=beam.DoFn.WindowParam) -> t.Iterab... FILE: weather_mv/loader_pipeline/streaming_test.py class ParsePathsTests (line 20) | class ParsePathsTests(unittest.TestCase): method setUp (line 21) | def setUp(self) -> None: method test_parse_message (line 48) | def test_parse_message(self): method test_parse_message__already_is_dict (line 52) | def test_parse_message__already_is_dict(self): method test_parse_message__bad_json (line 56) | def test_parse_message__bad_json(self): method test_parse_message__round_trip (line 60) | def test_parse_message__round_trip(self): method test_should_skip (line 67) | def test_should_skip(self): method test_should_skip__missing_values (line 71) | def test_should_skip__missing_values(self): method test_should_skip__mismatch_pattern (line 76) | def test_should_skip__mismatch_pattern(self): FILE: weather_mv/loader_pipeline/util.py function make_attrs_ee_compatible (line 53) | def make_attrs_ee_compatible(attrs: t.Dict) -> t.Dict: function to_json_serializable_type (line 83) | def to_json_serializable_type(value: t.Any) -> t.Any: function _check_for_coords_vars (line 133) | def _check_for_coords_vars(ds_data_var: str, target_var: str) -> bool: function get_utc_timestamp (line 138) | def get_utc_timestamp() -> float: function _only_target_coordinate_vars (line 142) | def _only_target_coordinate_vars(ds: xr.Dataset, data_vars: t.List[str])... function _only_target_vars (line 159) | def _only_target_vars(ds: xr.Dataset, data_vars: t.Optional[t.List[str]]... function ichunked (line 194) | def ichunked(iterable: t.Iterable, n: int) -> t.Iterator[t.Iterable]: function get_coordinates (line 207) | def get_coordinates(ds: xr.Dataset, uri: str = '') -> t.Iterator[t.Dict]: function _cleanup_bigquery (line 240) | def _cleanup_bigquery(bigquery_client: bigquery.Client, function _cleanup_bucket (line 252) | def _cleanup_bucket(storage_client: storage.Client, function validate_region (line 266) | def validate_region(output_table: t.Optional[str] = None, function _shard (line 330) | def _shard(elem, num_shards: int): class Shard (line 334) | class Shard(beam.DoFn): method __init__ (line 336) | def __init__(self, num_shards: int, use_metrics: bool): method process (line 342) | def process(self, element, *args, **kwargs): class RateLimit (line 345) | class RateLimit(beam.PTransform, abc.ABC): method __init__ (line 352) | def __init__(self, method process (line 384) | def process(self, elem: t.Any): method expand (line 398) | def expand(self, pcol: beam.PCollection): class _RateLimitDoFn (line 406) | class _RateLimitDoFn(beam.DoFn): method __init__ (line 409) | def __init__(self, rate_limit_fn: t.Callable, wait_time: datetime.time... method process (line 414) | def process(self, keyed_elem: t.Tuple[t.Any, t.Iterable[t.Any]]): FILE: weather_mv/loader_pipeline/util_test.py class GetCoordinatesTest (line 32) | class GetCoordinatesTest(TestDataBase): method setUp (line 33) | def setUp(self) -> None: method test_gets_indexed_coordinates (line 37) | def test_gets_indexed_coordinates(self): method test_no_duplicate_coordinates (line 44) | def test_no_duplicate_coordinates(self): class IChunksTests (line 52) | class IChunksTests(TestDataBase): method setUp (line 53) | def setUp(self) -> None: method test_even_chunks (line 59) | def test_even_chunks(self): method test_odd_chunks (line 72) | def test_odd_chunks(self): method test_get_coordinates (line 83) | def test_get_coordinates(self): class MakeAttrsEeCompatibleTests (line 111) | class MakeAttrsEeCompatibleTests(TestDataBase): method test_make_attrs_ee_compatible_a1 (line 112) | def test_make_attrs_ee_compatible_a1(self): method test_make_attrs_ee_compatible_a2 (line 172) | def test_make_attrs_ee_compatible_a2(self): class ToJsonSerializableTypeTests (line 195) | class ToJsonSerializableTypeTests(unittest.TestCase): method _convert (line 197) | def _convert(self, value): method test_to_json_serializable_type_none (line 200) | def test_to_json_serializable_type_none(self): method test_to_json_serializable_type_float (line 207) | def test_to_json_serializable_type_float(self): method test_to_json_serializable_type_int (line 218) | def test_to_json_serializable_type_int(self): method test_to_json_serializable_type_set (line 226) | def test_to_json_serializable_type_set(self): method test_to_json_serializable_type_ndarray (line 233) | def test_to_json_serializable_type_ndarray(self): method test_to_json_serializable_type_datetime (line 240) | def test_to_json_serializable_type_datetime(self): FILE: weather_sp/splitter_pipeline/__init__.py function cli (line 18) | def cli(extra=[]): FILE: weather_sp/splitter_pipeline/file_name_utils.py class OutFileInfo (line 28) | class OutFileInfo: method __repr__ (line 45) | def __repr__(self): method unformatted_output_path (line 48) | def unformatted_output_path(self): method split_dims (line 52) | def split_dims(self) -> t.List[str]: method formatted_output_path (line 57) | def formatted_output_path(self, splits: t.Dict[str, str]) -> str: function get_output_file_info (line 62) | def get_output_file_info(filename: str, FILE: weather_sp/splitter_pipeline/file_name_utils_test.py class FileNameUtilsTest (line 20) | class FileNameUtilsTest(unittest.TestCase): method test_get_output_file_info_pattern (line 22) | def test_get_output_file_info_pattern(self): method test_get_output_file_info_dir (line 35) | def test_get_output_file_info_dir(self): method test_get_output_file_info_no_fileending (line 48) | def test_get_output_file_info_no_fileending(self): method test_get_output_file_info_filecontainsdots (line 61) | def test_get_output_file_info_filecontainsdots(self): method test_get_output_file_info_dir_no_formatting (line 73) | def test_get_output_file_info_dir_no_formatting(self): method test_output_pattern_ignores_formatting (line 80) | def test_output_pattern_ignores_formatting(self): method test_split_dims (line 94) | def test_split_dims(self): FILE: weather_sp/splitter_pipeline/file_splitters.py function copy (line 53) | def copy(src: str, dst: str) -> None: class FileSplitter (line 70) | class FileSplitter(abc.ABC): method __init__ (line 73) | def __init__(self, input_path: str, output_info: OutFileInfo, method split_data (line 86) | def split_data(self) -> None: method _copy_to_local_file (line 90) | def _copy_to_local_file(self) -> t.Iterator[t.IO]: method should_skip (line 109) | def should_skip(self): method should_skip_file (line 122) | def should_skip_file(self, output_path: str) -> bool: class GribSplitter (line 137) | class GribSplitter(FileSplitter): method split_data (line 139) | def split_data(self) -> None: method _open_grib_locally (line 187) | def _open_grib_locally(self) -> t.Iterator[t.Iterator[pygrib.gribmessa... class GribSplitterV2 (line 193) | class GribSplitterV2(GribSplitter): method replace_non_numeric_bracket (line 199) | def replace_non_numeric_bracket(self, match: re.Match) -> str: method split_data (line 203) | def split_data(self) -> None: class NetCdfSplitter (line 274) | class NetCdfSplitter(FileSplitter): method split_data (line 278) | def split_data(self) -> None: method _open_dataset_locally (line 314) | def _open_dataset_locally(self) -> t.Iterator[xr.Dataset]: method _write_dataset (line 320) | def _write_dataset(self, dataset: xr.Dataset, split_dims: t.List[str])... method _get_output_for_dataset (line 329) | def _get_output_for_dataset(self, dataset: xr.Dataset, split_dims: t.L... class DrySplitter (line 339) | class DrySplitter(FileSplitter): method split_data (line 341) | def split_data(self) -> None: method _get_keys (line 347) | def _get_keys(self) -> t.Dict[str, str]: function get_splitter (line 351) | def get_splitter(file_path: str, FILE: weather_sp/splitter_pipeline/file_splitters_test.py function data_dir (line 39) | def data_dir(): function grib_splitter (line 48) | def grib_splitter(request): class TestGetSplitter (line 52) | class TestGetSplitter: method test_get_splitter_grib (line 54) | def test_get_splitter_grib(self, data_dir): method test_get_splitter_nc (line 64) | def test_get_splitter_nc(self, data_dir): method test_get_splitter_undetermined_grib (line 75) | def test_get_splitter_undetermined_grib(self, data_dir): method test_get_splitter_dryrun (line 85) | def test_get_splitter_dryrun(self): class TestGribSplitter (line 96) | class TestGribSplitter: method test_get_output_file_path (line 97) | def test_get_output_file_path(self, grib_splitter): method test_split_data_with_filter (line 110) | def test_split_data_with_filter(self, data_dir): method test_split_data (line 146) | def test_split_data(self, data_dir, grib_splitter): method test_skips_existing_split (line 180) | def test_skips_existing_split(self, data_dir, grib_splitter): method test_skips_existing_split_with_filter (line 195) | def test_skips_existing_split_with_filter(self, mock_should_skip_file,... method test_does_not_skip__if_forced (line 208) | def test_does_not_skip__if_forced(self, data_dir, grib_splitter): method test_split__fits_memory_bounds (line 226) | def test_split__fits_memory_bounds(self, data_dir, grib_splitter): class TestNetCdfSplitter (line 240) | class TestNetCdfSplitter: method test_get_output_file_path (line 242) | def test_get_output_file_path(self): method test_split_data (line 252) | def test_split_data(self, data_dir): method test_split_data__not_in_dims_raises (line 272) | def test_split_data__not_in_dims_raises(self, data_dir): method test_split_data__unsupported_dim_raises (line 283) | def test_split_data__unsupported_dim_raises(self, data_dir): method test_skips_existing_split (line 294) | def test_skips_existing_split(self, data_dir): method test_does_not_skip__if_forced (line 307) | def test_does_not_skip__if_forced(self, data_dir): method test_split_data__fits_memory_bounds (line 323) | def test_split_data__fits_memory_bounds(self, data_dir): method test_split_data__is_netcdf4 (line 335) | def test_split_data__is_netcdf4(self, data_dir): class TestDrySplitter (line 354) | class TestDrySplitter: method test_path_with_output_pattern (line 356) | def test_path_with_output_pattern(self): method test_path_with_output_pattern_no_formatting (line 367) | def test_path_with_output_pattern_no_formatting(self): FILE: weather_sp/splitter_pipeline/pipeline.py function configure_logger (line 34) | def configure_logger(verbosity: int) -> None: function split_file (line 41) | def split_file(input_file: str, function _get_base_input_directory (line 68) | def _get_base_input_directory(input_pattern: str) -> str: function get_output_base_name (line 76) | def get_output_base_name(input_path: str, function run (line 88) | def run(argv: t.List[str], save_main_session: bool = True): FILE: weather_sp/splitter_pipeline/pipeline_test.py class PipelineTest (line 24) | class PipelineTest(unittest.TestCase): method test_get_base_input_directory (line 26) | def test_get_base_input_directory(self): method test_get_output_base_name (line 40) | def test_get_output_base_name(self): method test_split_file (line 50) | def test_split_file(self, mock_get_splitter): method test_split_file_with_filter (line 65) | def test_split_file_with_filter(self, mock_get_splitter): FILE: weather_sp/splitter_pipeline/streaming.py class GroupMessagesByFixedWindows (line 34) | class GroupMessagesByFixedWindows(beam.PTransform): method __init__ (line 39) | def __init__(self, window_size: int, num_shards: int = 5): method expand (line 44) | def expand(self, pcoll): class AddTimestamp (line 58) | class AddTimestamp(beam.DoFn): method process (line 63) | def process(self, element, publish_time=beam.DoFn.TimestampParam) -> t... class ParsePaths (line 72) | class ParsePaths(beam.DoFn): method __init__ (line 75) | def __init__(self, uri_pattern: str): method try_parse_message (line 81) | def try_parse_message(cls, message_body: t.Union[str, t.Dict]) -> t.Dict: method to_object_path (line 91) | def to_object_path(self, payload: t.Dict) -> str: method should_skip (line 95) | def should_skip(self, message_body: t.Dict) -> bool: method process (line 102) | def process(self, key_value, window=beam.DoFn.WindowParam) -> t.Iterab... FILE: xql/main.py function parse_args (line 22) | def parse_args() -> Tuple[argparse.Namespace, List[str]]: FILE: xql/src/weather_lm/gemini.py function nl_to_sql_query (line 25) | def nl_to_sql_query(input_statement: str) -> str: function nl_to_weather_data (line 83) | def nl_to_weather_data(input_statement: str): FILE: xql/src/weather_lm/utils.py function get_table_map (line 26) | def get_table_map() -> t.Dict: function get_table_map_prompt (line 39) | def get_table_map_prompt() -> t.Tuple: function get_invocation_steps (line 56) | def get_invocation_steps(prompt: PromptTemplate, model: ChatGoogleGenera... FILE: xql/src/xql/apply.py function parse (line 62) | def parse(a: t.Union[xr.DataArray, str], b: t.Union[xr.DataArray, str]) ... function apply_orderby (line 87) | def apply_orderby(e: exp.Order, df: pd.DataFrame) -> pd.DataFrame: function aggregate_variables (line 114) | def aggregate_variables(agg_funcs: t.List[t.Dict[str, str]], function apply_group_by (line 160) | def apply_group_by(time_fields: t.List[str], ds: xr.Dataset, agg_funcs: ... function apply_aggregation (line 188) | def apply_aggregation(groups: t.Union[xr.Dataset, DatasetGroupBy], fun: ... function get_coords_to_squeeze (line 205) | def get_coords_to_squeeze(fields: t.List[str], ds: xr.Dataset) -> t.List... function get_table (line 225) | def get_table(e: exp.Expression) -> str: function parse_query (line 245) | def parse_query(query: str) -> xr.Dataset: function convert_to_dataframe (line 304) | def convert_to_dataframe(ds: xr.Dataset) -> pd.DataFrame: function filter_records (line 326) | def filter_records(df: pd.DataFrame, query: str) -> pd.DataFrame: function set_dataset_table (line 367) | def set_dataset_table(cmd: str) -> None: function list_key_values (line 389) | def list_key_values(input: t.Dict[str, str]) -> None: function display_help (line 400) | def display_help(cmd: str) -> None: function display_table_dataset_map (line 420) | def display_table_dataset_map(cmd: str) -> None: function run_query (line 439) | def run_query(query: str) -> None: function main (line 454) | def main(query: str): FILE: xql/src/xql/open.py function get_chunking (line 30) | def get_chunking(uri: str, variables: t.List[str]) -> t.Dict: function open_dataset (line 69) | def open_dataset(uri: str) -> t.Tuple[xr.Dataset, bool]: FILE: xql/src/xql/utils.py function timing (line 23) | def timing(f): function connect_dask_cluster (line 35) | def connect_dask_cluster() -> None: FILE: xql/src/xql/where.py function parse (line 24) | def parse(a: t.Union[xr.DataArray, str], b: t.Union[xr.DataArray, str]) ... function get_sop_terms (line 48) | def get_sop_terms(expression: exp.Expression): function check_conditional (line 79) | def check_conditional(expression: exp.Expression) -> bool: function parse_condition (line 85) | def parse_condition(expression: exp.Expression, condition_dict: dict) ->... function is_ascending_order (line 109) | def is_ascending_order(da: xr.DataArray) -> bool: function select_coordinate (line 116) | def select_coordinate(da: xr.DataArray, coordinate: str, operator: str, ... function get_coords_condition (line 140) | def get_coords_condition(coordinate: str, condition: t.Dict[str, str]): function filter_condition_dict (line 158) | def filter_condition_dict(condition_dict: dict, ds: xr.Dataset): function apply_select_condition (line 173) | def apply_select_condition(ds: xr.Dataset, condition_dict: dict) -> xr.D... function postorder (line 209) | def postorder(expression: exp.Expression, condition_dict: dict): function apply_where (line 229) | def apply_where(ds: xr.Dataset, expression: exp.Expression) -> xr.Dataset: