SYMBOL INDEX (288 symbols across 60 files)

FILE: dataverse/api/cli.py
  function main (line 9) | def main():

FILE: dataverse/api/emr.py
  function import_dynamic_etls (line 12) | def import_dynamic_etls():
  function main (line 39) | def main(config, verbose=False):

FILE: dataverse/config/interface.py
  class Config (line 23) | class Config:
    method __new__ (line 31) | def __new__(cls, *args, **kwargs):
    method load (line 35) | def load(cls, config: Union[str, dict, DictConfig, OmegaConf, Path]):
    method save (line 101) | def save(cls, config, path: Union[str, Path]):
    method default (line 123) | def default(cls, emr: bool = False):
    method set_default (line 227) | def set_default(cls, config: OmegaConf, emr: bool = False):

FILE: dataverse/etl/__sample/ducky.py
  function __sample___ducky___make_your_own_etl_processor (line 12) | def __sample___ducky___make_your_own_etl_processor(data: Union[RDD, Data...

FILE: dataverse/etl/__sample/github.py
  function __sample___github___using_decorator (line 15) | def __sample___github___using_decorator(data: Union[RDD, DataFrame], *ar...
  function __sample___github___config (line 23) | def __sample___github___config(data: Union[RDD, DataFrame], config: dict...

FILE: dataverse/etl/cleaning/char.py
  function cleaning___char___normalize_whitespace (line 20) | def cleaning___char___normalize_whitespace(
  function cleaning___char___remove_unprintable (line 53) | def cleaning___char___remove_unprintable(
  function strip_accents (line 88) | def strip_accents(text: str) -> str:
  function cleaning___char___remove_accent (line 98) | def cleaning___char___remove_accent(

FILE: dataverse/etl/cleaning/document.py
  function cleaning___document___split_by_word (line 17) | def cleaning___document___split_by_word(

FILE: dataverse/etl/cleaning/html.py
  function cleaning___html___extract_plain_text (line 18) | def cleaning___html___extract_plain_text(

FILE: dataverse/etl/cleaning/korean.py
  class KoreanType (line 18) | class KoreanType(IntEnum):
  function character_is_korean (line 44) | def character_is_korean(c):
  function decompose (line 53) | def decompose(c):
  function compose (line 71) | def compose(chosung, jungsung, jongsung):
  function cleaning___korean___filter_by_ratio (line 79) | def cleaning___korean___filter_by_ratio(
  function classify_korean_type (line 182) | def classify_korean_type(unicode):
  function reduce_repeated_emotions (line 193) | def reduce_repeated_emotions(text, num_repeats=2):
  function cleaning___korean___reduce_emoticon (line 202) | def cleaning___korean___reduce_emoticon(

FILE: dataverse/etl/cleaning/length.py
  function cleaning___length___char_len_filter (line 17) | def cleaning___length___char_len_filter(
  function cleaning___length___word_len_filter (line 67) | def cleaning___length___word_len_filter(

FILE: dataverse/etl/cleaning/number.py
  function cleaning___number___normalize (line 16) | def cleaning___number___normalize(

FILE: dataverse/etl/cleaning/table.py
  function cleaning___table___merge_col_vertical (line 16) | def cleaning___table___merge_col_vertical(

FILE: dataverse/etl/cleaning/unicode.py
  function cleaning___unicode___remove_punct (line 54) | def cleaning___unicode___remove_punct(
  function cleaning___unicode___replace_punct (line 85) | def cleaning___unicode___replace_punct(
  function cleaning___unicode___normalize (line 116) | def cleaning___unicode___normalize(

FILE: dataverse/etl/data_ingestion/arrow.py
  function find_arrow_paths (line 21) | def find_arrow_paths(directory):
  function get_dir_size (line 34) | def get_dir_size(arrow_paths):
  function arrow_table_to_dict (line 48) | def arrow_table_to_dict(arrow_path):
  function data_ingestion___arrow___hf2raw (line 78) | def data_ingestion___arrow___hf2raw(

FILE: dataverse/etl/data_ingestion/common_crawl.py
  function parse_doc (line 37) | def parse_doc(headers: List[str], doc: List[str]) -> Optional[dict]:
  function group_by_docs (line 85) | def group_by_docs(warc_lines: Iterable[str]) -> Iterable[dict]:
  function _close_when_exhausted (line 112) | def _close_when_exhausted(file) -> Iterable[str]:
  function open_segment_file (line 117) | def open_segment_file(segment: str, verbose: bool = True) -> Iterable[str]:
  function process_segment_file (line 132) | def process_segment_file(segment: str, verbose: bool = True) -> Iterable...
  function find_wet_files (line 138) | def find_wet_files(directory):
  function _tmp (line 150) | def _tmp(prefix: str = None, suffix: str = None, dir: Path = None) -> Path:
  function _yield_from (line 159) | def _yield_from(files: list) -> Iterable[str]:
  function open_read (line 164) | def open_read(filename: ReadableFileLike) -> Iterable[str]:
  function request_get_content (line 211) | def request_get_content(url: str, n_retry: int = 3, verbose: bool = True...
  function open_remote_file (line 248) | def open_remote_file(url: str, cache: Path, verbose: bool = True) -> Ite...
  function cc_wet_paths_url (line 282) | def cc_wet_paths_url(dump_id: str) -> str:
  function segment_url (line 286) | def segment_url(segment: str):
  function cc_segment_urls (line 290) | def cc_segment_urls(dump_id: str, cache_dir: Path, verbose: bool = True)...
  function open_segment_url (line 297) | def open_segment_url(segment: str, cache_dir: Path, verbose: bool = True...
  function process_segment_url (line 306) | def process_segment_url(segment: str, cache_dir: Path, verbose: bool = T...
  function data_ingestion___common_crawl___wet2raw (line 313) | def data_ingestion___common_crawl___wet2raw(
  function data_ingestion___common_crawl___dump2raw (line 357) | def data_ingestion___common_crawl___dump2raw(
  function convert_bytes (line 424) | def convert_bytes(data):
  function data_ingestion___common_crawl___raw2ufl (line 435) | def data_ingestion___common_crawl___raw2ufl(spark, data: RDD, *args, **k...

FILE: dataverse/etl/data_ingestion/csv.py
  function data_ingestion___csv___csv2raw (line 17) | def data_ingestion___csv___csv2raw(

FILE: dataverse/etl/data_ingestion/cultura_x.py
  function data_ingestion___cultura_x___raw2ufl (line 15) | def data_ingestion___cultura_x___raw2ufl(spark, ufl: RDD, *args, **kwargs):

FILE: dataverse/etl/data_ingestion/huggingface.py
  function data_ingestion___huggingface___hf2raw (line 18) | def data_ingestion___huggingface___hf2raw(

FILE: dataverse/etl/data_ingestion/parquet.py
  function data_ingestion___parquet___pq2raw (line 13) | def data_ingestion___parquet___pq2raw(

FILE: dataverse/etl/data_ingestion/red_pajama.py
  function convert2ufl (line 25) | def convert2ufl(row):
  function data_ingestion___red_pajama___parquet2ufl (line 32) | def data_ingestion___red_pajama___parquet2ufl(spark, input_paths, repart...
  function data_ingestion___red_pajama___hf2ufl (line 45) | def data_ingestion___red_pajama___hf2ufl(
  function data_ingestion___red_pajama___hf2raw (line 86) | def data_ingestion___red_pajama___hf2raw(
  function data_ingestion___red_pajama___raw2ufl_templatev1 (line 115) | def data_ingestion___red_pajama___raw2ufl_templatev1(spark, ufl, *args, ...
  function data_ingestion___red_pajama___raw2ufl_templatev2 (line 131) | def data_ingestion___red_pajama___raw2ufl_templatev2(spark, ufl, *args, ...

FILE: dataverse/etl/data_ingestion/slim_pajama.py
  function data_ingestion___slim_pajama___parquet2ufl (line 15) | def data_ingestion___slim_pajama___parquet2ufl(spark, input_paths, repar...
  function data_ingestion___slim_pajama___hf2ufl (line 26) | def data_ingestion___slim_pajama___hf2ufl(

FILE: dataverse/etl/data_ingestion/test.py
  function data_ingestion___test___generate_fake_ufl (line 17) | def data_ingestion___test___generate_fake_ufl(

FILE: dataverse/etl/data_save/huggingface.py
  function data_save___huggingface___ufl2hf_hub (line 22) | def data_save___huggingface___ufl2hf_hub(spark, ufl, hub_path, repartiti...
  function data_save___huggingface___ufl2hf (line 31) | def data_save___huggingface___ufl2hf(
  function data_save___huggingface___ufl2hf_obj (line 67) | def data_save___huggingface___ufl2hf_obj(

FILE: dataverse/etl/data_save/parquet.py
  function data_save___parquet___ufl2parquet (line 18) | def data_save___parquet___ufl2parquet(

FILE: dataverse/etl/deduplication/common_crawl.py
  function filter_lines (line 17) | def filter_lines(row, subset="text"):
  function deduplication___common_crawl___exact_line (line 32) | def deduplication___common_crawl___exact_line(

FILE: dataverse/etl/deduplication/exact.py
  function deduplication___exact___column (line 16) | def deduplication___exact___column(

FILE: dataverse/etl/deduplication/minhash.py
  function generate_edges (line 33) | def generate_edges(nodes: List[int]) -> List[Tuple[int, int]]:
  function get_hash (line 60) | def get_hash(text: str, n_bytes: int=8):
  function get_signatures (line 67) | def get_signatures(
  function optimal_param (line 105) | def optimal_param(
  function process_cluster (line 170) | def process_cluster(cluster: List[Any]) -> List[Any]:
  function deduplication___minhash___lsh_jaccard (line 174) | def deduplication___minhash___lsh_jaccard(

FILE: dataverse/etl/deduplication/polyglot.py
  function shingle_word (line 27) | def shingle_word(text: str, n_gram: int = 15, char_level: bool = False) ...
  function generate_minhash (line 49) | def generate_minhash(shingles: List, num_perm: int = 64, seed: int = 1) ...
  function jaccard_by_hashvalues (line 76) | def jaccard_by_hashvalues(src_hashvalues, tgt_hashvalues) -> float:
  function expand_instances_by_minhash (line 85) | def expand_instances_by_minhash(
  function explore_dedup_instance (line 95) | def explore_dedup_instance(hash_groups, threshold: float = 0.8):
  function deduplication___polyglot___minhash (line 113) | def deduplication___polyglot___minhash(

FILE: dataverse/etl/pii/card.py
  function pii___card___replace_card_number (line 17) | def pii___card___replace_card_number(

FILE: dataverse/etl/pii/nin.py
  function pii___nin___replace_korean_rrn (line 27) | def pii___nin___replace_korean_rrn(

FILE: dataverse/etl/pipeline.py
  class ETLPipeline (line 25) | class ETLPipeline:
    method __init__ (line 45) | def __init__(self):
    method __len__ (line 48) | def __len__(self):
    method status (line 51) | def status(self):
    method search (line 73) | def search(self, category=None, sub_category=None):
    method get (line 103) | def get(self, key):
    method setup_spark_conf (line 107) | def setup_spark_conf(self, config, verbose=False):
    method sample (line 157) | def sample(
    method run (line 222) | def run(
    method run_emr (line 356) | def run_emr(

FILE: dataverse/etl/quality/language.py
  function load_fasttext (line 25) | def load_fasttext(
  function language_predict_fasttext (line 62) | def language_predict_fasttext(row, model, top_k: int = 1, score_rounding...
  function language_predict_fasttext_by_partition (line 73) | def language_predict_fasttext_by_partition(rows, top_k: int = 1, score_r...
  function quality___language___fasttext_filter (line 88) | def quality___language___fasttext_filter(

FILE: dataverse/etl/registry.py
  function auto_register (line 42) | def auto_register(etl_categories=ETL_CATEGORIES):
  class ETLStructure (line 71) | class ETLStructure:
  class ETLRegistry (line 75) | class ETLRegistry:
    method __new__ (line 99) | def __new__(cls):
    method __init__ (line 104) | def __init__(self):
    method __len__ (line 116) | def __len__(self):
    method __repr__ (line 119) | def __repr__(self):
    method __str__ (line 122) | def __str__(self):
    method reset (line 125) | def reset(self):
    method register (line 131) | def register(self, key: str, etl: ETLStructure):
    method _update_status (line 175) | def _update_status(self, key: str):
    method search (line 185) | def search(self, category: str = None, sub_category: str = None):
    method _convert_to_report_format (line 229) | def _convert_to_report_format(
    method get (line 281) | def get(self, key: str) -> ETLStructure:
    method get_all (line 315) | def get_all(self):
  class ETLAutoRegistry (line 325) | class ETLAutoRegistry(abc.ABCMeta, type):
    method __new__ (line 326) | def __new__(cls, name, bases, attrs):
  class BaseETL (line 349) | class BaseETL(ETLStructure, metaclass=ETLAutoRegistry):
    method run (line 366) | def run(self, data: Union[RDD, DataFrame], *args, **kwargs):
    method __call__ (line 372) | def __call__(self, *args, **kwargs):
  function add_self (line 379) | def add_self(func):
  function register_etl (line 393) | def register_etl(func):

FILE: dataverse/etl/utils/log.py
  function utils___log___count (line 15) | def utils___log___count(

FILE: dataverse/etl/utils/sampling.py
  function utils___sampling___random (line 17) | def utils___sampling___random(

FILE: dataverse/etl/utils/statistics.py
  function utils___statistics___korean_nouns (line 16) | def utils___statistics___korean_nouns(

FILE: dataverse/tests/conftest.py
  function set_test_mode_env (line 10) | def set_test_mode_env():

FILE: dataverse/tests/test_cleaning_accent.py
  function helper___test___generate_accent (line 7) | def helper___test___generate_accent(spark, *args, **kwargs):
  function test_cleaning___accent____remove (line 13) | def test_cleaning___accent____remove():

FILE: dataverse/tests/test_cleaning_char.py
  function helper___test___generate_whitespace (line 17) | def helper___test___generate_whitespace(
  function test_cleaning___char___normalize_whitespace (line 34) | def test_cleaning___char___normalize_whitespace():
  function helper___test___generate_unprintable (line 68) | def helper___test___generate_unprintable(
  function test_cleaning___char___remove_unprintable (line 92) | def test_cleaning___char___remove_unprintable():

FILE: dataverse/tests/test_cleaning_document.py
  function fake_data_rdd (line 13) | def fake_data_rdd():
  function test_cleaning___document___split_by_word (line 17) | def test_cleaning___document___split_by_word():

FILE: dataverse/tests/test_cleaning_html.py
  function helper___test___generate_html (line 14) | def helper___test___generate_html(
  function test_cleaning___html___extract_plain_text (line 61) | def test_cleaning___html___extract_plain_text():
  function test_cleaning___html___extract_plain_text_trafilatura (line 86) | def test_cleaning___html___extract_plain_text_trafilatura():

FILE: dataverse/tests/test_cleaning_korean.py
  function helper___test___generate_korean (line 14) | def helper___test___generate_korean(
  function test_cleaning___korean___filter_by_ratio (line 105) | def test_cleaning___korean___filter_by_ratio():
  function test_cleaning___korean___filter_by_ratio_chars (line 139) | def test_cleaning___korean___filter_by_ratio_chars():
  function helper___test___generate_korean_emoticon (line 175) | def helper___test___generate_korean_emoticon(spark, *args, **kwargs):
  function test_cleaning___korean___reduce_emoticon (line 183) | def test_cleaning___korean___reduce_emoticon():

FILE: dataverse/tests/test_cleaning_length.py
  function helper___test___generate_data_for_test_length (line 12) | def helper___test___generate_data_for_test_length(spark, n=10, faker_see...
  function test_cleaning___length___char_len_filter (line 26) | def test_cleaning___length___char_len_filter():
  function test_cleaning___length___word_len_filter (line 93) | def test_cleaning___length___word_len_filter():

FILE: dataverse/tests/test_cleaning_number.py
  function helper___test___generate_number (line 8) | def helper___test___generate_number(spark, *args, **kwargs):
  function test_cleaning___number___normalize (line 21) | def test_cleaning___number___normalize():

FILE: dataverse/tests/test_cleaning_table.py
  function helper___test___generate_table (line 8) | def helper___test___generate_table(spark, *args, **kwargs):
  function test_cleaning___table___merge_col_vertical (line 14) | def test_cleaning___table___merge_col_vertical():

FILE: dataverse/tests/test_cleaning_unicode.py
  function helper___test___generate_unicode_data (line 8) | def helper___test___generate_unicode_data(spark, *args, **kwargs):
  function helper___test___generate_expected_unicode_data (line 21) | def helper___test___generate_expected_unicode_data(spark, type="remove"):
  function test_cleaning___unicode___remove_punct (line 56) | def test_cleaning___unicode___remove_punct():
  function test_cleaning___unicode___replace_punct (line 77) | def test_cleaning___unicode___replace_punct():
  function test_cleaning___unicode___normalize (line 98) | def test_cleaning___unicode___normalize():

FILE: dataverse/tests/test_deduplication_common_crawl.py
  function helper___test___generate_exact_line (line 9) | def helper___test___generate_exact_line(spark, *args, **kwagrs):
  function test_deduplication___common_crawl___exact_line (line 18) | def test_deduplication___common_crawl___exact_line():

FILE: dataverse/tests/test_deduplication_exact.py
  function helper___test___generate_duplicated_data (line 8) | def helper___test___generate_duplicated_data(spark, *args, **kwargs):
  function test_deduplication___exact_column (line 15) | def test_deduplication___exact_column():

FILE: dataverse/tests/test_deduplication_polyglot.py
  function helper___test___create_data_for_polyglot_minhash (line 8) | def helper___test___create_data_for_polyglot_minhash(spark, *args, **kwa...
  function test_deduplication___polyglot___minhash (line 18) | def test_deduplication___polyglot___minhash():

FILE: dataverse/tests/test_pii_card.py
  function helper___test___create_data_for_pii_card (line 12) | def helper___test___create_data_for_pii_card(spark, *args, **kwargs):
  function test_pii___card___replace (line 16) | def test_pii___card___replace():

FILE: dataverse/tests/test_pii_nin.py
  function helper___test___create_data_for_korean_rnn (line 12) | def helper___test___create_data_for_korean_rnn(spark, *args, **kwargs):
  function test_pii___nin___replace_korean_rnns (line 16) | def test_pii___nin___replace_korean_rnns():

FILE: dataverse/utils/analyze/pip.py
  function pip_get_package_path (line 5) | def pip_get_package_path(package_name):

FILE: dataverse/utils/analyze/python.py
  function python_is_script_executable (line 4) | def python_is_script_executable(file_path, verbose=False):

FILE: dataverse/utils/api/aws.py
  function aws_check_credentials (line 56) | def aws_check_credentials(verbose=True):
  class AWSClient (line 72) | class AWSClient:
    method __new__ (line 79) | def __new__(cls):
    method __init__ (line 84) | def __init__(self):
    method __str__ (line 101) | def __str__(self) -> str:
    method __repr__ (line 104) | def __repr__(self) -> str:
  function aws_get_state (line 120) | def aws_get_state():
  function aws_set_state (line 139) | def aws_set_state(state):
  function aws_ec2_instance_at_az (line 150) | def aws_ec2_instance_at_az(az):
  function aws_ec2_instance_info (line 169) | def aws_ec2_instance_info(instance):
  function aws_ec2_all_instance_info (line 179) | def aws_ec2_all_instance_info():
  function aws_ec2_get_price (line 204) | def aws_ec2_get_price(instance_type):
  function aws_ssm_run_commands (line 217) | def aws_ssm_run_commands(instance_ids, commands, verbose=True, return_ou...
  class EMRManager (line 282) | class EMRManager:
    method launch (line 286) | def launch(self, config):
    method _role_setup (line 326) | def _role_setup(self, config):
    method _instance_profile_setup (line 405) | def _instance_profile_setup(self, config):
    method _vpc_setup (line 423) | def _vpc_setup(self, config):
    method _set_default_instance (line 503) | def _set_default_instance(
    method _emr_cluster_create (line 561) | def _emr_cluster_create(self, config):
    method run (line 660) | def run(self, config, verbose=False):
    method _setup (line 691) | def _setup(self, config, verbose=False):
    method _get_working_dir (line 724) | def _get_working_dir(self, config):
    method _upload_config (line 750) | def _upload_config(self, config):
    method _upload_source_code (line 759) | def _upload_source_code(self, config):
    method _upload_dependencies (line 783) | def _upload_dependencies(self, config, package_name="dataverse"):
    method _upload_dynamic_etl_files (line 805) | def _upload_dynamic_etl_files(self, config):
    method _move_s3_to_ec2 (line 857) | def _move_s3_to_ec2(self, config, verbose=False):
    method _get_pip_package_path (line 880) | def _get_pip_package_path(self, config, verbose=False):
    method _setup_aws (line 900) | def _setup_aws(self, config, verbose=False):
    method _setup_dependencies (line 914) | def _setup_dependencies(self, config, verbose=False):
    method _setup_source_code (line 933) | def _setup_source_code(self, config, verbose=False):
    method wait (line 957) | def wait(self, config, step_id, verbose=True):
    method terminate (line 981) | def terminate(self, config):
    method _clean (line 1011) | def _clean(self):
    method _clean_stopped_emr (line 1020) | def _clean_stopped_emr(self):
    method _clean_unused_vpc (line 1043) | def _clean_unused_vpc(self):
    method _clean_unused_iam_role (line 1067) | def _clean_unused_iam_role(self):
    method _clean_unused_iam_instance_profile (line 1094) | def _clean_unused_iam_instance_profile(self):
    method terminate_by_id (line 1118) | def terminate_by_id(self, emr_id):
  function aws_iam_role_create (line 1137) | def aws_iam_role_create(
  function aws_iam_role_delete (line 1182) | def aws_iam_role_delete(role_name):
  function aws_iam_instance_profile_create (line 1206) | def aws_iam_instance_profile_create(instance_profile_name, role_name):
  function aws_iam_instance_profile_delete (line 1240) | def aws_iam_instance_profile_delete(instance_profile_name):
  function aws_iam_remove_all_instance_profile (line 1259) | def aws_iam_remove_all_instance_profile():
  function aws_vpc_create (line 1272) | def aws_vpc_create(cidr_block=None, tag_name='Dataverse-Temporary-VPC'):
  function aws_vpc_delete (line 1322) | def aws_vpc_delete(vpc_id):
  function aws_subnet_create (line 1390) | def aws_subnet_create(vpc_id, cird_block=None, tag_name='Dataverse-Tempo...
  function aws_subnet_delete (line 1420) | def aws_subnet_delete(vpc_id, subnet_id):
  function aws_subnet_az (line 1435) | def aws_subnet_az(subnet_id):
  function aws_emr_security_group_create (line 1444) | def aws_emr_security_group_create(
  function aws_security_group_delete (line 1495) | def aws_security_group_delete(vpc_id, security_group_id):
  function aws_security_group_remove_dependency (line 1510) | def aws_security_group_remove_dependency(security_group_id):
  function aws_gateway_create (line 1533) | def aws_gateway_create(vpc_id, tag_name='Dataverse-Gateway'):
  function aws_gateway_delete (line 1566) | def aws_gateway_delete(vpc_id, gateway_id):
  function aws_elastic_ip_allocate (line 1585) | def aws_elastic_ip_allocate(vpc_id, tag_name='Dataverse-Elastic-IP'):
  function aws_elastic_ip_release (line 1615) | def aws_elastic_ip_release(vpc_id, elastic_ip_id):
  function aws_nat_gateway_create (line 1637) | def aws_nat_gateway_create(
  function aws_nat_gateway_delete (line 1679) | def aws_nat_gateway_delete(vpc_id, nat_gateway_id):
  function aws_route_table_create (line 1700) | def aws_route_table_create(
  function aws_route_table_delete (line 1743) | def aws_route_table_delete(vpc_id, route_table_id):
  function aws_route_table_asscociate_subnet (line 1757) | def aws_route_table_asscociate_subnet(subnet_id, route_table_id):
  function aws_s3_path_parse (line 1761) | def aws_s3_path_parse(path):
  function aws_s3_create_bucket (line 1774) | def aws_s3_create_bucket(bucket):
  function aws_s3_delete_bucket (line 1787) | def aws_s3_delete_bucket(bucket):
  function aws_s3_read (line 1796) | def aws_s3_read(bucket, key):
  function aws_s3_download (line 1810) | def aws_s3_download(bucket, key, local_path):
  function aws_s3_upload (line 1843) | def aws_s3_upload(bucket, key, local_path):
  function aws_s3_write (line 1867) | def aws_s3_write(bucket, key, obj):
  function aws_s3_delete (line 1879) | def aws_s3_delete(bucket, key):
  function aws_s3_list_buckets (line 1906) | def aws_s3_list_buckets():
  function aws_s3_ls (line 1917) | def aws_s3_ls(query=None):
  function aws_s3_get_object_type (line 1997) | def aws_s3_get_object_type(bucket, key):

FILE: dataverse/utils/format/huggingface.py
  function load_huggingface_dataset (line 9) | def load_huggingface_dataset(name_or_path, split=None, from_disk=False):
  function huggingface2parquet (line 45) | def huggingface2parquet(

FILE: dataverse/utils/format/ufl.py
  function get_uuidv1 (line 8) | def get_uuidv1():
  function get_uuidv4 (line 11) | def get_uuidv4():

FILE: dataverse/utils/setting/system.py
  class SystemSetting (line 20) | class SystemSetting:
    method __new__ (line 50) | def __new__(cls):
    method __init__ (line 55) | def __init__(self):
    method _get_aws_bucket (line 65) | def _get_aws_bucket(self, verbose=True):
    method default_setting (line 104) | def default_setting(self):
    method update_by_env (line 145) | def update_by_env(self):
    method check_naming_convention (line 154) | def check_naming_convention(self, key):
    method get (line 189) | def get(self, key):
    method set (line 196) | def set(self, key, value):
    method __getattr__ (line 203) | def __getattr__(self, key):
    method __setattr__ (line 212) | def __setattr__(self, key, value):
    method __getitem__ (line 222) | def __getitem__(self, key):
    method __setitem__ (line 225) | def __setitem__(self, key, value):
    method delete (line 228) | def delete(self, key):
    method list (line 236) | def list(self):
    method __repr__ (line 242) | def __repr__(self):
    method __str__ (line 245) | def __str__(self):

FILE: dataverse/utils/setting/user.py
  class UserSetting (line 13) | class UserSetting:
    method __new__ (line 42) | def __new__(cls):
    method __init__ (line 47) | def __init__(self):
    method reset (line 64) | def reset(self):
    method sync_file (line 71) | def sync_file(self):
    method sync_class (line 78) | def sync_class(self):
    method load (line 85) | def load(self, path):
    method check_naming_convention (line 99) | def check_naming_convention(self, key):
    method get (line 135) | def get(self, key):
    method set (line 143) | def set(self, key, value):
    method __getattr__ (line 151) | def __getattr__(self, key):
    method __setattr__ (line 161) | def __setattr__(self, key, value):
    method __getitem__ (line 172) | def __getitem__(self, key):
    method __setitem__ (line 175) | def __setitem__(self, key, value):
    method delete (line 178) | def delete(self, key):
    method list (line 187) | def list(self):
    method __repr__ (line 194) | def __repr__(self):
    method __str__ (line 198) | def __str__(self):

FILE: docs/source/conf.py
  function process_signature (line 78) | def process_signature(
  function skip_undoc_members (line 96) | def skip_undoc_members(app, what, name, obj, skip, options):
  function setup (line 102) | def setup(app):

FILE: setup.py
  function get_requirements (line 9) | def get_requirements():
  function get_extras_require (line 15) | def get_extras_require():