[
  {
    "path": "README.md",
    "content": "# Tranco: A Research-Oriented Top Sites Ranking Hardened Against Manipulation\n\n*By Victor Le Pochat, Tom Van Goethem, Samaneh Tajalizadehkhoob, Maciej Korczyński and Wouter Joosen*\n\nThis repository contains the source code driving the generation of the Tranco ranking provided at [https://tranco-list.eu/](https://tranco-list.eu/). This new top websites ranking was proposed in our paper [Tranco: A Research-Oriented Top Sites Ranking Hardened Against Manipulation](https://tranco-list.eu/assets/tranco-ndss19.pdf).\n\n* `combined_lists.py` contains the core code for generating new lists based on a configuration passed to `combined_lists.generate_combined_list`.\n* `shared.py` and `global_config.py` contain several configuration variables; `shared.DEFAULT_TRANCO_CONFIG` gives the configuration of the default (daily updated) Tranco list.\n* `generate_daily_list.py` runs daily to generate the default Tranco list.\n* `job_handler.py` contains either the code for submitting jobs to an `rq` queue for processing, or code to relay requests for list generation to a remote host.\n* `job_server.py` accepts request for list generation on a remote host.\n* `notify_email.py` contains code to notify users when their list has been generated.\n* `generate_domain_parts.py` preprocesses rankings to extract the different components of domains.\n"
  },
  {
    "path": "combined_lists.py",
    "content": "# Imports\nimport csv\nimport datetime\nimport glob\nimport shutil\nimport time\nimport traceback\nimport zipfile\nfrom itertools import islice\nimport os\nimport tempfile\n\n# Imports of configuration variables\nfrom global_config import *\n\n# Constants\nGLOBAL_MAX_RANK = 1000000\nLIST_FILENAME_FORMAT = \"{}.csv\"\nfrom shared import ZIP_FILENAME_FORMAT\n\n# When using AWS services, set up retrieval and storage of lists for S3\nif USE_S3:\n    import boto3\n    s3_resource = boto3.resource('s3', region_name=\"us-east-1\")\n    toplists_archive_bucket = s3_resource.Bucket(name=TOPLISTS_ARCHIVE_S3_BUCKET)\n    from smart_open import smart_open\n\n# List ID generation\nfrom hashids import Hashids\nhsh = Hashids(salt=\"tsr\", min_length=4, alphabet=\"BCDFGHJKLMNPQRSTVWXYZ23456789\")\n\n# Mongo connection for storing configuration of generated lists\nfrom pymongo import MongoClient\nclient = MongoClient(MONGO_URL)\ndb = client[\"tranco\"]\n\ndef count_dict(dct, entry, value=1):\n    \"\"\" Helper function for updating dictionaries \"\"\"\n    if not entry in dct:\n        dct[entry] = 0\n    dct[entry] += value\n\ndef date_list(start_date, end_date):\n    \"\"\" Generate list of dates between start and end date \"\"\"\n    start_date_dt = datetime.datetime.strptime(start_date, \"%Y-%m-%d\")\n    end_date_dt = datetime.datetime.strptime(end_date, \"%Y-%m-%d\")\n    return [(start_date_dt + datetime.timedelta(days=x)) for x in range((end_date_dt - start_date_dt).days + 1)]\n\ndef _db_id_to_list_id(db_id):\n    \"\"\" List number to hash \"\"\"\n    if db_id:\n        return hsh.encode(db_id)\n    else:\n        return None\n\ndef _list_id_to_db_id(list_id):\n    \"\"\" Hash to list number \"\"\"\n    try:\n        return hsh.decode(list_id)[0]\n    except:\n        return None\n\ndef config_to_list_id(config, insert=True, skip_failed=False):\n    \"\"\" List configuration to list hash (either insert new configuration into database, or retrieve ID for existing list with that configuration)\n    :param config: list configuration\n    :param insert: whether to create a new list ID if the given configuration does not exist yet\n    :param skip_failed: skip failed lists\n    :return:\n    \"\"\"\n\n    if skip_failed:\n        query = {**config, \"failed\": {\"$ne\": True}}\n    else:\n        query = config\n    out = db[\"lists\"].find_one(query)\n    if out:\n        db_id = int(out[\"_id\"])\n    else:\n        if insert:\n            db_id = get_next_db_key()\n            insert_config_in_db(config, db_id)\n        else:\n            return None\n    return _db_id_to_list_id(db_id)\n\ndef list_id_to_config(list_id):\n    \"\"\" Retrieve configuration of existing list based on hash \"\"\"\n    db_id = _list_id_to_db_id(list_id)\n    if db_id:\n        return {**db[\"lists\"].find_one({\"_id\": int(db_id)}), \"list_id\": list_id}\n\ndef list_available(list_id):\n    \"\"\" Check if list is available for download \"\"\"\n    db_id = _list_id_to_db_id(list_id)\n    if not db_id:\n        return False\n    doc = db[\"lists\"].find_one({\"_id\": int(db_id)})\n    return doc is not None and doc.get(\"finished\", False) and not doc.get(\"failed\", True)\n\ndef get_next_db_key():\n    \"\"\" Get next key from list configuration database (for a new list) \"\"\"\n    counter_increase = db[\"counter\"].find_one_and_update({\"_id\": \"lists\"}, {'$inc': {'count': 1}})\n    return int(counter_increase[\"count\"])\n\ndef insert_config_in_db(config, db_id):\n    \"\"\" Insert a new configuration into the database, with the given key \"\"\"\n    db[\"lists\"].insert_one({**config, \"_id\": db_id, \"finished\": False,\n                            \"creationDate\": datetime.datetime.now().strftime(\"%Y-%m-%d\"),\n                            \"creationTime\": datetime.datetime.now().isoformat()})\n\ndef get_generated_list_fp(list_id):\n    \"\"\" Get file location of existing list (file-based archive) \"\"\"\n    return os.path.join(NETAPP_STORAGE_PATH, \"generated_lists/{}\".format(LIST_FILENAME_FORMAT.format(list_id)))\n\ndef get_generated_zip_fp(list_id):\n    \"\"\" Get file location of existing zip (file-based archive) \"\"\"\n    return os.path.join(NETAPP_STORAGE_PATH, \"generated_lists_zip/{}\".format(ZIP_FILENAME_FORMAT.format(list_id)))\n\ndef get_generated_list_s3(list_id):\n    \"\"\" Get file location of existing list (AWS S3) \"\"\"\n    return \"s3://{}/{}\".format(TOPLISTS_GENERATED_LIST_S3_BUCKET, LIST_FILENAME_FORMAT.format(list_id))\n\ndef get_generated_zip_s3(list_id):\n    \"\"\" Get file location of existing zip (AWS S3) \"\"\"\n    return \"s3://{}/{}\".format(TOPLISTS_DAILY_LIST_S3_BUCKET, ZIP_FILENAME_FORMAT.format(list_id))\n\ndef get_list_fp_for_day(provider, date, parts=False):\n    \"\"\" Get file location for source list (of one of the providers) \"\"\"\n    date = date.strftime(\"%Y%m%d\")\n    if parts:\n        fp = next(glob.iglob(os.path.join(NETAPP_STORAGE_PATH, \"archive/{}/parts/{}_{}_parts.csv\".format(provider, provider, date))))\n    else:\n        fp = next(glob.iglob(os.path.join(NETAPP_STORAGE_PATH, \"archive/{}/{}_{}.csv\".format(provider, provider, date))))\n    return fp\n\ndef get_s3_key_for_day(provider, date, parts=False):\n    \"\"\" Get S3 key for source list (of one of the providers) \"\"\"\n    date = date.strftime(\"%Y%m%d\")\n    if parts:\n        fp = \"{}/parts/{}_{}_parts.csv\".format(provider, provider, date)\n    else:\n        fp = \"{}/{}_{}.csv\".format(provider, provider, date)\n    return fp\n\ndef get_s3_url_for_day(provider, date, parts=False):\n    \"\"\" Get S3 url for source list (of one of the providers) \"\"\"\n    key = get_s3_key_for_day(provider, date, parts)\n    return \"s3://{}/{}\".format(TOPLISTS_ARCHIVE_S3_BUCKET, key)\n\ndef get_s3_url_for_fp(fp):\n    \"\"\" Get S3 url for source list (of one of the providers) \"\"\"\n    return \"s3://{}/{}\".format(TOPLISTS_ARCHIVE_S3_BUCKET, fp)\n\ndef generate_prefix_items_file(fp, list_prefix):\n    \"\"\" Create list of source list items (up to requested list length) \"\"\"\n    with open(fp, encoding='utf8') as f:\n        if list_prefix:\n            return [r.split(\",\") for r in islice(f.read().splitlines(), list_prefix)]\n        else:\n            return [r.split(\",\") for r in f.read().splitlines()]\n\ndef generate_prefix_items_s3(fp, list_prefix):\n    \"\"\" Create list of source list items (up to requested list length) \"\"\"\n    with smart_open(get_s3_url_for_fp(fp)) as f:\n        if list_prefix:\n            result = [r.decode(\"utf-8\").split(\",\") for r in islice(f.read().splitlines(), list_prefix)]\n        else:\n            result = [r.decode(\"utf-8\").split(\",\") for r in f.read().splitlines()]\n        return result\n\ndef rescale_rank(rank, max_rank_of_input, min_rank_of_output, max_rank_of_output):\n    \"\"\"\n    Rescale a given rank to the min/max range provided\n    This makes sure that shorter lists are not given a higher importance.\n    \"\"\"\n    return min_rank_of_output + (rank - 1)*((max_rank_of_output-min_rank_of_output)/(max_rank_of_input - 1))\n\ndef borda_count_fp(fps, list_prefix):\n    \"\"\" Generate aggregate scores for domains based on Borda count \"\"\"\n    borda_scores = {}\n    for fp in fps:\n        if USE_S3:\n            items = generate_prefix_items_s3(fp, list_prefix)\n        else:\n            items = generate_prefix_items_file(fp, list_prefix)\n        max_rank_of_input = len(items)\n        max_rank_of_output = min(GLOBAL_MAX_RANK, list_prefix if list_prefix else GLOBAL_MAX_RANK)\n        for rank, elem in items:\n            count_dict(borda_scores, elem, max_rank_of_output + 1 - rescale_rank(int(rank), max_rank_of_input, 1, max_rank_of_output))  # necessary to rescale shorter lists (i.e. Quantcast)\n    return borda_scores\n\ndef dowdall_count_fp(fps, list_prefix):\n    \"\"\" Generate aggregate scores for domains based on Dowdall count \"\"\"\n    dowdall_scores = {}\n    for fp in fps:\n        if USE_S3:\n            items = generate_prefix_items_s3(fp, list_prefix)\n        else:\n            items = generate_prefix_items_file(fp, list_prefix)\n        max_rank_of_input = len(items)\n        max_rank_of_output = min(GLOBAL_MAX_RANK, list_prefix if list_prefix else GLOBAL_MAX_RANK)\n        for rank, elem in items:\n            count_dict(dowdall_scores, elem, 1 / rescale_rank(int(rank), max_rank_of_input, 1, max_rank_of_output))  # necessary to rescale shorter lists (i.e. Quantcast)\n    return dowdall_scores\n\ndef filtered_parts_list_file(fp, list_prefix, f_pld=None, f_tlds=None, f_organization=None, f_subdomains=None, maintain_rank=True):\n    \"\"\" Get list of domains that conform to the set filters \"\"\"\n    with open(fp) as f:\n        if list_prefix:\n            parts_input = islice(f, list_prefix)\n        else:\n            parts_input = f\n        output = []\n        organizations_seen = set()\n        new_rank = 1\n        max_rank = 0\n        for line in parts_input:\n            max_rank += 1\n            rank, fqdn, pld, sld, subd, ps, tld, is_pld = line.rstrip().split(\",\")\n            if f_tlds and (tld not in f_tlds):\n                continue\n            if f_subdomains and (subd not in f_subdomains):\n                continue\n            if f_organization:\n                if sld in organizations_seen:\n                    continue\n                else:\n                    organizations_seen.add(sld)\n            if f_pld:\n                if is_pld != \"True\":\n                    continue\n            if maintain_rank:\n                output.append((rank, fqdn))\n            else:\n                output.append((new_rank, fqdn))\n                new_rank += 1\n    return (output, max_rank)\n\ndef filtered_parts_list_s3(fp, list_prefix, f_pld=None, f_tlds=None, f_organization=None, f_subdomains=None, maintain_rank=True):\n    \"\"\" Get list of domains that conform to the set filters \"\"\"\n    with smart_open(get_s3_url_for_fp(fp)) as f:\n        if list_prefix:\n            parts_input = islice(f, list_prefix)\n        else:\n            parts_input = f\n        output = []\n        organizations_seen = set()\n        new_rank = 1\n        max_rank = 0\n        for line in parts_input:\n            max_rank += 1\n            rank, fqdn, pld, sld, subd, ps, tld, is_pld = line.decode(\"utf-8\").rstrip().split(\",\")\n            if f_tlds and (tld not in f_tlds):\n                continue\n            if f_subdomains and (subd not in f_subdomains):\n                continue\n            if f_organization:\n                if sld in organizations_seen:\n                    continue\n                else:\n                    organizations_seen.add(sld)\n            if f_pld:\n                if is_pld != \"True\":\n                    continue\n            if maintain_rank:\n                output.append((rank, fqdn))\n            else:\n                output.append((new_rank, fqdn))\n                new_rank += 1\n    return (output, max_rank)\n\ndef get_filtered_parts_lists(fps, input_prefix, config, maintain_rank=True):\n    \"\"\" Get domains in given source lists that conform to the filters in the configuration \"\"\"\n    for fp in fps:\n        if USE_S3:\n            yield filtered_parts_list_s3(fp, input_prefix,\n                                          config.get(\"filterPLD\", None) == \"on\",\n                                          config.get('filterTLDValue').split(\",\") if config.get(\"filterTLDValue\",\n                                                                                                None) else None,\n                                          config.get(\"filterOrganization\", None) == \"on\",\n                                          config.get('filterSubdomainValue').split(\",\") if config.get(\n                                              \"filterSubdomainValue\", None) else None,\n                                         maintain_rank=maintain_rank\n                                          )\n        else:\n            yield filtered_parts_list_file(fp, input_prefix,\n                                         config.get(\"filterPLD\", None) == \"on\",\n                                         config.get('filterTLDValue').split(\",\") if config.get(\"filterTLDValue\",\n                                                                                               None) else None,\n                                         config.get(\"filterOrganization\", None) == \"on\",\n                                         config.get('filterSubdomainValue').split(\",\") if config.get(\n                                             \"filterSubdomainValue\", None) else None,\n                                           maintain_rank=maintain_rank\n                                         )\n\ndef borda_count_list(fps, input_prefix, config, maintain_rank=True):\n    \"\"\" Generate aggregate scores for list of filtered domains based on Borda count \"\"\"\n    borda_scores = {}\n    for (filtered_lst, max_rank) in get_filtered_parts_lists(fps, input_prefix, config):\n        if maintain_rank:\n            max_rank_of_input = max_rank\n        else:\n            max_rank_of_input = len(filtered_lst)\n        max_rank_of_output = min(GLOBAL_MAX_RANK, input_prefix if input_prefix else GLOBAL_MAX_RANK)\n        for rank, elem in filtered_lst:\n            count_dict(borda_scores, elem, max_rank_of_output + 1 - rescale_rank(int(rank), max_rank_of_input, 1, max_rank_of_output))  # necessary to rescale shorter lists\n    return borda_scores\n\ndef dowdall_count_list(fps, input_prefix, config, maintain_rank=True):\n    \"\"\" Generate aggregate scores for list of filtered domains based on Dowdall count \"\"\"\n    dowdall_scores = {}\n    for (filtered_lst, max_rank) in get_filtered_parts_lists(fps, input_prefix, config):\n        if maintain_rank:\n            max_rank_of_input = max_rank\n        else:\n            max_rank_of_input = len(filtered_lst)\n        max_rank_of_output = min(GLOBAL_MAX_RANK, input_prefix if input_prefix else GLOBAL_MAX_RANK)\n        for rank, elem in filtered_lst:\n            count_dict(dowdall_scores, elem, 1 / rescale_rank(int(rank), max_rank_of_input, 1, max_rank_of_output))  # necessary to rescale shorter lists\n    return dowdall_scores\n\ndef sort_counts(scores):\n    \"\"\" Sort domains based on aggregate scores \"\"\"\n    return sorted(scores.keys(), key=lambda elem: (-scores[elem], elem))\n\ndef filter_list_1(lst, filter_set, list_size=None):\n    \"\"\" Filter list of domains on given set of domains \"\"\"\n    if list_size:\n        result = []\n        for e in lst:\n            if e in filter_set:\n                result.append(e)\n                if len(result) >= list_size:\n                    break\n        return result\n    else:\n        return [e for e in lst if e in filter_set]\n\ndef filter_list_multiple(lst, filter_sets):\n    \"\"\" Filter list of domains on given sets of domains \"\"\"\n    return [e for e in lst if all(e in filter_set for filter_set in filter_sets)]\n\ndef count_presence_in_fps(fps, prefix):\n    \"\"\" Counts of occurrences in given files with domains \"\"\"\n    presence = {}\n    for fp in fps:\n        lst = generate_prefix_items_s3(fp, prefix)\n        for i in lst:\n            count_dict(presence, i, 1)\n\ndef count_presence_in_sets(sets,):\n    \"\"\" Counts of occurrences in given sets \"\"\"\n    presence = {}\n    for st in sets:\n        for i in st:\n            count_dict(presence, i, 1)\n    return presence\n\ndef items_in_any_list(fps, prefix):\n    \"\"\" Find domains that appear in any of the given lists \"\"\"\n    return set.union(*map(set, [[i[1] for i in generate_prefix_items_s3(fp, prefix)] for fp in fps]))\n\ndef generate_filter_minimum_presence(fps, prefix, minimum):\n    \"\"\" An item should appear on all the lists \"\"\"\n    presence = count_presence_in_fps(fps, prefix)\n    return {k for k, v in presence.items() if v >= minimum}\n\ndef generate_filter_minimum_presence_any(groups_of_fps, prefix, minimum):\n    \"\"\" An item should appear in `minimum` groups, where an item may appear in any list in that group \"\"\"\n    items_per_group = [items_in_any_list(group, prefix) for group in groups_of_fps]\n    presence = count_presence_in_sets(items_per_group,)\n    return {k for k, v in presence.items() if v >= minimum}\n\ndef truncate_list(lst, list_size=None):\n    \"\"\" Return only prefix of given list \"\"\"\n    return lst[:list_size] if list_size else lst\n\ndef write_sorted_counts(sorted_items, scores, fp):\n    \"\"\" Write domains and aggregate scores to file \"\"\"\n    with open(fp, 'w', encoding='utf8') as f:\n        csvw = csv.writer(f)\n        for idx, entry in enumerate(sorted_items):\n            csvw.writerow([idx + 1, entry, scores[entry]])\n\ndef write_list_to_file(lst, list_id):\n    \"\"\" Write ranks and domains to file \"\"\"\n    with open(get_generated_list_fp(list_id), 'w', encoding='utf8') as f:\n        csvw = csv.writer(f)\n        for idx, entry in enumerate(lst):\n            csvw.writerow([idx + 1, entry])\n\n\ndef write_zip_to_file(lst, list_id):\n    \"\"\" Write list of (top 1M) domains to zip file \"\"\"\n    with tempfile.SpooledTemporaryFile(mode='w+b') as z:\n        with tempfile.NamedTemporaryFile(mode='w+') as t:\n            csvw = csv.writer(t)\n            for idx, entry in enumerate(lst):\n                csvw.writerow([idx + 1, entry])\n\n            t.seek(0)\n\n            with zipfile.ZipFile(z, 'w') as a:\n                a.write(t.name, arcname=\"top-1m.csv\")\n\n            z.seek(0)\n\n            with open(get_generated_zip_fp(list_id), 'wb') as f:\n                f.write(z.read())\n\n\ndef write_list_to_s3(lst, list_id):\n    \"\"\" Write ranks and domains to file \"\"\"\n    with smart_open(get_generated_list_s3(list_id), 'w', encoding='utf8') as f:\n        csvw = csv.writer(f)\n        for idx, entry in enumerate(lst):\n            csvw.writerow([idx + 1, entry])\n\n\ndef write_zip_to_s3(lst, list_id):\n    \"\"\" Write list of (top 1M) domains to zip file \"\"\"\n    with tempfile.SpooledTemporaryFile(mode='w+b') as z:\n        with tempfile.NamedTemporaryFile(mode='w+') as t:\n            csvw = csv.writer(t)\n            for idx, entry in enumerate(lst):\n                csvw.writerow([idx + 1, entry])\n\n            t.seek(0)\n\n            with zipfile.ZipFile(z, 'w') as a:\n                a.write(t.name, arcname=\"top-1m.csv\")\n\n            z.seek(0)\n\n            with smart_open(get_generated_zip_s3(list_id), 'wb') as f:\n                f.write(z.read())\n\n\ndef copy_daily_list_s3(list_id):\n    \"\"\" Copy the daily list on S3 to the fixed URL \"\"\"\n    zip_key = ZIP_FILENAME_FORMAT.format(list_id)\n    source = {'Bucket': TOPLISTS_DAILY_LIST_S3_BUCKET, 'Key': zip_key}\n    target_bucket = s3_resource.Bucket(TOPLISTS_DAILY_LIST_S3_BUCKET)\n    target_bucket.copy(source, 'top-1m.csv.zip')\n\n\ndef copy_daily_list_file(list_id):\n    \"\"\" Copy the daily list on file-based archive to the fixed URL \"\"\"\n    zip_file = get_generated_zip_fp(list_id)\n    target_file = os.path.join(NETAPP_STORAGE_PATH, \"generated_lists_zip/{}\".format(\"top-1m.csv.zip\"))\n    shutil.copy2(zip_file, target_file)\n\ndef generate_combined_list(config, list_id, test=False):\n    \"\"\" Generate combined list by calculating aggregate scores on (potentially filtered) source lists of ranked domains \"\"\"\n    db_id = _list_id_to_db_id(list_id)\n    try:\n        ### INPUT ###\n\n        # If a filter on parts is selected, the preprocessed parts files should be used.\n        parts_filter = config.get(\"filterPLD\", False) or (config.get(\"filterTLD\", \"false\") != \"false\") or config.get(\"filterOrganization\", False) or config.get('filterSubdomain', False)\n        dates = date_list(config.get(\"startDate\"), config.get(\"endDate\"))\n\n        # Get source files to process\n        fps = []\n        fps_on_date = {date: [] for date in dates}\n        fps_on_provider = {provider: [] for provider in config['providers']}\n        for provider in config['providers']:\n            for date in dates:\n                if USE_S3:\n                    list_fp = get_s3_key_for_day(provider, date, parts_filter)\n                else:\n                    list_fp = get_list_fp_for_day(provider, date, parts_filter)\n                fps.append(list_fp)\n                fps_on_date[date].append(list_fp)\n                fps_on_provider[provider].append(list_fp)\n\n        # Get requested list prefix\n        if \"listPrefix\" in config and config['listPrefix']:\n            if config['listPrefix'] == \"full\":\n                input_prefix = None\n            elif config['listPrefix'] == \"custom\":\n                input_prefix = int(config['listPrefixCustomValue'])\n            else:\n                input_prefix = int(config['listPrefix'])\n        else:\n            input_prefix = None\n\n        # Generate (sorted) aggregate counts (on parts files if necessary)\n        if parts_filter:\n            if config['combinationMethod'] == 'borda':\n                scores = borda_count_list(fps, input_prefix, config)\n            elif config['combinationMethod'] == 'dowdall':\n                scores = dowdall_count_list(fps, input_prefix, config)\n            else:\n                raise Exception(\"Unknown combination method\")\n        else:\n            if config['combinationMethod'] == 'borda':\n                scores = borda_count_fp(fps, input_prefix)\n            elif config['combinationMethod'] == 'dowdall':\n                scores = dowdall_count_fp(fps, input_prefix)\n            else:\n                raise Exception(\"Unknown combination method\")\n        sorted_domains = sort_counts(scores)\n        domains = sorted_domains\n\n        ### FILTERS ###\n\n        filters_to_apply = []\n        if \"inclusionDays\" in config and config[\"inclusionDays\"]:\n            presence_filter = generate_filter_minimum_presence_any([fps_on_date[date] for date in dates], input_prefix, int(config[\"inclusionDaysValue\"]))\n            filters_to_apply.append(presence_filter)\n        if \"inclusionLists\" in config and config[\"inclusionLists\"]:\n            presence_filter = generate_filter_minimum_presence_any([fps_on_provider[provider] for provider in config['providers']], input_prefix, int(config[\"inclusionListsValue\"]))\n            filters_to_apply.append(presence_filter)\n        domains = filter_list_multiple(domains, filters_to_apply)\n\n        ### OUTPUT ###\n\n        if test:\n            return domains\n        else:\n            # Write list to file\n            if USE_S3:\n                write_list_to_s3(domains, list_id)\n            else:\n                write_list_to_file(domains, list_id)\n\n            # If the list is the daily default list, also generate a zip of the top 1M and copy to permanent URL\n            try:\n                if \"isDailyList\" in config and config[\"isDailyList\"] is True:\n                    if USE_S3:\n                        write_zip_to_s3(domains[:1000000], list_id)\n                        copy_daily_list_s3(list_id)\n                    else:\n                        write_zip_to_file(domains[:1000000], list_id)\n                        copy_daily_list_file(list_id)\n            except:\n                print(\"Zip creation failed\")\n                traceback.print_exc()\n\n            # Update generation success in database\n            db[\"lists\"].update_one({\"_id\": db_id}, {\"$set\": {\"finished\": True, \"failed\": False, \"list_id\": list_id}})\n\n        time.sleep(1)\n        # Report success\n        return True\n    except:\n        traceback.print_exc()\n        # Update generation failure in database\n        db[\"lists\"].update_one({\"_id\": db_id}, {\"$set\": {\"finished\": True, \"failed\": True}})\n        # Report failure\n        return False\n\n"
  },
  {
    "path": "generate_daily_list.py",
    "content": "import datetime\nimport sys\n\nfrom redis import Redis\nfrom rq import Queue\n\nimport combined_lists\nfrom shared import DATE_FORMAT_WITH_HYPHEN, DEFAULT_TRANCO_CONFIG\n\n\n\ndef get_date_interval_bounds(start_date, end_date, nb_days, nb_days_from):\n    if start_date:\n        start_date_dt = datetime.datetime.strptime(start_date, DATE_FORMAT_WITH_HYPHEN)\n        return (\n        start_date, (start_date_dt + datetime.timedelta(days=int(nb_days) - 1)).strftime(DATE_FORMAT_WITH_HYPHEN))\n    elif end_date:\n        end_date_dt = datetime.datetime.strptime(end_date, DATE_FORMAT_WITH_HYPHEN)\n        return ((end_date_dt - datetime.timedelta(days=int(nb_days) - 1)).strftime(DATE_FORMAT_WITH_HYPHEN), end_date)\n\n\ndef generate_todays_lists(day):\n    print(\"Generating lists for {}...\".format(day))\n    config = DEFAULT_TRANCO_CONFIG.copy()\n\n    if day == \"yesterday\":\n        date = (datetime.datetime.utcnow() - datetime.timedelta(days=1)).strftime(DATE_FORMAT_WITH_HYPHEN)\n    elif day == \"today\":\n        date = datetime.datetime.utcnow().strftime(DATE_FORMAT_WITH_HYPHEN)\n    else:\n        raise ValueError\n    config[\"startDate\"], config[\"endDate\"] = get_date_interval_bounds(None, date, 30, \"end\")\n    config[\"isDailyList\"] = True\n\n    print(\"Generating list...\")\n    list_id = combined_lists.config_to_list_id(config)\n    print(\"Generating list ID {}...\".format(list_id))\n    if not combined_lists.list_available(list_id):\n        conn = Redis('localhost', 6379)\n        generate_queue = Queue('generate', connection=conn, default_timeout=\"1h\")\n        if list_id not in generate_queue.job_ids:\n            generate_queue.enqueue(combined_lists.generate_combined_list, args=(config, list_id), job_id=str(list_id), timeout=\"1h\")\n            print(\"Submitted job for list ID {}\".format(list_id))\n\n\nif __name__ == '__main__':\n    day = \"yesterday\"\n    if len(sys.argv) > 1:\n        day = sys.argv[1]\n    generate_todays_lists(day)\n"
  },
  {
    "path": "generate_domain_parts.py",
    "content": "import csv\nimport sys\n\nimport tldextract\n\n\ndef generate_parts_list(input_fp, output_fp):\n    print(input_fp)\n    print(output_fp)\n    with open(output_fp, 'w', encoding='UTF-8') as output_file:\n        output = csv.writer(output_file)\n        with open(input_fp, encoding='UTF-8') as input_file:\n            for l in input_file:\n                rank, fqdn = l.rstrip('\\n').split(\",\")\n                ext = tldextract.extract(fqdn)\n                pld = ext.registered_domain\n                is_pld = pld == fqdn\n                ps = ext.suffix\n                tld = fqdn[fqdn.rfind(\".\") + 1:]\n                sld = ext.domain\n                subd = ext.subdomain\n                output.writerow([rank, fqdn, pld, sld, subd, ps, tld, is_pld])\n\nif __name__ == '__main__':\n    input_fp = sys.argv[1]\n    output_fp = \"/\".join(input_fp.split(\"/\")[:-1]) + \"/parts/\" + input_fp.split(\"/\")[-1][:-4] + \"_parts.csv\"\n    generate_parts_list(input_fp, output_fp)"
  },
  {
    "path": "global_config.py",
    "content": "NETAPP_STORAGE_PATH = None  # File-based archive\nMAILGUN_API_KEY = None  # API key for sending email notifications\nTOPLISTS_ARCHIVE_S3_BUCKET = None  # S3 bucket with archived rankings\nTOPLISTS_GENERATED_LIST_S3_BUCKET = None  # S3 bucket with generated lists\nTOPLISTS_DAILY_LIST_S3_BUCKET = None  # S3 bucket with daily default lists\nMONGO_URL = None  # Mongo instance for storing configurations of lists\nUSE_S3 = None  # Boolean indicating whether to use AWS services\nGENERATION_REMOTE = None  # Boolean indicating whether list generation is handled remotely\nGENERATION_REMOTE_ENDPOINT = None  # Endpoint accepting list generation jobs\nJOB_SERVER_PORT = None  # Port of server accepting list generation jobs"
  },
  {
    "path": "job_handler.py",
    "content": "import functools\n\nfrom redis import Redis\nfrom rq import Queue\nfrom rq.registry import StartedJobRegistry\n\nimport combined_lists\nimport notify_email\n\n\nclass JobHandler:\n    \"\"\"\n    Manage list generation run on this machine.\n    \"\"\"\n    def __init__(self, asyncio_loop):\n        self.loop = asyncio_loop\n        self.setup_job_queues()\n\n    def setup_job_queues(self):\n        \"\"\" Setup rq queues for submitting list generation and email notification jobs. \"\"\"\n        self.conn = Redis('localhost', 6379)\n        self.generate_queue = Queue('generate', connection=self.conn, default_timeout=\"1h\")\n        self.email_queue = Queue('notify_email', connection=self.conn)\n\n    async def submit_generate_job(self, config, list_id):\n        \"\"\" Submit a new job for generating a list (with the given config) \"\"\"\n        if list_id not in await self.loop.run_in_executor(None, self.current_jobs):\n            await self.loop.run_in_executor(None, functools.partial(self.generate_queue.enqueue, combined_lists.generate_combined_list, args=(config, list_id), job_id=str(list_id), timeout=\"1h\"))\n            return True\n        else:\n            return False\n\n    async def submit_email_job(self, email_address, list_id, list_size):\n        \"\"\" Submit a new job for sending an email once a list has been generated \"\"\"\n        generate_job = await self.loop.run_in_executor(None, self.generate_queue.fetch_job, list_id)\n        await self.loop.run_in_executor(None, functools.partial(self.email_queue.enqueue, notify_email.send_notification_mailgun_api, email_address, list_id, list_size, depends_on=generate_job))\n        return True\n\n    def current_jobs(self):\n        \"\"\" Track currently active and queued jobs \"\"\"\n        registry = StartedJobRegistry(queue=self.generate_queue)\n        jobs = registry.get_job_ids() + self.current_jobs()\n\n        return jobs\n\n    def jobs_ahead_of_job(self, list_id):\n        \"\"\" Count number of jobs ahead of current job \"\"\"\n        jobs = self.current_jobs()\n        if list_id in jobs:\n            return jobs.index(list_id)\n        else:\n            return 0\n\n    async def get_job_status(self, list_id):\n        \"\"\" Get current status of a job \"\"\"\n        job_success = await self.loop.run_in_executor(None, self.get_job_success, list_id)\n        jobs_ahead = await self.loop.run_in_executor(None, self.jobs_ahead_of_job, list_id)\n        return {\"completed\": job_success is not None, \"jobs_ahead\": jobs_ahead, \"success\": job_success}\n\n    def get_job_success(self, list_id):\n        \"\"\" Get current rq status of a job \"\"\"\n        return self.generate_queue.fetch_job(list_id).result\n\n\nclass JobHandlerRemote:\n    \"\"\"\n    Manage relaying jobs to a remote machine that generates lists.\n    \"\"\"\n    def __init__(self, asyncio_loop, endpoint=None, session=None):\n        \"\"\"\n\n        :param asyncio_loop:\n        :param endpoint: remote location that generates lists\n        :param session: client session for aiohttp\n        \"\"\"\n        if not endpoint or not session:\n            raise ValueError\n        self.endpoint = endpoint\n        self.session = session\n\n    async def submit_generate_job(self, config, list_id):\n        \"\"\" Submit a new job for generating a list (with the given config) \"\"\"\n        async with self.session.post(\"{}/submit_generate\".format(self.endpoint), json={\"config\": config, \"list_id\": list_id}) as response:\n            jsn = await response.json()\n            return jsn[\"success\"]\n\n    async def submit_email_job(self, email_address, list_id, list_size):\n        \"\"\" Submit a new job for sending an email once a list has been generated \"\"\"\n        async with self.session.post(\"{}/submit_email\".format(self.endpoint), json={\"email_address\": email_address, \"list_id\": list_id, \"list_size\": list_size}) as response:\n            jsn = await response.json()\n            return jsn[\"success\"]\n\n    async def get_job_status(self, list_id):\n        \"\"\" Get current status of a job \"\"\"\n        async with self.session.get(\"{}/job_status\".format(self.endpoint), params={\"list_id\": list_id}) as response:\n            jsn = await response.json()\n            return jsn\n\n    async def retrieve_list(self, list_id, slice_size):\n        \"\"\" Retrieve the contents of a remotely generated list \"\"\"\n        async with self.session.get(\"{}/retrieve_list\".format(self.endpoint), json={\"list_id\": list_id, \"slice_size\": slice_size}) as response:\n            while True:\n                chunk = await response.content.read(1024)\n                if not chunk:\n                    break\n                yield chunk\n"
  },
  {
    "path": "job_server.py",
    "content": "import asyncio\nimport aitertools\nfrom aiohttp import web\n\nimport combined_lists\nimport job_handler\nfrom global_config import JOB_SERVER_PORT\n\n\nclass JobServer:\n    \"\"\" Job server for accepting requests for generating a custom Tranco list (hosted on remote machine) \"\"\"\n\n    def __init__(self, loop):\n        self.web_app = None\n        self.server = None\n        self.runner = None\n        self.routes = web.RouteTableDef()\n        self.loop = loop\n        self.job_handler: job_handler.JobHandler = None\n\n    async def submit_generate_job(self, request):\n        \"\"\" Submit a new job for generating a list (with the given config) \"\"\"\n        post_data = await request.json()\n        print(\"Generating \", post_data)\n        result = await self.job_handler.submit_generate_job(post_data[\"config\"], post_data[\"list_id\"])\n        return web.json_response({\"success\": result})\n\n    async def submit_email_job(self, request):\n        \"\"\" Submit a new job for sending an email once a list has been generated \"\"\"\n        post_data = await request.json()\n        result = await self.job_handler.submit_email_job(post_data[\"email_address\"], post_data[\"list_id\"], post_data[\"list_size\"])\n        return web.json_response({\"success\": result})\n\n    async def get_job_status(self, request):\n        \"\"\" Get current status of a job \"\"\"\n        list_id = request.query['list_id']\n        print(\"Getting status for \", list_id)\n        return web.json_response(await self.job_handler.get_job_status(list_id))\n\n    async def retrieve_list(self, request):\n        \"\"\" Retrieve the contents of a remotely generated list \"\"\"\n        post_data = await request.json()\n        list_id = post_data[\"list_id\"]\n        slice_size = post_data[\"slice_size\"]\n        file_path = await self.loop.run_in_executor(None, combined_lists.get_generated_list_fp, list_id)\n\n        async def generator():\n            with open(file_path) as csvf:\n                async for line in aitertools.islice(csvf, slice_size):\n                    yield line.encode(\"utf-8\")\n\n        return web.Response(body=generator(),\n                            content_type=\"text/csv\",\n                            charset=\"utf-8\",\n                            )\n\n    async def initialize_routes(self):\n        self.web_app.add_routes([\n            web.post('/submit_generate', self.submit_generate_job),\n            web.post('/submit_email', self.submit_email_job),\n            web.get('/job_status', self.get_job_status),\n            web.get('/retrieve_list', self.retrieve_list)\n        ])\n\n    async def run(self):\n        self.job_handler = job_handler.JobHandler(self.loop)\n\n        self.web_app = web.Application()\n\n        await self.initialize_routes()\n        self.runner = web.AppRunner(self.web_app)\n        await self.runner.setup()\n        self.server = web.TCPSite(self.runner, '0.0.0.0', JOB_SERVER_PORT)\n        await self.server.start()\n\n\nif __name__ == '__main__':\n    loop = asyncio.get_event_loop()\n    server = JobServer(loop)\n    loop.run_until_complete(server.run())\n    loop.run_forever()"
  },
  {
    "path": "notify_email.py",
    "content": "import smtplib\nfrom email.message import EmailMessage\nimport email.utils\n\nimport requests\nfrom rq import Queue, Connection, get_current_connection\nfrom global_config import MAILGUN_API_KEY\n\ndef send_notification_mailgun_api(email_address, list_id, list_size):\n    with Connection(get_current_connection()):\n        q = Queue('generate')\n        job = q.fetch_job(list_id)\n        success = job.result\n\n    if success:\n        subject = 'The Tranco list: generation succeeded'\n        body = \"Hello,\\n\\nWe have successfully generated your requested Tranco list with ID {}. You may retrieve it at https://tranco-list.eu/list/{}/{}\\n\\nTranco\\nhttps://tranco-list.eu/\".format(list_id, list_id, list_size)\n    else:\n        subject = 'The Tranco list: generation failed'\n        body = \"Hello,\\n\\nUnfortunately, we were currently unable to generate your requested Tranco list with ID {}. Please try again later.\\n\\nTranco\\nhttps://tranco-list.eu/\".format(list_id)\n\n    r = requests.post(\n            \"https://api.eu.mailgun.net/v3/mg.tranco-list.eu/messages\",\n            auth=(\"api\", MAILGUN_API_KEY),\n            data={\"from\": \"Tranco <noreply@mg.tranco-list.eu>\",\n                  \"to\": [email_address],\n                  \"subject\": subject,\n                  \"text\": body})\n    return int(r.status_code) == 200"
  },
  {
    "path": "requirements.txt",
    "content": "boto3\nsmart_open\nhashids\npymongo\nredis\nrq\naiohttp\naitertools"
  },
  {
    "path": "shared.py",
    "content": "DATE_FORMAT_WITH_HYPHEN = \"%Y-%m-%d\"\nDEFAULT_TRANCO_CONFIG = {\"nbDays\": \"30\", \"nbDaysFrom\": \"end\",\n                  \"combinationMethod\": \"dowdall\",  # TODO make choice based on assessment on stability etc.\n                  \"listPrefix\": 'full',\n                  \"includeDomains\": 'all',  # TODO make choice\n                  \"filterPLD\": \"on\",\n                  \"providers\": [\"alexa\", \"umbrella\", \"majestic\", \"quantcast\"]\n        }\nZIP_FILENAME_FORMAT = \"tranco_{}-1m.csv.zip\""
  }
]