[ { "path": ".github/workflows/publish-to-pypi.yml", "content": "name: Publish Python 🐍 distributions πŸ“¦ to PyPI\non: push\n\njobs:\n build-n-publish:\n name: Build and publish Python 🐍 distributions πŸ“¦ to PyPI\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v3\n - name: Set up Python\n uses: actions/setup-python@v4\n with:\n python-version: \"3.10\"\n\n - name: Install poetry\n run: >-\n python3 -m\n pip install\n poetry\n --user\n\n - name: Build distribution πŸ“¦\n run: >-\n python3 -m\n poetry\n build\n\n - name: Publish distribution πŸ“¦ to PyPI\n if: startsWith(github.ref, 'refs/tags')\n uses: pypa/gh-action-pypi-publish@release/v1\n with:\n password: ${{ secrets.PYPI_API_TOKEN }}" }, { "path": ".gitignore", "content": "/venv/\n/.idea\n**/__pycache__/\n**/.pytest_cache/\n/.ipynb_checkpoints/\n**/output/\n**/.DS_Store\n*.pyc\n.env\ndist" }, { "path": ".pre-commit-config.yaml", "content": "repos:\n- repo: https://github.com/psf/black\n rev: 24.2.0\n hooks:\n - id: black\n language_version: python\n args: [--line-length=88, --quiet]" }, { "path": "LICENSE", "content": " DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE\n Version 2, December 2004\n\n Copyright (C) 2004 Sam Hocevar \n\n Everyone is permitted to copy and distribute verbatim or modified\n copies of this license document, and changing it is allowed as long\n as the name is changed.\n\n DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE\n TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION\n\n 0. You just DO WHAT THE FUCK YOU WANT TO.\n" }, { "path": "README.md", "content": "\"3FAD4652-488F-4F6F-A744-4C2AA5855E92\"\n\n**StaffSpy** is a staff fetcher library for LinkedIn.\n\n## Features\n\n- Fetches staff from a company on **LinkedIn**\n- Obtains skills, experiences, certifications & more\n- Fetch individuals users / comments on posts\n- Export all your connections with their contact info\n- Aggregates the employees in a Pandas DataFrame\n\n### Installation\n\n```\npip install -U \"staffspy[browser]\"\n```\n\nOr for latest code from this repo directly\n\n```\npip install \"git+https://github.com/cullenwatson/StaffSpy.git#egg=staffspy[browser]\"\n```\n\n_Python version >= [3.10](https://www.python.org/downloads/release/python-3100/) required_\n\n### Usage\n\n```python\nfrom staffspy import LinkedInAccount, SolverType, DriverType, BrowserType\n\naccount = LinkedInAccount(\n # driver_type=DriverType( # if issues with webdriver, specify its exact location, download link in the FAQ\n # browser_type=BrowserType.CHROME,\n # executable_path=\"/Users/pc/chromedriver-mac-arm64/chromedriver\"\n # ),\n session_file=\"session.pkl\", # save login cookies to only log in once (lasts a week or so)\n log_level=1, # 0 for no logs\n)\n\n# search by company\nstaff = account.scrape_staff(\n company_name=\"openai\",\n search_term=\"software engineer\",\n location=\"london\",\n extra_profile_data=True, # fetch all past experiences, schools, & skills\n max_results=50, # can go up to 1000\n # block=True # if you want to block the user after scraping, to exclude from future search results\n # connect=True # if you want to connect with the users until you hit your limit\n)\n# or fetch by user ids\nusers = account.scrape_users(\n user_ids=['williamhgates', 'rbranson', 'jeffweiner08']\n # connect=True,\n # block=True\n)\n\n# fetch all comments on two of Bill Gates' posts \ncomments = account.scrape_comments(\n ['7252421958540091394','7253083989547048961']\n)\n\n# fetch company details\ncompanies = account.scrape_companies(\n company_names=['openai', 'microsoft']\n)\n\n# fetch connections (also gets their contact info if available)\nconnections = account.scrape_connections(\n extra_profile_data=True,\n max_results=50\n)\n\n# export any of the results to csv\nstaff.to_csv(\"staff.csv\", index=False)\n```\n\n#### Browser login\n\nIf you rather use a browser to log in, install the browser add-on to StaffSpy .\n\n`pip install staffspy[browser]`\n\nIf you do not pass the `username` & `password` params, then a browser will open to sign in to LinkedIn on the first sign-in. Press enter after signing in to begin scraping.\n\n### Output\n\n| profile_id | name | first_name | last_name | location | age | position | followers | connections | company | past_company1 | past_company2 | school1 | school2 | skill1 | skill2 | skill3 | is_connection | premium | creator | potential_email | profile_link | profile_photo |\n| ---------------- | -------------- | ---------- | --------- | ------------------------------- | --- | ------------------------------- | --------- | ----------- | ------- | ------------- | ------------- | ---------------------------------- | ------------------------- | -------- | ---------- | ---------- | ------------- | ------- | ------- | ------------------------------------------------ | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| javiersierra2102 | Javier Sierra | Javier | Sierra | London, England, United Kingdom | 39 | Software Engineer | 735 | 725 | OpenAI | Meta | Oculus VR | Hult International Business School | Universidad SimΓ³n BolΓ­var | Java | JavaScript | C++ | FALSE | FALSE | FALSE | javier.sierra@openai.com, jsierra@openai.com | https://www.linkedin.com/in/javiersierra2102 | https://media.licdn.com/dms/image/C4D03AQHEyUg1kGT08Q/profile-displayphoto-shrink_800_800/0/1516504680512?e=1727913600&v=beta&t=3enCmNDBtJ7LxfbW6j1hDD8qNtHjO2jb2XTONECxUXw |\n| dougli | Douglas Li | Douglas | Li | London, England, United Kingdom | 37 | @ OpenAI UK, previously at Meta | 583 | 401 | OpenAI | Shift Lab | Facebook | Washington University in St. Louis | | Java | Python | JavaScript | FALSE | TRUE | FALSE | douglas.li@openai.com, dli@openai.com | https://www.linkedin.com/in/dougli | https://media.licdn.com/dms/image/D4E03AQETmRyb3_GB8A/profile-displayphoto-shrink_800_800/0/1687996628597?e=1727913600&v=beta&t=HRYGJ4RxsTMcPF1YcSikXlbz99hx353csho3PWT6fOQ |\n| nkartashov | Nick Kartashov | Nick | Kartashov | London, England, United Kingdom | 33 | Software Engineer | 2186 | 2182 | OpenAI | Google | DeepMind | St. Petersburg Academic University | Bioinformatics Institute | Teamwork | Java | Haskell | FALSE | FALSE | FALSE | nick.kartashov@openai.com, nkartashov@openai.com | https://www.linkedin.com/in/nkartashov | https://media.licdn.com/dms/image/D4E03AQEjOKxC5UgwWw/profile-displayphoto-shrink_800_800/0/1680706122689?e=1727913600&v=beta&t=m-JnG9nm0zxp1Z7njnInwbCoXyqa3AN-vJZntLfbzQ4 |\n\n\n### Parameters for `LinkedInAccount()`\n\n```plaintext\nOptional\nβ”œβ”€β”€ session_file (str):\n| file path to save session cookies, so only one manual login is needed.\n| can use mult profiles this way\n|\n| For automated login\nβ”œβ”€β”€ username (str):\n| linkedin account email\nβ”‚\nβ”œβ”€β”€ password (str):\n| linkedin account password\n|\nβ”œβ”€β”€ driver_type (DriverType):\n| signs in with the given BrowserType (Chrome, Firefox) and executable_path\n|\nβ”œβ”€β”€ solver_service (SolverType):\n| solves the captcha using the desired service - either CapSolver, or 2Captcha (worse of the two)\n|\nβ”œβ”€β”€ solver_api_key (str):\n| api key for the solver provider\nβ”‚\nβ”œβ”€β”€ log_level (int):\n| Controls the verbosity of the runtime printouts\n| (0 prints only errors, 1 is info, 2 is all logs. Default is 0.)\n```\n\n### Parameters for `scrape_staff()`\n\n```plaintext\nOptional\nβ”œβ”€β”€ company_name (str):\n| company identifier on linkedin, will search for that company if that company id does not exist\n| e.g. openai from https://www.linkedin.com/company/openai\n|\nβ”œβ”€β”€ search_term (str):\n| staff title to search for\n| e.g. software engineer\n|\nβ”œβ”€β”€ location (str):\n| location the staff resides\n| e.g. london\nβ”‚\nβ”œβ”€β”€ extra_profile_data (bool)\n| fetches educations, experiences, skills, certifications (Default false)\nβ”‚\nβ”œβ”€β”€ max_results (int):\n| number of staff to fetch, default/max is 1000 for a search imposed by LinkedIn\n|\nβ”œβ”€β”€ block (bool):\n| whether to block the user after scraping\n|\nβ”œβ”€β”€ connect (bool):\n| whether to conncet with the user after scraping\n```\n\n### Parameters for `scrape_users()`\n\n```plaintext\nβ”œβ”€β”€ user_ids (list):\n| user ids to scrape from\n| e.g. dougmcmillon from https://www.linkedin.com/in/dougmcmillon\n|\nβ”œβ”€β”€ block (bool):\n| whether to block the user after scraping\n|\nβ”œβ”€β”€ connect (bool):\n| whether to conncet with the user after scraping\n```\n\n\n### Parameters for `scrape_comments()`\n\n```plaintext\nβ”œβ”€β”€ post_ids (list):\n| post ids to scrape from\n| e.g. 7252381444906364929 from https://www.linkedin.com/posts/williamhgates_technology-transformtheeveryday-activity-7252381444906364929-Bkls\n```\n\n\n### Parameters for `scrape_companies()`\n\n```plaintext\nβ”œβ”€β”€ company_names (list):\n| list of company names to scrape details from\n| e.g. ['openai', 'microsoft', 'google']\n```\n\n\n### Parameters for `scrape_connections()`\n\n```plaintext\nβ”œβ”€β”€ max_results (int):\n| maximum number of connections to fetch\n|\nβ”œβ”€β”€ extra_profile_data (bool):\n| fetches educations, experiences, skills, certifications & contact info for each connection (Default false)\n```\n\n### LinkedIn notes\n\n - only 1000 max results per search\n - extra_profile_data increases runtime by O(n)\n - if rate limited, the program will stop scraping\n - if using non-browser sign in, turn off 2fa\n\n---\n\n## Frequently Asked Questions\n\n---\n\n**Q: Can I get my account banned?** \n**A:** It is a possibility, although there are no recorded incidents. Let me know if you are the first. However, to protect you, the code does not allow you to run it if LinkedIn is blocking you\n\n---\n\n**Q: Scraped 999 staff members, with 869 hidden LinkedIn Members?** \n**A:** It means your LinkedIn account is bad. Not sure how they classify it but unverified email, new account, low connections and a bunch of factors go into it.\n\n---\n\n**Q: How to get around the 1000 search limit result?** \n**A:** Check the examples folder. We can block the user after searching and try many different locations and search terms to maximize results.\n\n---\n\n**Q: Exception: driver not found for selenium?** \n**A:** You need chromedriver installed (not the chrome): https://googlechromelabs.github.io/chrome-for-testing/#stable\n\n---\n\n**Q: Encountering issues with your queries?** \n**A:** If problems\npersist, [submit an issue](https://github.com/cullenwatson/StaffSpy/issues).\n\n\n### Staff Schema\n\n```plaintext\nStaff\nβ”œβ”€β”€ Personal Information\nβ”‚ β”œβ”€β”€ search_term\nβ”‚ β”œβ”€β”€ id\nβ”‚ β”œβ”€β”€ name\nβ”‚ β”œβ”€β”€ first_name\nβ”‚ β”œβ”€β”€ last_name\nβ”‚ β”œβ”€β”€ location\nβ”‚ └── bio\nβ”‚\nβ”œβ”€β”€ Professional Details\nβ”‚ β”œβ”€β”€ position\nβ”‚ β”œβ”€β”€ profile_id\nβ”‚ β”œβ”€β”€ profile_link\nβ”‚ β”œβ”€β”€ potential_emails\nβ”‚ └── estimated_age\nβ”‚\nβ”œβ”€β”€ Social Connectivity\nβ”‚ β”œβ”€β”€ followers\nβ”‚ β”œβ”€β”€ connections\nβ”‚ └── mutuals_count\nβ”‚\nβ”œβ”€β”€ Status\nβ”‚ β”œβ”€β”€ influencer\nβ”‚ β”œβ”€β”€ creator\nβ”‚ β”œβ”€β”€ premium\nβ”‚ β”œβ”€β”€ open_to_work\nβ”‚ β”œβ”€β”€ is_hiring\nβ”‚ └── is_connection\nβ”‚\nβ”œβ”€β”€ Visuals\nβ”‚ β”œβ”€β”€ profile_photo\nβ”‚ └── banner_photo\nβ”‚\nβ”œβ”€β”€ Skills\nβ”‚ β”œβ”€β”€ name\nβ”‚ └── endorsements\nβ”‚\nβ”œβ”€β”€ Experiences\nβ”‚ β”œβ”€β”€ from_date\nβ”‚ β”œβ”€β”€ to_date\nβ”‚ β”œβ”€β”€ duration\nβ”‚ β”œβ”€β”€ title\nβ”‚ β”œβ”€β”€ company\nβ”‚ β”œβ”€β”€ location\nβ”‚ └── emp_type\nβ”‚\nβ”œβ”€β”€ Certifications\nβ”‚ β”œβ”€β”€ title\nβ”‚ β”œβ”€β”€ issuer\nβ”‚ β”œβ”€β”€ date_issued\nβ”‚ β”œβ”€β”€ cert_id\nβ”‚ └── cert_link\nβ”‚\nβ”œβ”€β”€ Educational Background\n| β”œβ”€β”€ years\n| β”œβ”€β”€ school\n| └── degree\nβ”‚\n└── Connection Info (only when a connection and enabled on their profile)\n β”œβ”€β”€ email_address\n β”œβ”€β”€ address\n β”œβ”€β”€ birthday\n β”œβ”€β”€ websites\n β”œβ”€β”€ phone_numbers\n └── created_at\n```\n" }, { "path": "examples/daily_auto_connect.py", "content": "\"\"\" Script to connect with 10 software engineers daily from random tech companies \"\"\"\n\nfrom staffspy import LinkedInAccount, DriverType, BrowserType\nimport random\nimport time\nfrom datetime import datetime\nimport schedule\n\n# List of tech companies to randomly choose from\nTECH_COMPANIES = [\n \"microsoft\",\n \"google\",\n \"apple\",\n \"meta\",\n \"amazon\",\n \"netflix\",\n \"salesforce\",\n \"adobe\",\n \"intel\",\n \"nvidia\",\n \"oracle\",\n \"ibm\",\n \"vmware\",\n \"twitter\",\n \"linkedin\",\n \"airbnb\",\n \"uber\",\n \"stripe\",\n \"snowflake\",\n \"databricks\",\n]\n\n\ndef connect_with_staff():\n print(f\"Starting connection run at {datetime.now()}\")\n\n # Initialize LinkedIn account\n account = LinkedInAccount(session_file=\"session.pkl\", log_level=1)\n\n # Choose a random company\n company = random.choice(TECH_COMPANIES)\n print(f\"Selected company: {company}\")\n\n # Connect with 10 users\n account.scrape_staff(\n company_name=company,\n search_term=\"software engineer\",\n max_results=10,\n extra_profile_data=True,\n connect=True,\n )\n\n\nif __name__ == \"__main__\":\n # Schedule to run once a day at 10 AM\n schedule.every().day.at(\"10:00\").do(connect_with_staff)\n\n # Run immediately on script start\n connect_with_staff()\n\n # Keep the script running\n while True:\n schedule.run_pending()\n time.sleep(60)\n" }, { "path": "examples/upload_staff_to_clay.py", "content": "\"\"\"\nUploads staff to the Clay platform to then further enrich the staff (e.g. waterfall strategy to find their verified emails)\n\"\"\"\n\nfrom staffspy import LinkedInAccount\nfrom staffspy.utils.utils import upload_to_clay\n\nsession_file = \"session.pkl\"\naccount = LinkedInAccount(session_file=session_file, log_level=2)\n\nconnections = account.scrape_connections(extra_profile_data=True, max_results=3)\n\nclay_webhook_url = (\n \"https://api.clay.com/v3/sources/webhook/pull-in-data-from-a-webhook-XXXXXXXXXXXXXX\"\n)\nupload_to_clay(webhook_url=clay_webhook_url, data=connections)\n" }, { "path": "examples/x_corp_staff.py", "content": "\"\"\"\nCASE STUDY: X CORP EMPLOYEES\nRESULT: We retrieved 1087 profiles. Not as good as expected but still a good result for company that has 2800 employees.\nfinal csv - https://drive.google.com/file/d/1aC-GF4RXf9wzGrpxQyGPBxlnLo2X5vm4\n\nStrategies to get around LinkedIn 1000 result limit:\n1) It blocks the user after searching to prevent it from appearing in future searches.\n2) It tries various searches with department and location to get more results.\n\nLastly, it saves the results in CSV files and then combines them into one DataFrame at the end to view the results.\n\"\"\"\n\nimport os\nfrom datetime import datetime\nimport pandas as pd\nimport glob\n\n\nfrom staffspy import LinkedInAccount\n\nsession_file = \"session.pkl\"\naccount = LinkedInAccount(session_file=str(session_file), log_level=2)\n\n\ndepartments = [\n # Leadership\n \"CEO\",\n \"CFO\",\n \"CTO\",\n \"COO\",\n \"executive\",\n \"director\",\n \"vice president\",\n \"head\",\n \"lead\",\n # Engineering/Tech\n \"software\",\n \"developer\",\n \"engineer\",\n \"architect\",\n \"devops\",\n \"QA\",\n \"data\",\n \"IT\",\n \"security\",\n # Business/Operations\n \"sales\",\n \"account\",\n \"business development\",\n \"operations\",\n \"project manager\",\n \"product manager\",\n # Support Functions\n \"HR\",\n \"recruiter\",\n \"marketing\",\n \"finance\",\n \"legal\",\n \"accounting\",\n \"admin\",\n \"support\",\n # Customer-Facing\n \"customer success\",\n \"account manager\",\n \"sales representative\",\n \"customer support\",\n # Specialists\n \"analyst\",\n \"consultant\",\n \"coordinator\",\n \"specialist\",\n]\nlocations = [\n \"San Francisco\",\n \"New York\",\n \"Los Angeles\",\n \"Seattle\",\n \"Miami\",\n \"Boston\",\n \"Austin\",\n \"Chicago\",\n \"Toronto\",\n \"London\",\n \"Singapore\",\n \"Tokyo\",\n \"Dublin\",\n]\n\n\ndef save_results(users: pd.DataFrame):\n output_dir = f\"output/{company_name}\"\n os.makedirs(output_dir, exist_ok=True)\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n output_path = f\"{output_dir}/users_{timestamp}.csv\"\n users.to_csv(output_path, index=False)\n\n\ndef scrape_and_save(term=None, location=None):\n users = account.scrape_staff(\n company_name=company_name,\n search_term=term,\n location=location,\n extra_profile_data=True,\n max_results=1000,\n block=True,\n )\n if not users.empty:\n save_results(users)\n\n\ncompany_name = \"x-corp\"\n\n# generic search\nfor _ in range(5):\n scrape_and_save()\n\n# Search by departments\nfor department in departments:\n scrape_and_save(term=department)\n\n# Search by locations\nfor location in locations:\n scrape_and_save(location=location)\n\n# load all csvs into one df\nfiles = glob.glob(\"output/x-corp/*.csv\")\ndfs = [pd.read_csv(f) for f in files]\ncombined_df = pd.concat(dfs, ignore_index=True)\n\n# Filter out hidden profiles\nfiltered_df = combined_df[combined_df[\"urn\"] != \"headless\"]\nfiltered_df = filtered_df[filtered_df[\"current_company\"] == \"X\"]\nfiltered_df = filtered_df.drop_duplicates(subset=\"id\")\n\nfiltered_urns = len(set(filtered_df[\"urn\"]))\nprint(f\"Total unique profiles: {filtered_urns}\")\ncompany_name = \"x-corp\"\nfiltered_df.to_csv(\n f\"output/{company_name}/final_result_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv\",\n index=False,\n)\n" }, { "path": "pyproject.toml", "content": "[tool.poetry]\nname = \"staffspy\"\nversion = \"0.2.25\"\ndescription = \"Staff scraper library for LinkedIn\"\nauthors = [\"Cullen Watson \"]\nreadme = \"README.md\"\n\n[tool.poetry.dependencies]\npython = \"^3.10\"\npydantic = \"^2.7.2\"\npandas = \"^2.2.2\"\nrequests = \"^2.32.3\"\ntldextract = \"^5.1.2\"\nselenium = { version = \"^4.3.0\", optional = true }\ntenacity = \"^8.5.0\"\npython-dateutil = \"^2.9.0.post0\"\nbeautifulsoup4 = \"^4.12.3\"\n2captcha-python = \"^1.2.8\"\n\n[tool.poetry.extras]\nbrowser = [\"selenium\"]\n\n[tool.poetry.group.dev.dependencies]\npre-commit = \"^3.7.1\"\nblack = \"^24.4.2\"\njupyter = \"^1.0.0\"\n\n[build-system]\nrequires = [\"poetry-core\"]\nbuild-backend = \"poetry.core.masonry.api\"\n" }, { "path": "staffspy/__init__.py", "content": "import json\nimport pandas as pd\n\nfrom staffspy.linkedin.comments import CommentFetcher\nfrom staffspy.linkedin.linkedin import LinkedInScraper\nfrom staffspy.utils.models import Staff\nfrom staffspy.solvers.capsolver import CapSolver\nfrom staffspy.solvers.solver_type import SolverType\nfrom staffspy.solvers.two_captcha import TwoCaptchaSolver\nfrom staffspy.utils.utils import (\n set_logger_level,\n logger,\n Login,\n parse_company_data,\n extract_emails_from_text,\n clean_df,\n)\nfrom staffspy.utils.driver_type import DriverType, BrowserType\n\n__all__ = [\n \"LinkedInAccount\",\n \"SolverType\",\n \"DriverType\",\n \"BrowserType\",\n]\n\n\nclass LinkedInAccount:\n \"\"\"LinkedinAccount storing cookie data and providing outer facing methods for client\"\"\"\n\n solver_map = {\n SolverType.CAPSOLVER: CapSolver,\n SolverType.TWO_CAPTCHA: TwoCaptchaSolver,\n }\n\n def __init__(\n self,\n session_file: str = None,\n username: str = None,\n password: str = None,\n log_level: int = 0,\n solver_api_key: str = None,\n solver_service: SolverType = SolverType.CAPSOLVER,\n driver_type: DriverType = None,\n ):\n self.session_file = session_file\n self.username = username\n self.password = password\n self.log_level = log_level\n self.solver = self.solver_map[solver_service](solver_api_key)\n self.driver_type = driver_type\n self.session = None\n self.linkedin_scraper = None\n self.on_block = False\n self.login()\n\n def login(self):\n set_logger_level(self.log_level)\n login = Login(\n self.username,\n self.password,\n self.solver,\n self.session_file,\n self.driver_type,\n )\n self.session = login.load_session()\n\n def scrape_staff(\n self,\n company_name: str = None,\n search_term: str = None,\n location: str = None,\n extra_profile_data: bool = False,\n max_results: int = 1000,\n block: bool = False,\n connect: bool = False,\n ):\n if self.on_block:\n return logger.error(\n \"Account is on cooldown as a safety precaution after receiving a 429 (TooManyRequests) from LinkedIn. Please recreate a new LinkedInAccount to proceed.\"\n )\n \"\"\"Main function entry point to scrape LinkedIn staff\"\"\"\n li_scraper = LinkedInScraper(self.session)\n staff = li_scraper.scrape_staff(\n company_name=company_name,\n extra_profile_data=extra_profile_data,\n search_term=search_term,\n location=location,\n max_results=max_results,\n block=block,\n connect=connect,\n )\n if li_scraper.on_block:\n self.on_block = True\n staff_dicts = [staff.to_dict() for staff in staff]\n staff_df = pd.DataFrame(staff_dicts)\n if staff_df.empty:\n return staff_df\n\n staff_df = clean_df(staff_df)\n linkedin_member_df = staff_df[staff_df[\"name\"] == \"LinkedIn Member\"]\n non_linkedin_member_df = staff_df[staff_df[\"name\"] != \"LinkedIn Member\"]\n staff_df = pd.concat([non_linkedin_member_df, linkedin_member_df])\n logger.info(\n f\"3) Staff from {company_name}: {len(staff_df)} total, {len(linkedin_member_df)} hidden, {len(staff_df) - len(linkedin_member_df)} visible\"\n )\n return staff_df.reset_index(drop=True)\n\n def scrape_users(\n self, user_ids: list[str], block: bool = False, connect: bool = False\n ) -> pd.DataFrame | None:\n \"\"\"Scrape users from Linkedin by user IDs\"\"\"\n if self.on_block:\n return logger.error(\n \"Account is on cooldown as a safety precaution after receiving a 429 (TooManyRequests) from LinkedIn. Please recreate a new LinkedInAccount to proceed.\"\n )\n\n li_scraper = LinkedInScraper(self.session)\n li_scraper.num_staff = len(user_ids)\n users = [\n Staff(\n id=\"\",\n search_term=\"manual\",\n profile_id=user_id,\n profile_link=f\"https://www.linkedin.com/in/{user_id}\",\n )\n for user_id in user_ids\n ]\n\n for i, user in enumerate(users, start=1):\n user.id, user.urn = li_scraper.fetch_user_profile_data_from_public_id(\n user.profile_id, \"user_id\"\n )\n if user.id:\n li_scraper.fetch_all_info_for_employee(user, i)\n if block:\n li_scraper.block_user(user)\n elif connect:\n li_scraper.connect_user(user)\n\n users_dicts = [user.to_dict() for user in users if user.id]\n users_df = pd.DataFrame(users_dicts)\n\n if users_df.empty:\n return users_df\n linkedin_member_df = users_df[users_df[\"name\"] == \"LinkedIn Member\"]\n non_linkedin_member_df = users_df[users_df[\"name\"] != \"LinkedIn Member\"]\n users_df = pd.concat([non_linkedin_member_df, linkedin_member_df])\n logger.info(f\"Scraped {len(users_df)} users\")\n return users_df\n\n def scrape_comments(self, post_ids: list[str]) -> pd.DataFrame:\n \"\"\"Scrape comments from Linkedin by post IDs\"\"\"\n if self.on_block:\n return logger.error(\n \"Account is on cooldown as a safety precaution after receiving a 429 (TooManyRequests) from LinkedIn. Please recreate a new LinkedInAccount to proceed.\"\n )\n\n comment_fetcher = CommentFetcher(self.session)\n all_comments = []\n for i, post_id in enumerate(post_ids, start=1):\n comments = comment_fetcher.fetch_comments(post_id)\n all_comments.extend(comments)\n\n comment_dict = [comment.to_dict() for comment in all_comments]\n comment_df = pd.DataFrame(comment_dict)\n\n if not comment_df.empty:\n comment_df[\"emails\"] = comment_df[\"text\"].apply(extract_emails_from_text)\n comment_df = comment_df.sort_values(by=\"created_at\", ascending=False)\n\n return comment_df\n\n def scrape_companies(\n self,\n company_names: list[str] = None,\n ) -> pd.DataFrame:\n \"\"\"Scrape company details from Linkedin\"\"\"\n if self.on_block:\n return logger.error(\n \"Account is on cooldown as a safety precaution after receiving a 429 (TooManyRequests) from LinkedIn. Please recreate a new LinkedInAccount to proceed.\"\n )\n\n if not company_names:\n raise ValueError(\"company_names list cannot be empty\")\n\n li_scraper = LinkedInScraper(self.session)\n company_dfs = []\n\n for company_name in company_names:\n try:\n company_res = li_scraper.fetch_or_search_company(company_name)\n try:\n company_data = company_res.json()\n except json.decoder.JSONDecodeError:\n logger.error(f\"Failed to fetch company data for {company_name}\")\n continue\n\n company_df = parse_company_data(company_data, search_term=company_name)\n company_dfs.append(company_df)\n\n except Exception as e:\n logger.error(f\"Failed to process company {company_name}: {str(e)}\")\n continue\n\n if not company_dfs:\n return pd.DataFrame()\n\n return pd.concat(company_dfs, ignore_index=True)\n\n def scrape_connections(\n self,\n max_results: int = 10**8,\n extra_profile_data: bool = False,\n ) -> pd.DataFrame:\n \"\"\"Scrape connections from Linkedin\"\"\"\n if self.on_block:\n return logger.error(\n \"Account is on cooldown as a safety precaution after receiving a 429 (TooManyRequests) from LinkedIn. Please recreate a new LinkedInAccount to proceed.\"\n )\n li_scraper = LinkedInScraper(self.session)\n\n connections = li_scraper.scrape_connections(\n max_results=max_results,\n extra_profile_data=extra_profile_data,\n )\n connections_df = pd.DataFrame()\n if connections:\n staff_dicts = [staff.to_dict() for staff in connections]\n connections_df = pd.DataFrame(staff_dicts)\n connections_df = clean_df(connections_df)\n\n return connections_df\n" }, { "path": "staffspy/linkedin/certifications.py", "content": "import json\nimport logging\n\nfrom staffspy.utils.exceptions import TooManyRequests\nfrom staffspy.utils.models import Certification\n\nlogger = logging.getLogger(__name__)\n\n\nclass CertificationFetcher:\n def __init__(self, session):\n self.session = session\n self.endpoint = \"https://www.linkedin.com/voyager/api/graphql?queryId=voyagerIdentityDashProfileComponents.277ba7d7b9afffb04683953cede751fb&queryName=ProfileComponentsBySectionType&variables=(tabIndex:0,sectionType:certifications,profileUrn:urn%3Ali%3Afsd_profile%3A{employee_id},count:50)\"\n\n def fetch_certifications(self, staff):\n ep = self.endpoint.format(employee_id=staff.id)\n res = self.session.get(ep)\n logger.debug(f\"certs, status code - {res.status_code}\")\n if res.status_code == 429:\n raise TooManyRequests(\"429 Too Many Requests\")\n if not res.ok:\n logger.debug(res.text[:200])\n return False\n try:\n res_json = res.json()\n except json.decoder.JSONDecodeError:\n logger.debug(res.text[:200])\n return False\n\n try:\n elems = res_json[\"data\"][\"identityDashProfileComponentsBySectionType\"][\n \"elements\"\n ]\n except (KeyError, IndexError, TypeError) as e:\n logger.debug(res_json)\n return False\n\n if elems:\n cert_elems = elems[0][\"components\"][\"pagedListComponent\"][\"components\"][\n \"elements\"\n ]\n staff.certifications = self.parse_certifications(cert_elems)\n return True\n\n def parse_certifications(self, sections):\n certs = []\n for section in sections:\n elem = section[\"components\"][\"entityComponent\"]\n if not elem:\n break\n title = elem[\"titleV2\"][\"text\"][\"text\"]\n issuer = elem[\"subtitle\"][\"text\"] if elem[\"subtitle\"] else None\n date_issued = (\n elem[\"caption\"][\"text\"].replace(\"Issued \", \"\")\n if elem[\"caption\"]\n else None\n )\n cert_id = (\n elem[\"metadata\"][\"text\"].replace(\"Credential ID \", \"\")\n if elem[\"metadata\"]\n else None\n )\n try:\n subcomp = elem[\"subComponents\"][\"components\"][0]\n cert_link = subcomp[\"components\"][\"actionComponent\"][\"action\"][\n \"navigationAction\"\n ][\"actionTarget\"]\n except:\n cert_link = None\n cert = Certification(\n title=title,\n issuer=issuer,\n date_issued=date_issued,\n cert_link=cert_link,\n cert_id=cert_id,\n )\n certs.append(cert)\n\n return certs\n" }, { "path": "staffspy/linkedin/comments.py", "content": "import json\nimport re\nfrom datetime import datetime as dt\n\nfrom staffspy.utils.exceptions import TooManyRequests\nfrom staffspy.utils.models import Comment\n\nfrom staffspy.utils.utils import logger\n\n\nclass CommentFetcher:\n\n def __init__(self, session):\n self.session = session\n self.endpoint = \"https://www.linkedin.com/voyager/api/graphql?queryId=voyagerSocialDashComments.8cb29aedde780600a7ad17fc7ebb8277&queryName=SocialDashCommentsBySocialDetail&variables=(origins:List(),count:100,socialDetailUrn:urn%3Ali%3Afsd_socialDetail%3A%28urn%3Ali%3Aactivity%3A{post_id}%2Curn%3Ali%3Aactivity%3A7254884361622208512%2Curn%3Ali%3AhighlightedReply%3A-%29,sortOrder:REVERSE_CHRONOLOGICAL,start:{start})\"\n self.post_id = None\n self.num_commments = 100\n\n def fetch_comments(self, post_id: str):\n all_comments = []\n self.post_id = post_id\n\n for i in range(0, 200_000, self.num_commments):\n logger.info(f\"Fetching comments for post {post_id}, start {i}\")\n\n ep = self.endpoint.format(post_id=post_id, start=i)\n res = self.session.get(ep)\n logger.debug(f\"comments info, status code - {res.status_code}\")\n\n if res.status_code == 429:\n raise TooManyRequests(\"429 Too Many Requests\")\n if not res.ok:\n logger.debug(res.text[:200])\n return False\n try:\n comments_json = res.json()\n except json.decoder.JSONDecodeError:\n logger.debug(res.text[:200])\n return False\n\n comments, num_results = self.parse_comments(comments_json)\n all_comments.extend(comments)\n if not num_results:\n break\n\n return all_comments\n\n def parse_comments(self, comments_json: dict):\n \"\"\"Parse the comment data from the employee profile.\"\"\"\n comments = []\n for element in (\n results := comments_json.get(\"data\", {})\n .get(\"socialDashCommentsBySocialDetail\", {})\n .get(\"elements\", [])\n ):\n internal_profile_id = (commenter := element[\"commenter\"])[\n \"commenterProfileId\"\n ]\n name = commenter[\"title\"][\"text\"]\n linkedin_id_match = re.search(\"/in/(.+)\", commenter[\"navigationUrl\"])\n linkedin_id = linkedin_id_match.group(1) if linkedin_id_match else None\n\n commentary = element.get(\"commentary\", {}).get(\"text\", \"\")\n comment_id = element[\"urn\"].split(\",\")[-1].rstrip(\")\")\n num_likes = element[\"socialDetail\"][\"totalSocialActivityCounts\"][\"numLikes\"]\n comment = Comment(\n post_id=self.post_id,\n comment_id=comment_id,\n internal_profile_id=internal_profile_id,\n public_profile_id=linkedin_id,\n name=name,\n text=commentary,\n num_likes=num_likes,\n created_at=dt.utcfromtimestamp(element[\"createdAt\"] / 1000),\n )\n comments.append(comment)\n\n return comments, len(results)\n" }, { "path": "staffspy/linkedin/contact_info.py", "content": "from calendar import month_name\nfrom datetime import datetime\nimport json\nimport requests\nimport logging\n\nimport pytz\n\nfrom staffspy.utils.exceptions import TooManyRequests\nfrom staffspy.utils.models import ContactInfo, Staff\n\nlogger = logging.getLogger(__name__)\n\n\nclass ContactInfoFetcher:\n def __init__(self, session):\n self.session = session\n self.endpoint = \"https://www.linkedin.com/voyager/api/graphql?queryId=voyagerIdentityDashProfiles.13618f886ce95bf503079f49245fbd6f&queryName=ProfilesByMemberIdentity&variables=(memberIdentity:{employee_id},count:1)\"\n\n def fetch_contact_info(self, base_staff):\n ep = self.endpoint.format(employee_id=base_staff.id)\n try:\n res = self.session.get(ep)\n except requests.exceptions.TooManyRedirects as e:\n logger.error(\"Too many redirects encountered: %s\", e)\n return None\n logger.debug(f\"bio info, status code - {res.status_code}\")\n if res.status_code == 429:\n return TooManyRequests(\"429 Too Many Requests\")\n if not res.ok:\n logger.debug(res.text)\n return False\n try:\n employee_json = res.json()\n except json.decoder.JSONDecodeError:\n logger.debug(res.text)\n return False\n\n self.parse_emp_contact_info(base_staff, employee_json)\n return True\n\n def parse_emp_contact_info(self, emp: Staff, emp_dict: dict):\n \"\"\"Parse the employee data from the employee profile.\"\"\"\n contact_info = ContactInfo()\n emp_dict = emp_dict[\"data\"][\"identityDashProfilesByMemberIdentity\"][\"elements\"][\n 0\n ]\n try:\n contact_info.email_address = emp_dict[\"emailAddress\"][\"emailAddress\"]\n except (KeyError, IndexError, TypeError):\n pass\n\n try:\n contact_info.address = emp_dict[\"address\"]\n except (KeyError, IndexError, TypeError):\n pass\n\n try:\n month = month_name[emp_dict[\"birthDateOn\"][\"month\"]]\n day = emp_dict[\"birthDateOn\"][\"day\"]\n contact_info.birthday = f\"{month} {day}\"\n except (KeyError, IndexError, TypeError):\n pass\n\n try:\n contact_info.websites = [x[\"url\"] for x in emp_dict[\"websites\"]]\n except (KeyError, IndexError, TypeError):\n pass\n\n try:\n contact_info.phone_numbers = [\n x[\"phoneNumber\"][\"number\"] for x in emp_dict[\"phoneNumbers\"]\n ]\n except (KeyError, IndexError, TypeError):\n pass\n\n try:\n created_at = emp_dict[\"memberRelationship\"][\n \"memberRelationshipDataResolutionResult\"\n ][\"connection\"][\"createdAt\"]\n timezone = pytz.timezone(\"UTC\")\n dt = datetime.fromtimestamp(created_at / 1000, tz=timezone)\n contact_info.created_at = dt.strftime(\"%Y-%m-%d %H:%M:%S %Z\")\n except (KeyError, IndexError, TypeError):\n pass\n emp.contact_info = contact_info\n" }, { "path": "staffspy/linkedin/employee.py", "content": "import json\nimport logging\nimport re\n\nimport staffspy.utils.utils as utils\nfrom staffspy.utils.exceptions import TooManyRequests\nfrom staffspy.utils.models import Staff\n\nlogger = logging.getLogger(__name__)\n\n\nclass EmployeeFetcher:\n def __init__(self, session):\n self.session = session\n self.endpoint = \"https://www.linkedin.com/voyager/api/voyagerIdentityDashProfiles?count=1&decorationId=com.linkedin.voyager.dash.deco.identity.profile.TopCardComplete-138&memberIdentity={employee_id}&q=memberIdentity\"\n\n self.domain = None\n\n def fetch_employee(self, base_staff, domain):\n self.domain = domain\n ep = self.endpoint.format(employee_id=base_staff.id)\n res = self.session.get(ep)\n logger.debug(f\"basic info, status code - {res.status_code}\")\n if res.status_code == 429:\n return TooManyRequests(\"429 Too Many Requests\")\n if not res.ok:\n logger.debug(res.text[:200])\n return False\n try:\n res_json = res.json()\n except json.decoder.JSONDecodeError:\n logger.debug(res.text[:200])\n return False\n\n try:\n employee_json = res_json[\"elements\"][0]\n except (KeyError, IndexError, TypeError):\n logger.debug(res_json)\n return False\n\n self.parse_emp(base_staff, employee_json)\n return True\n\n def parse_emp(self, emp: Staff, emp_dict: dict):\n \"\"\"Parse the employee data from the employee profile.\"\"\"\n\n def get_photo_url(emp_dict: dict, key: str):\n try:\n photo_data = emp_dict[key][\"displayImageReference\"][\"vectorImage\"]\n photo_base_url = photo_data[\"rootUrl\"]\n photo_ext_url = photo_data[\"artifacts\"][-1][\n \"fileIdentifyingUrlPathSegment\"\n ]\n return f\"{photo_base_url}{photo_ext_url}\"\n except (KeyError, TypeError, IndexError, ValueError):\n return None\n\n emp.profile_photo = get_photo_url(emp_dict, \"profilePicture\")\n emp.banner_photo = get_photo_url(emp_dict, \"backgroundPicture\")\n emp.profile_id = emp_dict[\"publicIdentifier\"]\n try:\n emp.headline = emp_dict.get(\"headline\")\n if not emp.headline:\n emp.headline = emp_dict[\"memberRelationship\"][\"memberRelationshipData\"][\n \"noInvitation\"\n ][\"targetInviteeResolutionResult\"][\"headline\"]\n except:\n pass\n union_type = next(\n iter(emp_dict[\"memberRelationship\"][\"memberRelationshipUnion\"])\n )\n emp.is_connection = \"no\"\n if union_type == \"connection\":\n emp.is_connection = \"yes\"\n elif union_type == \"noConnection\":\n invitation = (\n emp_dict[\"memberRelationship\"][\"memberRelationshipUnion\"][\n \"noConnection\"\n ]\n .get(\"invitationUnion\", {})\n .get(\"invitation\", {})\n )\n if invitation and invitation.get(\"invitationState\") == \"PENDING\":\n emp.is_connection = \"pending\"\n\n emp.open_to_work = emp_dict[\"profilePicture\"].get(\"frameType\") == \"OPEN_TO_WORK\"\n emp.is_hiring = emp_dict[\"profilePicture\"].get(\"frameType\") == \"HIRING\"\n\n emp.first_name = emp_dict[\"firstName\"]\n emp.last_name = emp_dict[\"lastName\"].split(\",\")[0]\n if not emp.name:\n name = filter(None, [emp.first_name, emp.last_name])\n emp.name = \" \".join(name)\n emp.potential_emails = (\n utils.create_emails(emp.first_name, emp.last_name, self.domain)\n if self.domain\n else None\n )\n\n emp.followers = emp_dict.get(\"followingState\", {}).get(\"followerCount\")\n emp.connections = emp_dict[\"connections\"][\"paging\"][\"total\"]\n emp.location = (\n emp_dict.get(\"geoLocation\", {}).get(\"geo\", {}).get(\"defaultLocalizedName\")\n )\n\n # Handle empty elements case for company\n top_positions = emp_dict.get(\"profileTopPosition\", {}).get(\"elements\", [])\n if top_positions:\n emp.company = top_positions[0].get(\"companyName\", None)\n else:\n emp.company = None\n\n edu_cards = emp_dict.get(\"profileTopEducation\", {}).get(\"elements\", [])\n if edu_cards:\n emp.school = edu_cards[0].get(\n \"schoolName\", edu_cards[0].get(\"school\", {}).get(\"name\")\n )\n emp.influencer = emp_dict.get(\"influencer\", False)\n emp.creator = emp_dict.get(\"creator\", False)\n emp.premium = emp_dict.get(\"premium\", False)\n emp.mutual_connections = 0\n\n try:\n profile_insight = emp_dict.get(\"profileInsight\", {}).get(\"elements\", [])\n if profile_insight:\n mutual_connections_str = profile_insight[0][\"text\"][\"text\"]\n match = re.search(r\"\\d+\", mutual_connections_str)\n if match:\n emp.mutual_connections = int(match.group()) + 2\n else:\n emp.mutual_connections = (\n 2 if \" and \" in mutual_connections_str else 1\n )\n except (KeyError, TypeError, IndexError, ValueError) as e:\n pass\n" }, { "path": "staffspy/linkedin/employee_bio.py", "content": "import json\nimport logging\n\nfrom staffspy.utils.exceptions import TooManyRequests\n\nlogger = logging.getLogger(__name__)\n\n\nclass EmployeeBioFetcher:\n def __init__(self, session):\n self.session = session\n self.endpoint = \"https://www.linkedin.com/voyager/api/graphql?queryId=voyagerIdentityDashProfileCards.9ad2590cb61a073ad514922fa752f566&queryName=ProfileTabInitialCards&variables=(count:50,profileUrn:urn%3Ali%3Afsd_profile%3A{employee_id})\"\n\n def fetch_employee_bio(self, base_staff):\n ep = self.endpoint.format(employee_id=base_staff.id)\n res = self.session.get(ep)\n logger.debug(f\"bio info, status code - {res.status_code}\")\n if res.status_code == 429:\n return TooManyRequests(\"429 Too Many Requests\")\n if not res.ok:\n logger.debug(res.text)\n return False\n try:\n data = res.json()\n except json.decoder.JSONDecodeError:\n logger.debug(res.text)\n return False\n\n try:\n base_staff.bio = data[\"data\"][\"identityDashProfileCardsByInitialCards\"][\n \"elements\"\n ][3][\"topComponents\"][1][\"components\"][\"textComponent\"][\"text\"][\"text\"]\n except (KeyError, IndexError, TypeError):\n return False\n\n return True\n" }, { "path": "staffspy/linkedin/experiences.py", "content": "import json\nimport logging\n\nimport staffspy.utils.utils as utils\nfrom staffspy.utils.exceptions import TooManyRequests\nfrom staffspy.utils.models import Experience\n\nlogger = logging.getLogger(__name__)\n\n\nclass ExperiencesFetcher:\n def __init__(self, session):\n self.session = session\n self.endpoint = \"https://www.linkedin.com/voyager/api/graphql?queryId=voyagerIdentityDashProfileComponents.277ba7d7b9afffb04683953cede751fb&queryName=ProfileComponentsBySectionType&variables=(tabIndex:0,sectionType:experience,profileUrn:urn%3Ali%3Afsd_profile%3A{employee_id},count:50)\"\n\n def fetch_experiences(self, staff):\n ep = self.endpoint.format(employee_id=staff.id)\n res = self.session.get(ep)\n logger.debug(f\"exps, status code - {res.status_code}\")\n if res.reason == \"INKApi Error\":\n raise Exception(\n \"Delete session file and log in again\",\n res.status_code,\n res.text[:200],\n res.reason,\n )\n elif res.status_code == 429:\n return TooManyRequests(\"429 Too Many Requests\")\n elif not res.ok:\n logger.debug(res.text[:200])\n return False\n try:\n res_json = res.json()\n except json.decoder.JSONDecodeError:\n logger.debug(res.text[:200])\n return False\n\n try:\n skills_json = res_json[\"data\"][\n \"identityDashProfileComponentsBySectionType\"\n ][\"elements\"][0][\"components\"][\"pagedListComponent\"][\"components\"][\n \"elements\"\n ]\n except (KeyError, IndexError, TypeError) as e:\n logger.debug(res_json)\n return False\n\n staff.experiences = self.parse_experiences(skills_json)\n return True\n\n def parse_experiences(self, elements):\n exps = []\n for elem in elements:\n try:\n components = elem.get(\"components\")\n if components is None:\n continue\n\n entity = components.get(\"entityComponent\")\n if entity is None:\n continue\n\n sub_components = entity.get(\"subComponents\")\n if (\n sub_components is None\n or len(sub_components.get(\"components\", [])) == 0\n or sub_components[\"components\"][0].get(\"components\") is None\n or sub_components[\"components\"][0][\"components\"].get(\n \"pagedListComponent\"\n )\n is None\n ):\n\n emp_type = start_date = end_date = None\n\n caption = entity.get(\"caption\")\n duration = caption.get(\"text\") if caption else None\n if duration:\n start_date, end_date = utils.parse_dates(duration)\n from_date, to_date = utils.parse_duration(duration)\n if from_date:\n duration_parts = duration.split(\" Β· \")\n if len(duration_parts) > 1:\n duration = duration_parts[1]\n\n subtitle = entity.get(\"subtitle\")\n company = subtitle.get(\"text\") if subtitle else None\n\n titleV2 = entity.get(\"titleV2\")\n title_text = titleV2.get(\"text\") if titleV2 else None\n title = title_text.get(\"text\") if title_text else None\n\n metadata = entity.get(\"metadata\")\n location = metadata.get(\"text\") if metadata else None\n\n if company:\n parts = company.split(\" Β· \")\n if len(parts) > 1:\n company = parts[0]\n emp_type = parts[-1].lower()\n\n exp = Experience(\n duration=duration,\n title=title,\n company=company,\n emp_type=emp_type,\n start_date=start_date,\n end_date=end_date,\n location=location,\n )\n exps.append(exp)\n else:\n multi_exps = self.parse_multi_exp(entity)\n exps += multi_exps\n\n except Exception as e:\n logger.exception(e)\n\n return exps\n\n def parse_multi_exp(self, entity):\n exps = []\n company = entity[\"titleV2\"][\"text\"][\"text\"]\n elements = entity[\"subComponents\"][\"components\"][0][\"components\"][\n \"pagedListComponent\"\n ][\"components\"][\"elements\"]\n for elem in elements:\n entity = elem[\"components\"][\"entityComponent\"]\n duration = entity[\"caption\"][\"text\"]\n title = entity[\"titleV2\"][\"text\"][\"text\"]\n emp_type = (\n entity[\"subtitle\"][\"text\"].lower() if entity[\"subtitle\"] else None\n )\n location = entity[\"metadata\"][\"text\"] if entity[\"metadata\"] else None\n start_date, end_date = utils.parse_dates(duration)\n from_date, to_date = utils.parse_duration(duration)\n if from_date:\n duration = duration.split(\" Β· \")[1]\n exp = Experience(\n duration=duration,\n title=title,\n company=company,\n emp_type=emp_type,\n start_date=start_date,\n end_date=end_date,\n location=location,\n )\n exps.append(exp)\n return exps\n" }, { "path": "staffspy/linkedin/languages.py", "content": "import json\nimport logging\n\nfrom staffspy.utils.exceptions import TooManyRequests\nfrom staffspy.utils.models import Skill, Staff\n\nlogger = logging.getLogger(__name__)\n\n\nclass LanguagesFetcher:\n def __init__(self, session):\n self.session = session\n self.endpoint = \"https://www.linkedin.com/voyager/api/graphql?queryId=voyagerIdentityDashProfileComponents.9117695ef207012719e3e0681c667e14&queryName=ProfileComponentsBySectionType&variables=(tabIndex:0,sectionType:languages,profileUrn:urn%3Ali%3Afsd_profile%3A{employee_id},count:50)\"\n\n def fetch_languages(self, staff: Staff):\n ep = self.endpoint.format(employee_id=staff.id)\n res = self.session.get(ep)\n logger.debug(f\"skills, status code - {res.status_code}\")\n if res.status_code == 429:\n return TooManyRequests(\"429 Too Many Requests\")\n if not res.ok:\n logger.debug(res.text)\n return False\n try:\n res_json = res.json()\n except json.decoder.JSONDecodeError:\n logger.debug(res.text)\n return False\n\n if res_json.get(\"errors\"):\n return False\n staff.languages = self.parse_languages(res_json)\n return True\n\n def parse_languages(self, language_json: dict) -> list[str]:\n languages = []\n elements = language_json[\"data\"][\"identityDashProfileComponentsBySectionType\"][\n \"elements\"\n ][0][\"components\"][\"pagedListComponent\"][\"components\"][\"elements\"]\n\n for element in elements:\n if comp := element[\"components\"][\"entityComponent\"]:\n title = comp[\"titleV2\"][\"text\"][\"text\"]\n languages.append(title)\n\n return languages\n" }, { "path": "staffspy/linkedin/linkedin.py", "content": "\"\"\"\nstaffspy.linkedin.linkedin\n~~~~~~~~~~~~~~~~~~~\n\nThis module contains routines to scrape LinkedIn.\n\"\"\"\n\nimport json\nimport re\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom urllib.parse import quote, unquote\n\nimport requests\n\nimport staffspy.utils.utils as utils\nfrom staffspy.utils.exceptions import TooManyRequests, BadCookies, GeoUrnNotFound\nfrom staffspy.linkedin.contact_info import ContactInfoFetcher\nfrom staffspy.linkedin.certifications import CertificationFetcher\nfrom staffspy.linkedin.employee import EmployeeFetcher\nfrom staffspy.linkedin.employee_bio import EmployeeBioFetcher\nfrom staffspy.linkedin.experiences import ExperiencesFetcher\nfrom staffspy.linkedin.languages import LanguagesFetcher\nfrom staffspy.linkedin.schools import SchoolsFetcher\nfrom staffspy.linkedin.skills import SkillsFetcher\nfrom staffspy.utils.models import Staff\nfrom staffspy.utils.utils import logger\n\n\nclass LinkedInScraper:\n employees_ep = \"https://www.linkedin.com/voyager/api/graphql?variables=(start:{offset},query:(flagshipSearchIntent:SEARCH_SRP,{search}queryParameters:List({company_id}{location}(key:resultType,value:List(PEOPLE))),includeFiltersInResponse:false),count:{count})&queryId=voyagerSearchDashClusters.66adc6056cf4138949ca5dcb31bb1749\"\n company_id_ep = \"https://www.linkedin.com/voyager/api/organization/companies?q=universalName&universalName=\"\n company_search_ep = \"https://www.linkedin.com/voyager/api/graphql?queryId=voyagerSearchDashClusters.02af3bc8bc85a169bb76bb4805d05759&queryName=SearchClusterCollection&variables=(query:(flagshipSearchIntent:SEARCH_SRP,keywords:{company},includeFiltersInResponse:false,queryParameters:(keywords:List({company}),resultType:List(COMPANIES))),count:10,origin:GLOBAL_SEARCH_HEADER,start:0)\"\n location_id_ep = \"https://www.linkedin.com/voyager/api/graphql?queryId=voyagerSearchDashReusableTypeahead.57a4fa1dd92d3266ed968fdbab2d7bf5&queryName=SearchReusableTypeaheadByType&variables=(query:(showFullLastNameForConnections:false,typeaheadFilterQuery:(geoSearchTypes:List(MARKET_AREA,COUNTRY_REGION,ADMIN_DIVISION_1,CITY))),keywords:{location},type:GEO,start:0)\"\n public_user_id_ep = (\n \"https://www.linkedin.com/voyager/api/identity/profiles/{user_id}/profileView\"\n )\n connections_ep = \"https://www.linkedin.com/voyager/api/graphql?queryId=voyagerSearchDashClusters.dfcd3603c2779eddd541f572936f4324&queryName=SearchClusterCollection&variables=(query:(queryParameters:(resultType:List(FOLLOWERS)),flagshipSearchIntent:MYNETWORK_CURATION_HUB,includeFiltersInResponse:true),count:50,origin:CurationHub,start:{offset})\"\n block_user_ep = \"https://www.linkedin.com/voyager/api/voyagerTrustDashContentReportingForm?action=entityBlock\"\n connect_to_user_ep = \"https://www.linkedin.com/voyager/api/voyagerRelationshipsDashMemberRelationships?action=verifyQuotaAndCreateV2&decorationId=com.linkedin.voyager.dash.deco.relationships.InvitationCreationResultWithInvitee-1\"\n\n def __init__(self, session: requests.Session):\n self.session = session\n (\n self.company_id,\n self.staff_count,\n self.num_staff,\n self.company_name,\n self.domain,\n self.max_results,\n self.search_term,\n self.location,\n self.raw_location,\n ) = (None, None, None, None, None, None, None, None, None)\n self.on_block = False\n self.connect_block = False\n self.certs = CertificationFetcher(self.session)\n self.skills = SkillsFetcher(self.session)\n self.employees = EmployeeFetcher(self.session)\n self.schools = SchoolsFetcher(self.session)\n self.experiences = ExperiencesFetcher(self.session)\n self.bio = EmployeeBioFetcher(self.session)\n self.languages = LanguagesFetcher(self.session)\n self.contact = ContactInfoFetcher(self.session)\n\n def search_companies(self, company_name: str):\n \"\"\"Get the company id and staff count from the company name.\"\"\"\n\n company_search_ep = self.company_search_ep.format(company=quote(company_name))\n self.session.headers[\"x-li-graphql-pegasus-client\"] = \"true\"\n res = self.session.get(company_search_ep)\n self.session.headers.pop(\"x-li-graphql-pegasus-client\", \"\")\n if not res.ok:\n raise Exception(\n f\"Failed to search for company {company_name}\",\n res.status_code,\n res.text[:200],\n )\n logger.debug(\n f\"Searched companies for name '{company_name}' - res code {res.status_code}-\"\n )\n companies = res.json()[\"data\"][\"searchDashClustersByAll\"][\"elements\"]\n\n err_msg = f\"No companies found for name {company_name}\"\n if len(companies) < 2:\n raise Exception(err_msg)\n try:\n num_results = companies[0][\"items\"][0][\"item\"][\"simpleTextV2\"][\"text\"][\n \"text\"\n ]\n first_company = companies[1][\"items\"][0][\"item\"].get(\"entityResult\")\n if not first_company and len(companies) > 2:\n first_company = companies[2][\"items\"][0][\"item\"].get(\"entityResult\")\n if not first_company:\n raise Exception(err_msg)\n\n company_link = first_company[\"navigationUrl\"]\n company_name_id = unquote(\n re.search(r\"/company/([^/]+)\", company_link).group(1)\n )\n company_name_new = first_company[\"title\"][\"text\"]\n except Exception as e:\n raise Exception(\n f\"Failed to load json in search_companies {str(e)}, Response: {res.text[:200]}\"\n )\n\n logger.info(\n f\"Searched company {company_name} on LinkedIn and were {num_results}, using first result with company name - '{company_name_new}' and company id - '{company_name_id}'\"\n )\n return company_name_id\n\n def fetch_or_search_company(self, company_name):\n \"\"\"Fetch the company details by name, or search if not found.\"\"\"\n res = self.session.get(f\"{self.company_id_ep}{company_name}\")\n\n if res.status_code not in (200, 404):\n raise Exception(\n f\"Failed to find company {company_name} (likely due to outdated login if you know it's valid company)\",\n res.status_code,\n res.text[:200],\n )\n elif res.status_code == 404:\n logger.info(\n f\"Failed to directly use company '{company_name}' as company id, now searching for the company\"\n )\n company_name = self.search_companies(company_name)\n res = self.session.get(f\"{self.company_id_ep}{company_name}\")\n if res.status_code != 200:\n raise Exception(\n f\"Failed to find company after performing a direct and generic search for {company_name}\",\n res.status_code,\n res.text[:200],\n )\n\n if not res.ok:\n logger.debug(f\"res code {res.status_code} - fetched company \")\n return res\n\n def _get_company_id_and_staff_count(self, company_name: str):\n \"\"\"Extract company id and staff count from the company details.\"\"\"\n res = self.fetch_or_search_company(company_name)\n\n try:\n response_json = res.json()\n except json.decoder.JSONDecodeError:\n logger.debug(res.text[:200])\n raise Exception(\n f\"Failed to load json in get_company_id_and_staff_count {res.text[:200]}\"\n )\n\n company = response_json[\"elements\"][0]\n self.domain = (\n utils.extract_base_domain(company[\"companyPageUrl\"])\n if company.get(\"companyPageUrl\")\n else None\n )\n staff_count = company[\"staffCount\"]\n company_id = company[\"trackingInfo\"][\"objectUrn\"].split(\":\")[-1]\n company_name = company[\"universalName\"]\n\n logger.info(f\"Found company '{company_name}' with {staff_count} staff\")\n return company_id, staff_count\n\n def parse_staff(self, elements: list[dict]):\n \"\"\"Parse the staff from the search results\"\"\"\n staff = []\n\n for elem in elements:\n for card in elem.get(\"items\", []):\n person = card.get(\"item\", {}).get(\"entityResult\", {})\n if not person:\n continue\n pattern = (\n r\"urn:li:fsd_profile:([^,]+),(?:SEARCH_SRP|MYNETWORK_CURATION_HUB)\"\n )\n match = re.search(pattern, person[\"entityUrn\"])\n linkedin_id = match.group(1) if match else None\n person_urn = person[\"trackingUrn\"].split(\":\")[-1]\n\n name = person[\"title\"][\"text\"].strip()\n headline = (\n person.get(\"primarySubtitle\", {}).get(\"text\", \"\")\n if person.get(\"primarySubtitle\")\n else \"\"\n )\n profile_link = person[\"navigationUrl\"].split(\"?\")[0]\n staff.append(\n Staff(\n urn=person_urn,\n id=linkedin_id,\n name=name,\n headline=headline,\n search_term=\" - \".join(\n filter(\n None,\n [\n self.company_name,\n self.search_term,\n self.raw_location,\n ],\n )\n ),\n profile_link=profile_link,\n )\n )\n return staff\n\n def fetch_staff(self, offset: int):\n \"\"\"Fetch the staff using LinkedIn search\"\"\"\n ep = self.employees_ep.format(\n offset=offset,\n company_id=(\n f\"(key:currentCompany,value:List({self.company_id})),\"\n if self.company_id\n else \"\"\n ),\n count=50,\n search=f\"keywords:{quote(self.search_term)},\" if self.search_term else \"\",\n location=(\n f\"(key:geoUrn,value:List({self.location})),\" if self.location else \"\"\n ),\n )\n res = self.session.get(ep)\n if not res.ok:\n logger.debug(f\"employees, status code - {res.status_code}\")\n if res.status_code == 400:\n raise BadCookies(\"Outdated login, delete the session file to log in again\")\n elif res.status_code == 429:\n raise TooManyRequests(\"429 Too Many Requests\")\n if not res.ok:\n return None, 0\n try:\n res_json = res.json()\n except json.decoder.JSONDecodeError:\n logger.debug(res.text)\n return None, 0\n\n try:\n elements = res_json[\"data\"][\"searchDashClustersByAll\"][\"elements\"]\n total_count = res_json[\"data\"][\"searchDashClustersByAll\"][\"metadata\"][\n \"totalResultCount\"\n ]\n\n except (KeyError, IndexError, TypeError):\n logger.debug(res_json)\n return None, 0\n new_staff = self.parse_staff(elements) if elements else []\n return new_staff, total_count\n\n def fetch_connections_page(self, offset: int):\n self.session.headers[\"x-li-graphql-pegasus-client\"] = \"true\"\n res = self.session.get(self.connections_ep.format(offset=offset))\n self.session.headers.pop(\"x-li-graphql-pegasus-client\", \"\")\n if not res.ok:\n logger.debug(f\"employees, status code - {res.status_code}\")\n if res.status_code == 400:\n raise BadCookies(\"Outdated login, delete the session file to log in again\")\n elif res.status_code == 429:\n raise TooManyRequests(\"429 Too Many Requests\")\n if not res.ok:\n return\n try:\n res_json = res.json()\n except json.decoder.JSONDecodeError:\n logger.debug(res.text)\n return\n\n try:\n elements = res_json[\"data\"][\"searchDashClustersByAll\"][\"elements\"]\n total_count = res_json[\"data\"][\"searchDashClustersByAll\"][\"metadata\"][\n \"totalResultCount\"\n ]\n\n except (KeyError, IndexError, TypeError):\n logger.debug(res_json)\n return\n\n new_staff = self.parse_staff(elements) if elements else []\n return new_staff, total_count\n\n def scrape_connections(\n self,\n max_results: int = 10**8,\n extra_profile_data: bool = False,\n ):\n self.search_term = \"connections\"\n staff_list: list[Staff] = []\n\n try:\n initial_staff, total_search_result_count = self.fetch_connections_page(0)\n if initial_staff:\n staff_list.extend(initial_staff)\n\n self.num_staff = min(total_search_result_count, max_results)\n for offset in range(50, self.num_staff, 50):\n staff, _ = self.fetch_connections_page(offset)\n logger.debug(\n f\"Connections from search: {len(staff)} new, {len(staff_list) + len(staff)} total\"\n )\n if not staff:\n break\n staff_list.extend(staff)\n except (BadCookies, TooManyRequests) as e:\n self.on_block = True\n logger.error(f\"Exiting early due to fatal error: {str(e)}\")\n return staff_list[:max_results]\n\n reduced_staff_list = staff_list[:max_results]\n\n non_restricted = list(\n filter(lambda x: x.name != \"LinkedIn Member\", reduced_staff_list)\n )\n\n if extra_profile_data:\n try:\n for i, employee in enumerate(non_restricted, start=1):\n self.fetch_all_info_for_employee(employee, i)\n except TooManyRequests as e:\n logger.error(str(e))\n return reduced_staff_list\n\n def fetch_location_id(self):\n \"\"\"Fetch the location id for the location to be used in LinkedIn search\"\"\"\n ep = self.location_id_ep.format(location=quote(self.raw_location))\n res = self.session.get(ep)\n try:\n res_json = res.json()\n except json.decoder.JSONDecodeError:\n if res.reason == \"INKApi Error\":\n raise Exception(\n \"Delete session file and log in again\",\n res.status_code,\n res.text[:200],\n res.reason,\n )\n raise GeoUrnNotFound(\n \"Failed to send request to get geo id\",\n res.status_code,\n res.text[:200],\n res.reason,\n )\n\n try:\n elems = res_json[\"data\"][\"searchDashReusableTypeaheadByType\"][\"elements\"]\n except (KeyError, IndexError, TypeError):\n raise GeoUrnNotFound(\"Failed to locate geo id\", res_json[:200])\n\n geo_id = None\n if elems:\n urn = elems[0][\"trackingUrn\"]\n m = re.search(\"urn:li:geo:(.+)\", urn)\n if m:\n geo_id = m.group(1)\n if not geo_id:\n raise GeoUrnNotFound(\"Failed to parse geo id\")\n self.location = geo_id\n\n def scrape_staff(\n self,\n company_name: str | None,\n search_term: str,\n location: str,\n extra_profile_data: bool,\n max_results: int,\n block: bool,\n connect: bool,\n ):\n \"\"\"Main function entry point to scrape LinkedIn staff\"\"\"\n self.search_term = search_term\n self.company_name = company_name\n self.max_results = max_results\n self.raw_location = location\n self.company_id = None\n\n if self.company_name:\n self.company_id, staff_count = self._get_company_id_and_staff_count(\n company_name\n )\n\n staff_list: list[Staff] = []\n\n if self.raw_location:\n try:\n self.fetch_location_id()\n except GeoUrnNotFound as e:\n logger.error(str(e))\n return staff_list[:max_results]\n\n try:\n initial_staff, total_count = self.fetch_staff(0)\n if initial_staff:\n staff_list.extend(initial_staff)\n location = f\", location: '{location}'\" if location else \"\"\n logger.info(\n f\"1) Search results for company: '{company_name}'{location} - {total_count:,} staff\"\n )\n\n self.num_staff = min(total_count, max_results, 1000)\n for offset in range(50, self.num_staff, 50):\n staff, _ = self.fetch_staff(offset)\n logger.debug(\n f\"Staff members from search: {len(staff)} new, {len(staff_list) + len(staff)} total\"\n )\n if not staff:\n break\n staff_list.extend(staff)\n location = f\", location: '{location}'\" if location else \"\"\n logger.info(\n f\"2) Total results collected for company: '{company_name}'{location} - {len(staff_list)} results\"\n )\n except (BadCookies, TooManyRequests) as e:\n self.on_block = True\n logger.error(f\"Exiting early due to fatal error: {str(e)}\")\n return staff_list[:max_results]\n\n reduced_staff_list = staff_list[:max_results]\n non_restricted = list(\n filter(lambda x: x.name != \"LinkedIn Member\", reduced_staff_list)\n )\n\n if extra_profile_data:\n try:\n for i, employee in enumerate(non_restricted, start=1):\n self.fetch_all_info_for_employee(employee, i)\n if block:\n self.block_user(employee)\n elif connect:\n self.connect_user(employee)\n\n except TooManyRequests as e:\n logger.error(str(e))\n\n return reduced_staff_list\n\n def fetch_all_info_for_employee(self, employee: Staff, index: int):\n \"\"\"Simultaniously fetch all the data for an employee\"\"\"\n logger.info(\n f\"Fetching data for account {employee.id} {index:>4} / {self.num_staff} - {employee.profile_link}\"\n )\n\n task_functions = [\n (self.employees.fetch_employee, (employee, self.domain), \"employee\"),\n (self.skills.fetch_skills, (employee,), \"skills\"),\n (self.experiences.fetch_experiences, (employee,), \"experiences\"),\n (self.certs.fetch_certifications, (employee,), \"certifications\"),\n (self.schools.fetch_schools, (employee,), \"schools\"),\n (self.bio.fetch_employee_bio, (employee,), \"bio\"),\n (self.languages.fetch_languages, (employee,), \"languages\"),\n ]\n\n with ThreadPoolExecutor(max_workers=len(task_functions)) as executor:\n tasks = {\n executor.submit(func, *args): name\n for func, args, name in task_functions\n }\n\n for future in as_completed(tasks):\n result = future.result()\n\n if employee.is_connection:\n self.contact.fetch_contact_info(employee)\n\n def fetch_user_profile_data_from_public_id(self, user_id: str, key: str):\n \"\"\"Fetches data given the public LinkedIn user id\"\"\"\n endpoint = self.public_user_id_ep.format(user_id=user_id)\n response = self.session.get(endpoint)\n\n try:\n response_json = response.json()\n except json.decoder.JSONDecodeError:\n logger.debug(response.text[:200])\n raise Exception(\n f\"Failed to load JSON from endpoint\",\n response.status_code,\n response.reason,\n )\n\n keys = {\n \"user_id\": (\"positionView\", \"profileId\"),\n \"company_id\": (\n \"positionView\",\n \"elements\",\n 0,\n \"company\",\n \"miniCompany\",\n \"universalName\",\n ),\n }\n\n try:\n data = response_json\n for k in keys[key]:\n data = data[k]\n urn = response_json[\"profile\"][\"miniProfile\"][\"objectUrn\"].split(\":\")[-1]\n return data, urn\n except (KeyError, TypeError, IndexError) as e:\n logger.warning(f\"Failed to find user_id {user_id}\")\n if key == \"user_id\":\n return \"\"\n raise Exception(f\"Failed to fetch '{key}' for user_id {user_id}: {e}\")\n\n def block_user(self, employee: Staff) -> None:\n \"\"\"Block a user on LinkedIn given their urn\"\"\"\n if employee.urn == \"headless\":\n return\n self.session.headers[\"Content-Type\"] = (\n \"application/x-protobuf2; symbol-table=voyager-20757\"\n )\n\n urn_string = f\"urn:li:member:{employee.urn}\"\n length_byte = bytes([len(urn_string)])\n body = b\"\\x00\\x01\\x14\\nblockeeUrn\\x14\" + length_byte + urn_string.encode()\n\n res = self.session.post(\n self.block_user_ep,\n data=body,\n )\n self.session.headers.pop(\"Content-Type\", \"\")\n\n if res.ok:\n logger.info(f\"Successfully blocked user {employee.id}\")\n elif res.status_code == 403:\n logger.warning(\n f\"Failed to block user - status code 403, one possible reason is you have alread blocked/unblocked this person in past 48 hours and on cooldown: {employee.profile_link}\"\n )\n else:\n logger.warning(\n f\"Failed to block user - status code {res.status_code} {employee.id}: {employee.name}\"\n )\n\n def connect_user(self, employee: Staff) -> None:\n \"\"\"Connects with a user on LinkedIn given their profile id\"\"\"\n if self.connect_block:\n return logger.info(\n f\"Skipping connection request for user due to previou block: {employee.id} - {employee.profile_link} \"\n )\n if employee.urn == \"headless\":\n return\n if employee.is_connection != \"no\":\n return logger.info(\n f\"Already connected or pending connection request to user {employee.id} - {employee.profile_link}\"\n )\n self.session.headers[\"Content-Type\"] = (\n \"application/x-protobuf2; symbol-table=voyager-20757\"\n )\n body = (\n b\"\\x00\\x01\\x03\\xe2\\x05\\x00\\x01\\x03\\xd3w\\x00\\x01\\x03\\xd5\\x06\\x14:urn:li:fsd_profile:\"\n + employee.id.encode()\n )\n\n res = self.session.post(\n self.connect_to_user_ep,\n data=body,\n )\n self.session.headers.pop(\"Content-Type\", \"\")\n\n if res.ok:\n logger.info(\n f\"Successfully sent connection request to user {employee.id} - {employee.profile_link}\"\n )\n elif res.status_code == 429:\n self.connect_block = True\n logger.warning(\n f\"Failed to connect to user - status code 429 - pausing connection requests for this scrape: {employee.id} - {employee.profile_link}\"\n )\n else:\n logger.warning(\n f\"Failed to connect to user - status code {res.status_code} {employee.id} -{employee.profile_link}\"\n )\n" }, { "path": "staffspy/linkedin/schools.py", "content": "import json\nimport logging\n\nfrom staffspy.utils.exceptions import TooManyRequests\nfrom staffspy.utils.models import School\nfrom staffspy.utils.utils import parse_dates\n\nlogger = logging.getLogger(__name__)\n\n\nclass SchoolsFetcher:\n\n def __init__(self, session):\n self.session = session\n self.endpoint = \"https://www.linkedin.com/voyager/api/graphql?queryId=voyagerIdentityDashProfileComponents.277ba7d7b9afffb04683953cede751fb&queryName=ProfileComponentsBySectionType&variables=(tabIndex:0,sectionType:education,profileUrn:urn%3Ali%3Afsd_profile%3A{employee_id},count:50)\"\n\n def fetch_schools(self, staff):\n ep = self.endpoint.format(employee_id=staff.id)\n res = self.session.get(ep)\n logger.debug(f\"schools, status code - {res.status_code}\")\n if res.status_code == 429:\n return TooManyRequests(\"429 Too Many Requests\")\n\n if not res.ok:\n logger.debug(res.text[:200])\n return False\n try:\n res_json = res.json()\n except json.decoder.JSONDecodeError:\n logger.debug(res.text[:200])\n return False\n\n try:\n elements = res_json[\"data\"][\"identityDashProfileComponentsBySectionType\"][\n \"elements\"\n ][0][\"components\"][\"pagedListComponent\"][\"components\"][\"elements\"]\n except (KeyError, IndexError, TypeError) as e:\n logger.debug(res_json)\n return False\n\n staff.schools = self.parse_schools(elements)\n return True\n\n def parse_schools(self, elements):\n schools = []\n start = end = None\n for elem in elements:\n entity = elem[\"components\"][\"entityComponent\"]\n if not entity:\n break\n years = entity[\"caption\"][\"text\"] if entity[\"caption\"] else None\n school_name = entity[\"titleV2\"][\"text\"][\"text\"]\n\n if years:\n start, end = parse_dates(years)\n degree = entity[\"subtitle\"][\"text\"] if entity[\"subtitle\"] else None\n school = School(\n start_date=start, end_date=end, school=school_name, degree=degree\n )\n schools.append(school)\n\n return schools\n" }, { "path": "staffspy/linkedin/skills.py", "content": "import json\nimport logging\n\nfrom staffspy.utils.exceptions import TooManyRequests\nfrom staffspy.utils.models import Skill, Staff\n\nlogger = logging.getLogger(__name__)\n\n\nclass SkillsFetcher:\n def __init__(self, session):\n self.session = session\n self.endpoint = \"https://www.linkedin.com/voyager/api/graphql?queryId=voyagerIdentityDashProfileComponents.277ba7d7b9afffb04683953cede751fb&queryName=ProfileComponentsBySectionType&variables=(tabIndex:0,sectionType:skills,profileUrn:urn%3Ali%3Afsd_profile%3A{employee_id},count:50)\"\n\n def fetch_skills(self, staff: Staff):\n ep = self.endpoint.format(employee_id=staff.id)\n res = self.session.get(ep)\n logger.debug(f\"skills, status code - {res.status_code}\")\n if res.status_code == 429:\n return TooManyRequests(\"429 Too Many Requests\")\n if not res.ok:\n logger.debug(res.text[:200])\n return False\n try:\n res_json = res.json()\n except json.decoder.JSONDecodeError:\n logger.debug(res.text[:200])\n return False\n\n if res_json.get(\"errors\"):\n return False\n tab_comp = res_json[\"data\"][\"identityDashProfileComponentsBySectionType\"][\n \"elements\"\n ][0][\"components\"][\"tabComponent\"]\n if tab_comp:\n sections = tab_comp[\"sections\"]\n staff.skills = self.parse_skills(sections)\n return True\n\n def parse_skills(self, sections):\n names = set()\n skills = []\n for section in sections:\n elems = section[\"subComponent\"][\"components\"][\"pagedListComponent\"][\n \"components\"\n ][\"elements\"]\n for elem in elems:\n passed_assessment, endorsements = None, 0\n entity = elem[\"components\"][\"entityComponent\"]\n name = entity[\"titleV2\"][\"text\"][\"text\"]\n if name in names:\n continue\n names.add(name)\n components = entity[\"subComponents\"][\"components\"]\n for component in components:\n\n try:\n candidate = component[\"components\"][\"insightComponent\"][\"text\"][\n \"text\"\n ][\"text\"]\n if \" endorsements\" in candidate:\n endorsements = int(candidate.replace(\" endorsements\", \"\"))\n if \"Passed LinkedIn Skill Assessment\" in candidate:\n passed_assessment = True\n except:\n pass\n\n skills.append(\n Skill(\n name=name,\n endorsements=endorsements,\n passed_assessment=passed_assessment,\n )\n )\n return skills\n" }, { "path": "staffspy/solvers/capsolver.py", "content": "import json\nimport time\n\nimport requests\nfrom tenacity import retry, stop_after_attempt, retry_if_result\n\nfrom staffspy.solvers.solver import Solver\n\n\ndef is_none(value):\n return value is None\n\n\nclass CapSolver(Solver):\n \"\"\"https://www.capsolver.com/\"\"\"\n\n @retry(stop=stop_after_attempt(10), retry=retry_if_result(is_none))\n def solve(self, blob_data: str, page_url: str = None):\n from staffspy.utils.utils import logger\n\n logger.info(f\"Waiting on CapSolver to solve captcha...\")\n\n payload = {\n \"clientKey\": self.solver_api_key,\n \"task\": {\n \"type\": \"FunCaptchaTaskProxyLess\",\n \"websitePublicKey\": self.public_key,\n \"websiteURL\": self.page_url,\n \"data\": json.dumps({\"blob\": blob_data}) if blob_data else \"\",\n },\n }\n res = requests.post(\"https://api.capsolver.com/createTask\", json=payload)\n resp = res.json()\n task_id = resp.get(\"taskId\")\n if not task_id:\n raise Exception(\n \"CapSolver failed to create task, try another captcha solver like 2Captcha if this persists or use browser sign in `pip install staffspy[browser]` and then remove the username/password params to the LinkedInAccount()\",\n res.text,\n )\n logger.info(f\"Received captcha solver taskId: {task_id} / Getting result...\")\n\n while True:\n time.sleep(1) # delay\n payload = {\"clientKey\": self.solver_api_key, \"taskId\": task_id}\n res = requests.post(\"https://api.capsolver.com/getTaskResult\", json=payload)\n resp = res.json()\n status = resp.get(\"status\")\n if status == \"ready\":\n logger.info(f\"CapSolver finished solving captcha\")\n return resp.get(\"solution\", {}).get(\"token\")\n if status == \"failed\" or resp.get(\"errorId\"):\n logger.info(f\"Captcha solve failed! response: {res.text}\")\n return None\n" }, { "path": "staffspy/solvers/solver.py", "content": "from abc import ABC,abstractmethod\n\n\nclass Solver(ABC):\n public_key = \"3117BF26-4762-4F5A-8ED9-A85E69209A46\"\n page_url = \"https://iframe.arkoselabs.com\"\n\n def __init__(self, solver_api_key:str):\n self.solver_api_key=solver_api_key\n\n @abstractmethod\n def solve(self, blob_data: str, page_ur: str=None):\n pass\n" }, { "path": "staffspy/solvers/solver_type.py", "content": "from enum import Enum\n\nclass SolverType(Enum):\n CAPSOLVER = 'capsolver'\n TWO_CAPTCHA = 'twocaptcha'\n" }, { "path": "staffspy/solvers/two_captcha.py", "content": "from tenacity import retry_if_exception_type, stop_after_attempt, retry\nfrom twocaptcha import TwoCaptcha, TimeoutException, ApiException, NetworkException\n\nfrom staffspy.solvers.solver import Solver\n\n\nclass TwoCaptchaSolver(Solver):\n \"\"\"https://2captcha.com/\"\"\"\n\n attempt = 1\n\n @retry(\n stop=stop_after_attempt(5),\n retry=retry_if_exception_type(\n (TimeoutException, ApiException, NetworkException)\n ),\n )\n def solve(self, blob_data: str, page_url: str = None):\n super().solve(blob_data, page_url)\n from staffspy.utils.utils import logger\n\n logger.info(\n f\"Waiting on 2Captcha to solve captcha attempt {self.attempt} / 5 ...\"\n )\n self.attempt += 1\n\n solver = TwoCaptcha(self.solver_api_key)\n\n result = solver.funcaptcha(\n sitekey=self.public_key,\n url=page_url,\n **{\"data[blob]\": blob_data},\n surl=\"https://iframe.arkoselabs.com\",\n )\n logger.info(f\"2Captcha finished solving captcha\")\n return result[\"code\"]\n" }, { "path": "staffspy/utils/driver_type.py", "content": "from enum import Enum\nfrom typing import Optional\n\n\nclass BrowserType(Enum):\n CHROME = \"chrome\"\n FIREFOX = \"firefox\"\n\n\nclass DriverType:\n def __init__(\n self, browser_type: BrowserType, executable_path: Optional[str] = None\n ):\n self.browser_type = browser_type\n self.executable_path = executable_path\n" }, { "path": "staffspy/utils/exceptions.py", "content": "class TooManyRequests(Exception):\n \"\"\"Too many requests.\"\"\"\n\n\nclass BadCookies(Exception):\n \"\"\"Login expiration.\"\"\"\n\n\nclass GeoUrnNotFound(Exception):\n \"\"\"Could not find geo urn for given location.\"\"\"\n\n\nclass BlobException(Exception):\n \"\"\"Could not find the blob needed to solve the captcha.\"\"\"\n" }, { "path": "staffspy/utils/models.py", "content": "from datetime import datetime, date\n\nfrom pydantic import BaseModel\nfrom datetime import datetime as dt\n\nfrom staffspy.utils.utils import extract_emails_from_text\n\n\nclass Comment(BaseModel):\n post_id: str\n comment_id: str | None = None\n internal_profile_id: str | None = None\n public_profile_id: str | None = None\n name: str | None = None\n text: str | None = None\n num_likes: int | None = None\n created_at: dt | None = None\n\n def to_dict(self):\n return {\n \"post_id\": self.post_id,\n \"comment_id\": self.comment_id,\n \"internal_profile_id\": self.internal_profile_id,\n \"public_profile_id\": self.public_profile_id,\n \"name\": self.name,\n \"text\": self.text,\n \"num_likes\": self.num_likes,\n \"created_at\": self.created_at,\n }\n\n\nclass School(BaseModel):\n start_date: date | None = None\n end_date: date | None = None\n school: str | None = None\n degree: str | None = None\n\n def to_dict(self):\n return {\n \"start_date\": self.start_date.isoformat() if self.start_date else None,\n \"end_date\": self.end_date.isoformat() if self.end_date else None,\n \"school\": self.school,\n \"degree\": self.degree,\n }\n\n\nclass Skill(BaseModel):\n name: str | None = None\n endorsements: int | None = None\n passed_assessment: bool | None = None\n\n def to_dict(self):\n return {\n \"name\": self.name,\n \"endorsements\": self.endorsements if self.endorsements else 0,\n \"passed_assessment\": self.passed_assessment,\n }\n\n\nclass ContactInfo(BaseModel):\n email_address: str | None = None\n websites: list | None = None\n phone_numbers: list | None = None\n address: str | None = None\n birthday: str | None = None\n created_at: str | None = None\n\n def to_dict(self):\n return {\n \"email_address\": self.email_address,\n \"websites\": self.websites,\n \"phone_numbers\": self.phone_numbers,\n \"address\": self.address,\n \"birthday\": self.birthday,\n \"created_at\": self.created_at,\n }\n\n\nclass Certification(BaseModel):\n title: str | None = None\n issuer: str | None = None\n date_issued: str | None = None\n cert_id: str | None = None\n cert_link: str | None = None\n\n def to_dict(self):\n return {\n \"title\": self.title,\n \"issuer\": self.issuer,\n \"date_issued\": self.date_issued,\n \"cert_id\": self.cert_id,\n \"cert_link\": self.cert_link,\n }\n\n\nclass Experience(BaseModel):\n duration: str | None = None\n title: str | None = None\n company: str | None = None\n location: str | None = None\n emp_type: str | None = None\n start_date: date | None = None\n end_date: date | None = None\n\n def to_dict(self):\n return {\n \"start_date\": self.start_date.isoformat() if self.start_date else None,\n \"end_date\": self.end_date.isoformat() if self.end_date else None,\n \"duration\": self.duration,\n \"title\": self.title,\n \"company\": self.company,\n \"location\": self.location,\n \"emp_type\": self.emp_type,\n }\n\n\nclass Staff(BaseModel):\n urn: str | None = None\n search_term: str\n id: str\n name: str | None = None\n headline: str | None = None\n current_position: str | None = None\n\n profile_id: str | None = None\n profile_link: str | None = None\n first_name: str | None = None\n last_name: str | None = None\n potential_emails: list | None = None\n bio: str | None = None\n emails_in_bio: str | None = None\n followers: int | None = None\n connections: int | None = None\n mutual_connections: int | None = None\n is_connection: str | None = None # yes, no, pending\n location: str | None = None\n company: str | None = None\n school: str | None = None\n influencer: bool | None = None\n creator: bool | None = None\n premium: bool | None = None\n open_to_work: bool | None = None\n is_hiring: bool | None = None\n profile_photo: str | None = None\n banner_photo: str | None = None\n skills: list[Skill] | None = None\n experiences: list[Experience] | None = None\n certifications: list[Certification] | None = None\n contact_info: ContactInfo | None = None\n schools: list[School] | None = None\n languages: list[str] | None = None\n\n def get_top_skills(self):\n top_three_skills = []\n if self.skills:\n sorted_skills = sorted(\n self.skills, key=lambda x: x.endorsements, reverse=True\n )\n top_three_skills = [skill.name for skill in sorted_skills[:3]]\n top_three_skills += [None] * (3 - len(top_three_skills))\n return top_three_skills\n\n def to_dict(self):\n sorted_schools = (\n sorted(\n self.schools,\n key=lambda x: (x.end_date is None, x.end_date),\n reverse=True,\n )\n if self.schools\n else []\n )\n\n top_three_school_names = [school.school for school in sorted_schools[:3]]\n top_three_school_names += [None] * (3 - len(top_three_school_names))\n estimated_age = self.estimate_age_based_on_education()\n\n sorted_experiences = (\n sorted(\n self.experiences,\n key=lambda x: (x.end_date is None, x.end_date),\n reverse=True,\n )\n if self.experiences\n else []\n )\n\n top_three_companies = []\n seen_companies = set()\n for exp in sorted_experiences:\n if exp.company not in seen_companies:\n top_three_companies.append(exp.company)\n seen_companies.add(exp.company)\n if len(top_three_companies) == 3:\n break\n\n top_three_companies += [None] * (3 - len(top_three_companies))\n top_three_skills = self.get_top_skills()\n self.emails_in_bio = extract_emails_from_text(self.bio) if self.bio else None\n self.current_position = (\n sorted_experiences[0].title\n if len(sorted_experiences) > 0 and sorted_experiences[0].end_date is None\n else None\n )\n\n contact_info = self.contact_info.to_dict() if self.contact_info else {}\n return {\n \"search_term\": self.search_term,\n \"id\": self.id,\n \"urn\": self.urn,\n \"profile_link\": self.profile_link,\n \"profile_id\": self.profile_id,\n \"name\": self.name,\n \"first_name\": self.first_name,\n \"last_name\": self.last_name,\n \"location\": self.location,\n \"headline\": self.headline,\n \"estimated_age\": estimated_age,\n \"followers\": self.followers,\n \"connections\": self.connections,\n \"mutuals\": self.mutual_connections,\n \"is_connection\": self.is_connection,\n \"premium\": self.premium,\n \"creator\": self.creator,\n \"influencer\": self.influencer,\n \"open_to_work\": self.open_to_work,\n \"is_hiring\": self.is_hiring,\n \"current_position\": self.current_position,\n \"current_company\": top_three_companies[0],\n \"past_company_1\": top_three_companies[1],\n \"past_company_2\": top_three_companies[2],\n \"school_1\": top_three_school_names[0],\n \"school_2\": top_three_school_names[1],\n \"top_skill_1\": top_three_skills[0],\n \"top_skill_2\": top_three_skills[1],\n \"top_skill_3\": top_three_skills[2],\n \"bio\": self.bio,\n \"experiences\": (\n [exp.to_dict() for exp in self.experiences]\n if self.experiences\n else None\n ),\n \"schools\": (\n [school.to_dict() for school in self.schools] if self.schools else None\n ),\n \"skills\": (\n [skill.to_dict() for skill in self.skills] if self.skills else None\n ),\n \"certifications\": (\n [cert.to_dict() for cert in self.certifications]\n if self.certifications\n else None\n ),\n \"languages\": self.languages,\n \"emails_in_bio\": (\n \", \".join(self.emails_in_bio) if self.emails_in_bio else None\n ),\n \"potential_emails\": self.potential_emails,\n \"profile_photo\": self.profile_photo,\n \"banner_photo\": self.banner_photo,\n \"connection_created_at\": contact_info.get(\"created_at\"),\n \"connection_email\": contact_info.get(\"email_address\"),\n \"connection_phone_numbers\": contact_info.get(\"phone_numbers\"),\n \"connection_websites\": contact_info.get(\"websites\"),\n \"connection_street_address\": contact_info.get(\"address\"),\n \"connection_birthday\": contact_info.get(\"birthday\"),\n }\n\n def estimate_age_based_on_education(self):\n \"\"\"Adds 18 to their first college start date\"\"\"\n college_words = [\"uni\", \"college\"]\n\n sorted_schools = (\n sorted(\n [school for school in self.schools if school.start_date],\n key=lambda x: x.start_date,\n )\n if self.schools\n else []\n )\n\n current_date = datetime.now().date()\n for school in sorted_schools:\n if (\n any(word in school.school.lower() for word in college_words)\n or school.degree\n ):\n if school.start_date:\n years_in_education = (current_date - school.start_date).days // 365\n return int(18 + years_in_education)\n return None\n" }, { "path": "staffspy/utils/utils.py", "content": "import logging\nimport os\nimport pickle\nimport re\nfrom datetime import datetime\n\nimport pandas as pd\nfrom typing import Optional\nfrom urllib.parse import quote\n\nimport requests\nimport tldextract\nfrom bs4 import BeautifulSoup\nfrom dateutil.parser import parse\nfrom tenacity import stop_after_attempt, retry_if_exception_type, retry, RetryError\n\nfrom staffspy.solvers.solver import Solver\nfrom staffspy.utils.driver_type import DriverType, BrowserType\nfrom staffspy.utils.exceptions import BlobException\n\nlogger = logging.getLogger(\"StaffSpy\")\nlogger.propagate = False\nif not logger.handlers:\n logger.setLevel(logging.INFO)\n console_handler = logging.StreamHandler()\n format = \"%(asctime)s - %(name)s - %(levelname)s - %(message)s\"\n formatter = logging.Formatter(format)\n console_handler.setFormatter(formatter)\n logger.addHandler(console_handler)\n\n\ndef set_csrf_token(session):\n csrf_token = session.cookies[\"JSESSIONID\"].replace('\"', \"\")\n session.headers.update({\"Csrf-Token\": csrf_token})\n return session\n\n\ndef extract_base_domain(url: str):\n extracted = tldextract.extract(url)\n base_domain = \"{}.{}\".format(extracted.domain, extracted.suffix)\n return base_domain\n\n\ndef create_emails(first, last, domain):\n first = \"\".join(filter(str.isalpha, first)).lower()\n last = \"\".join(filter(str.isalpha, last)).lower()\n emails = [\n f\"{first}.{last}@{domain}\",\n f\"{first[:1]}{last}@{domain}\",\n f\"{first[:2]}{last}@{domain}\",\n f\"{first}{last[:1]}@{domain}\",\n f\"{first}{last[:2]}@{domain}\",\n ]\n return emails\n\n\ndef get_webdriver(driver_type: Optional[DriverType] = None):\n try:\n from selenium import webdriver\n from selenium.webdriver.chrome.service import Service as ChromeService\n from selenium.webdriver.firefox.service import Service as FirefoxService\n except ImportError as e:\n raise Exception(\n 'install package `pip install \"staffspy[browser]\"` to login with browser'\n )\n\n if driver_type:\n if str(driver_type.browser_type) == str(BrowserType.CHROME):\n if driver_type.executable_path:\n service = ChromeService(executable_path=driver_type.executable_path)\n return webdriver.Chrome(service=service)\n else:\n return webdriver.Chrome()\n elif str(driver_type.browser_type) == str(BrowserType.FIREFOX):\n if driver_type.executable_path:\n service = FirefoxService(executable_path=driver_type.executable_path)\n return webdriver.Firefox(service=service)\n else:\n return webdriver.Firefox()\n else:\n for browser in [webdriver.Chrome, webdriver.Firefox]:\n try:\n return browser()\n except Exception:\n continue\n return None\n\n\nclass Login:\n\n def __init__(\n self,\n username: str,\n password: str,\n solver: Solver,\n session_file: str,\n driver_type: DriverType = None,\n ):\n (\n self.username,\n self.password,\n self.solver,\n self.session_file,\n self.driver_type,\n ) = (username, password, solver, session_file, driver_type)\n\n def solve_captcha(self, session, data, payload):\n url = data[\"challenge_url\"]\n r = session.post(url, data=payload)\n\n soup = BeautifulSoup(r.text, \"html.parser\")\n\n code_tag = soup.find(\"code\", id=\"securedDataExchange\")\n\n logger.info(\"Searching for captcha blob in linkedin to begin captcha solving\")\n if code_tag:\n comment = code_tag.contents[0]\n extracted_code = str(comment).strip('').strip()\n logger.debug(\"Extracted captcha blob:\", extracted_code)\n elif \"Please choose a more secure password.\" in r.text:\n raise Exception(\n \"linkedin is requiring a more secure password. reset pw and try again\"\n )\n else:\n raise BlobException(\n \"blob to solve captcha not found - rerunning the program usually solves this\"\n )\n\n if not self.solver:\n raise Exception(\n \"captcha hit - provide solver_api_key and solver_service name to solve or switch to the browser-based login with `pip install staffspy[browser]`\"\n )\n token = self.solver.solve(extracted_code, url)\n if not token:\n raise Exception(\"failed to solve captcha after 10 attempts\")\n\n captcha_site_key = soup.find(\"input\", {\"name\": \"captchaSiteKey\"})[\"value\"]\n challenge_id = soup.find(\"input\", {\"name\": \"challengeId\"})[\"value\"]\n challenge_data = soup.find(\"input\", {\"name\": \"challengeData\"})[\"value\"]\n challenge_details = soup.find(\"input\", {\"name\": \"challengeDetails\"})[\"value\"]\n challenge_type = soup.find(\"input\", {\"name\": \"challengeType\"})[\"value\"]\n challenge_source = soup.find(\"input\", {\"name\": \"challengeSource\"})[\"value\"]\n request_submission_id = soup.find(\"input\", {\"name\": \"requestSubmissionId\"})[\n \"value\"\n ]\n display_time = soup.find(\"input\", {\"name\": \"displayTime\"})[\"value\"]\n page_instance = soup.find(\"input\", {\"name\": \"pageInstance\"})[\"value\"]\n failure_redirect_uri = soup.find(\"input\", {\"name\": \"failureRedirectUri\"})[\n \"value\"\n ]\n sign_in_link = soup.find(\"input\", {\"name\": \"signInLink\"})[\"value\"]\n join_now_link = soup.find(\"input\", {\"name\": \"joinNowLink\"})[\"value\"]\n for cookie in session.cookies:\n if cookie.name == \"JSESSIONID\":\n jsession_value = cookie.value.split(\"ajax:\")[1].strip('\"')\n break\n else:\n raise Exception(\"jsessionid not found, raise issue on GitHub\")\n csrf_token = f\"ajax:{jsession_value}\"\n payload = {\n \"csrfToken\": csrf_token,\n \"captchaSiteKey\": captcha_site_key,\n \"challengeId\": challenge_id,\n \"language\": \"en-US\",\n \"displayTime\": display_time,\n \"challengeType\": challenge_type,\n \"challengeSource\": challenge_source,\n \"requestSubmissionId\": request_submission_id,\n \"captchaUserResponseToken\": token,\n \"challengeData\": challenge_data,\n \"pageInstance\": page_instance,\n \"challengeDetails\": challenge_details,\n \"failureRedirectUri\": failure_redirect_uri,\n \"signInLink\": sign_in_link,\n \"joinNowLink\": join_now_link,\n \"_s\": \"CONSUMER_LOGIN\",\n }\n encoded_payload = {\n key: f'{quote(str(value), \"\")}' for key, value in payload.items()\n }\n query_string = \"&\".join(\n [f\"{key}={value}\" for key, value in encoded_payload.items()]\n )\n response = session.post(\n \"https://www.linkedin.com/checkpoint/challenge/verify\", data=query_string\n )\n\n if not response.ok:\n raise Exception(f\"verify captcha failed {response.text[:200]}\")\n\n @retry(stop=stop_after_attempt(5), retry=retry_if_exception_type(BlobException))\n def login_requests(self):\n\n url = \"https://www.linkedin.com/uas/authenticate\"\n\n encoded_username = quote(self.username)\n encoded_password = quote(self.password)\n session = requests.Session()\n session.headers = {\n \"X-Li-User-Agent\": \"LIAuthLibrary:44.0.* com.linkedin.LinkedIn:9.29.8962 iPhone:17.5.1\",\n \"User-Agent\": \"LinkedIn/9.29.8962 CFNetwork/1496.0.7 Darwin/23.5.0\",\n \"X-User-Language\": \"en\",\n \"X-User-Locale\": \"en_US\",\n \"Accept-Language\": \"en-us\",\n }\n\n response = session.get(url)\n if response.status_code != 200:\n raise Exception(\n f\"failed to begin auth process: {response.status_code} {response.text}\"\n )\n for cookie in session.cookies:\n if cookie.name == \"JSESSIONID\":\n jsession_value = cookie.value.split(\"ajax:\")[1].strip('\"')\n break\n else:\n raise Exception(\"jsessionid not found, raise issue on GitHub\")\n session.headers[\"content-type\"] = \"application/x-www-form-urlencoded\"\n csrf_token = f\"ajax%3A{jsession_value}\"\n payload = f\"session_key={encoded_username}&session_password={encoded_password}&JSESSIONID=%22{csrf_token}%22\"\n response = session.post(url, data=payload)\n data = response.json()\n\n if data[\"login_result\"] == \"BAD_USERNAME_OR_PASSWORD\":\n raise Exception(\"incorrect username or password\")\n elif data[\"login_result\"] == \"CHALLENGE\":\n self.solve_captcha(session, data, payload)\n\n session = set_csrf_token(session)\n return session\n\n def login_browser(self):\n \"\"\"Backup login method\"\"\"\n driver = get_webdriver(self.driver_type)\n\n if driver is None:\n logger.debug(\"No browser found for selenium\")\n raise Exception(\"driver not found for selenium\")\n\n driver.get(\"https://linkedin.com/login\")\n input(\"Press enter after logged in\")\n\n selenium_cookies = driver.get_cookies()\n driver.quit()\n\n session = requests.Session()\n for cookie in selenium_cookies:\n session.cookies.set(cookie[\"name\"], cookie[\"value\"])\n\n session = set_csrf_token(session)\n return session\n\n def save_session(self, session, session_file: str):\n data = {\"cookies\": session.cookies, \"headers\": session.headers}\n with open(session_file, \"wb\") as f:\n pickle.dump(data, f)\n\n def load_session(self):\n \"\"\"Load session from session file, otherwise login\"\"\"\n session = None\n if not self.session_file or not os.path.exists(self.session_file):\n if self.username and self.password:\n try:\n session = self.login_requests()\n except RetryError as retry_err:\n retry_err.reraise()\n else:\n session = self.login_browser()\n if not session:\n raise Exception(\"Failed to log in.\")\n if self.session_file:\n self.save_session(session, self.session_file)\n else:\n with open(self.session_file, \"rb\") as f:\n data = pickle.load(f)\n session = requests.Session()\n session.cookies.update(data[\"cookies\"])\n session.headers.update(data[\"headers\"])\n session.headers.update(\n {\n \"User-Agent\": \"Mozilla/5.0 (Linux; U; Android 4.4.2; en-us; SCH-I535 Build/KOT49H) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30\",\n \"X-RestLi-Protocol-Version\": \"2.0.0\",\n \"X-Li-Track\": '{\"clientVersion\":\"1.13.1665\"}',\n }\n )\n if not self.check_logged_in(session):\n raise Exception(\n \"Failed to log in. Likely outdated session file and cookies have expired. Best practice to delete the file and rerun the LinkedAccount() code\"\n )\n return session\n\n def check_logged_in(self, session):\n logger.info(\"Testing if logged in by checking arbitrary LinkedIn company page\")\n try:\n res = session.get(\n \"https://www.linkedin.com/voyager/api/organization/companies?q=universalName&universalName=amazon\"\n )\n if res.status_code != 200:\n logger.error(f\"{res.status_code} status code returned from linkedin\")\n return False\n except Exception as e:\n logger.error(f\"Failed to get arbitrary company page: {e}\")\n return False\n logger.info(\"Account successfully logged in - res code 200\")\n return True\n\n\ndef parse_date(date_str):\n formats = [\"%b %Y\", \"%Y\"]\n for fmt in formats:\n try:\n return datetime.strptime(date_str, fmt)\n except ValueError:\n continue\n return None\n\n\ndef parse_duration(duration):\n from_date = to_date = None\n dates = duration.split(\" Β· \")\n if len(dates) > 1:\n date_range, _ = duration.split(\" Β· \")\n dates = date_range.split(\" - \")\n from_date_str = dates[0]\n to_date_str = dates[1] if dates[1] != \"Present\" else None\n from_date = parse_date(from_date_str) if from_date_str else None\n to_date = parse_date(to_date_str) if to_date_str else None\n\n return from_date, to_date\n\n\ndef set_logger_level(verbose: int = 0):\n \"\"\"\n Adjusts the logger's level. This function allows the logging level to be changed at runtime.\n\n Parameters:\n - verbose: int {0, 1, 2} (default=0, no logs)\n \"\"\"\n if verbose is None:\n return\n level_name = {2: \"DEBUG\", 1: \"INFO\", 0: \"WARNING\"}.get(verbose, \"INFO\")\n level = getattr(logging, level_name.upper(), None)\n if level is not None:\n logger.setLevel(level)\n else:\n raise ValueError(f\"Invalid log level: {level_name}\")\n\n\ndef parse_dates(date_str):\n regex = r\"(\\b\\w+ \\d{4}|\\b\\d{4}|\\bPresent)\"\n matches = re.findall(regex, date_str)\n\n start_date, end_date = None, None\n if matches:\n if \"Present\" in matches:\n if len(matches) == 1:\n start_date = None\n end_date = None\n else:\n start_date = parse(matches[0]).date()\n end_date = None\n else:\n if len(matches) == 2:\n start_date = parse(matches[0]).date()\n end_date = parse(matches[1]).date()\n elif len(matches) == 1:\n start_date = parse(matches[0]).date()\n\n return start_date, end_date\n\n\ndef extract_emails_from_text(text: str) -> list[str] | None:\n if not text:\n return None\n email_regex = re.compile(r\"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}\")\n return email_regex.findall(text)\n\n\ndef parse_company_data(json_data, search_term=None):\n company_info = json_data[\"elements\"][0]\n\n company_name = company_info.get(\"name\", \"\")\n staff_count = company_info.get(\"staffCount\", None)\n company_type = company_info.get(\"type\", \"\")\n description = company_info.get(\"description\", \"\")\n\n industries_list = [\n ind.get(\"localizedName\", \"\")\n for ind in company_info.get(\"companyIndustries\", [])\n ]\n\n headquarter = company_info.get(\"headquarter\", {})\n headquarter_full = f'{headquarter.get(\"line1\", \"\")}, {headquarter.get(\"city\", \"\")}, {headquarter.get(\"country\", \"\")} {headquarter.get(\"postalCode\", \"\")}'\n\n logo_data = company_info.get(\"logo\", {})\n vector_image = logo_data.get(\"image\", {}).get(\"com.linkedin.common.VectorImage\", {})\n root_url = vector_image.get(\"rootUrl\", \"\")\n artifacts = vector_image.get(\"artifacts\", [])\n\n logo_url = None\n if artifacts:\n first_artifact = artifacts[0]\n file_path = first_artifact.get(\"fileIdentifyingUrlPathSegment\", \"\")\n logo_url = root_url + file_path\n\n tracking_info = company_info.get(\"trackingInfo\", {})\n object_urn = tracking_info.get(\"objectUrn\", \"\")\n internal_id = None\n if object_urn.startswith(\"urn:li:company:\"):\n internal_id = object_urn.split(\":\")[-1]\n\n bg_photo = company_info.get(\"backgroundCoverPhoto\", {})\n vector_image = bg_photo.get(\"com.linkedin.common.VectorImage\", {})\n root_url = vector_image.get(\"rootUrl\", \"\")\n artifacts = vector_image.get(\"artifacts\", [])\n banner_url = None\n if artifacts:\n chosen_artifact = artifacts[0]\n file_segment = chosen_artifact.get(\"fileIdentifyingUrlPathSegment\", \"\")\n banner_url = root_url + file_segment\n\n company_df = pd.DataFrame(\n {\n \"search_term\": [search_term],\n \"linkedin_company_id\": [internal_id],\n \"company_name\": [company_name],\n \"staff_count\": [staff_count],\n \"company_type\": [company_type],\n \"industries\": [industries_list],\n \"headquarters_address\": [headquarter_full],\n \"description\": [description],\n \"logo_url\": [logo_url],\n \"banner_url\": [banner_url],\n }\n )\n return company_df\n\n\ndef clean_df(staff_df):\n if \"estimated_age\" in staff_df.columns:\n staff_df[\"estimated_age\"] = staff_df[\"estimated_age\"].astype(\"Int64\")\n if \"followers\" in staff_df.columns:\n staff_df[\"followers\"] = staff_df[\"followers\"].astype(\"Int64\")\n if \"connections\" in staff_df.columns:\n staff_df[\"connections\"] = staff_df[\"connections\"].astype(\"Int64\")\n if \"mutuals\" in staff_df.columns:\n staff_df[\"mutuals\"] = staff_df[\"mutuals\"].astype(\"Int64\")\n return staff_df\n\n\ndef upload_to_clay(webhook_url: str, data: pd.DataFrame):\n records = data.to_dict(\"records\")\n\n responses = []\n for i, row in enumerate(records, start=1):\n try:\n response = requests.post(\n webhook_url, headers={\"Accept\": \"application/json\"}, json=row\n )\n response.raise_for_status()\n logger.info(f\"Uploaded row to Clay: {i} / {len(records)}\")\n except requests.exceptions.RequestException as e:\n logger.error(f\"Failed to upload row to Clay: {str(e)}\")\n responses.append({\"error\": str(e), \"data\": row})\n\n return responses\n\n\nif __name__ == \"__main__\":\n p = parse_dates(\"May 2018 - Jun 2024\")\n" } ]