[ { "path": ".dockerignore", "content": "# Git\r\n.git\r\n.gitignore\r\n.gitattributes\r\n\r\n# CI\r\n.codeclimate.yml\r\n.travis.yml\r\n.taskcluster.yml\r\n\r\n# Docker\r\ndocker-compose.yml\r\nDockerfile\r\n.docker\r\n.dockerignore\r\n\r\n# Byte-compiled / optimized / DLL files\r\n**/__pycache__/\r\n**/*.py[cod]\r\n\r\n# C extensions\r\n*.so\r\n\r\n# Distribution / packaging\r\n.Python\r\nenv/\r\nbuild/\r\ndevelop-eggs/\r\ndist/\r\ndownloads/\r\neggs/\r\nlib/\r\nlib64/\r\nparts/\r\nsdist/\r\nvar/\r\n*.egg-info/\r\n.installed.cfg\r\n*.egg\r\n\r\n# PyInstaller\r\n# Usually these files are written by a python script from a template\r\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\r\n*.manifest\r\n*.spec\r\n\r\n# Installer logs\r\npip-log.txt\r\npip-delete-this-directory.txt\r\n\r\n# Unit test / coverage reports\r\nhtmlcov/\r\n.tox/\r\n.coverage\r\n.cache\r\nnosetests.xml\r\ncoverage.xml\r\n\r\n# Translations\r\n*.mo\r\n*.pot\r\n\r\n# Django stuff:\r\n*.log\r\n\r\n# Sphinx documentation\r\ndocs/_build/\r\n\r\n# PyBuilder\r\ntarget/\r\n\r\n# Virtual environment\r\n.env\r\n.venv/\r\nvenv/\r\n\r\n# PyCharm\r\n.idea\r\n\r\n# Python mode for VIM\r\n.ropeproject\r\n**/.ropeproject\r\n\r\n# Vim swap files\r\n**/*.swp\r\n\r\n# VS Code\r\n.vscode/" }, { "path": ".env.example", "content": "NETBOX_URL=\nNETBOX_TOKEN=\nREPO_URL=https://github.com/netbox-community/devicetype-library.git\nREPO_BRANCH=master\nIGNORE_SSL_ERRORS=False\n#REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt # you should enable this if you are running on a linux system\n#SLUGS=c9300-48u isr4431 isr4331\n" }, { "path": ".github/dependabot.yml", "content": "---\nversion: 2\nupdates:\n - package-ecosystem: pip\n directory: /\n schedule:\n interval: monthly\n time: '02:00'\n timezone: America/New_York\n labels:\n - dependencies\n target-branch: master\n assignees:\n - \"danner26\"" }, { "path": ".github/workflows/docker.yml", "content": "---\nname: ci\n\non:\n push:\n branches:\n - 'master'\n - 'main'\n pull_request:\n branches:\n - 'master'\n - 'main'\n workflow_dispatch:\n release:\n types: [published, edited]\n\njobs:\n build-and-push-images:\n runs-on: ubuntu-latest\n steps:\n -\n name: Checkout\n uses: actions/checkout@v2\n -\n name: Docker meta\n id: meta\n uses: docker/metadata-action@v3\n with:\n images: |\n ghcr.io/minitriga/Netbox-Device-Type-Library-Import\n tags: |\n type=raw,value=latest,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }}\n type=ref,event=branch\n type=ref,event=pr\n type=semver,pattern={{version}}\n type=semver,pattern={{major}}\n type=semver,pattern={{major}}.{{minor}}\n - \n name: Set up QEMU\n uses: docker/setup-qemu-action@v1\n -\n name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v1\n -\n name: Login to GitHub Container Registry\n if: github.event_name != 'pull_request'\n uses: docker/login-action@v1 \n with:\n registry: ghcr.io\n username: ${{ github.repository_owner }}\n password: ${{ secrets.GITHUB_TOKEN }}\n -\n name: Build and push\n uses: docker/build-push-action@v2\n with:\n context: .\n push: ${{ github.event_name != 'pull_request' }}\n platforms: linux/amd64\n tags: ${{ steps.meta.outputs.tags }}\n labels: ${{ steps.meta.outputs.labels }}" }, { "path": ".github/workflows/stale.yml", "content": "---\n#close-stale-issues (https://github.com/marketplace/actions/close-stale-issues)\nname: Close stale PRs\non: # yamllint disable-line rule:truthy\n schedule:\n - cron: 0 4 * * *\n\njobs:\n stale:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/stale@v5\n with:\n close-pr-message: >\n This PR has been automatically closed due to lack of activity.\n days-before-stale: 30\n days-before-close: 7\n operations-per-run: 100\n remove-stale-when-updated: false\n stale-pr-label: stale\n stale-pr-message: >\n This PR has been automatically marked as stale because it has not\n had recent activity. It will be closed automatically if no further\n progress is made." }, { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\npip-wheel-metadata/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\ndb.sqlite3-journal\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n.python-version\n\n# pipenv\n# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n# However, in case of collaboration, if having platform-specific dependencies or dependencies\n# having no cross-platform support, pipenv may install dependencies that don't work, or not\n# install all needed dependencies.\n#Pipfile.lock\n\n# PEP 582; used by e.g. github.com/David-OConnor/pyflow\n__pypackages__/\n\n# Celery stuff\ncelerybeat-schedule\ncelerybeat.pid\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env*\n!.env.example\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n\n# Editor\n.vscode\n\nrepo\n" }, { "path": "Dockerfile", "content": "FROM python:3.9-alpine\n\nENV REPO_URL=https://github.com/netbox-community/devicetype-library.git\nWORKDIR /app\nCOPY requirements.txt .\n\n# Install dependencies\nRUN apk add --no-cache git ca-certificates && \\\n python3 -m pip install --upgrade pip && \\\n pip3 install -r requirements.txt\n\n# Copy over src code\nCOPY *.py ./\n\n# -u to avoid stdout buffering\nCMD [\"python3\",\"-u\",\"nb-dt-import.py\"]\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2021 Alexander Gittings\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "README.md", "content": "# Netbox Device Type Import\n\nThis library is intended to be your friend and help you import all the device-types defined within the the [NetBox Device Type Library Repository](https://github.com/netbox-community/devicetype-library).\n\n> Tested working with 2.9.4, 2.10.4\n\n## 🪄 Description\n\nThis script will clone a copy of the `netbox-community/devicetype-library` repository to your machine to allow it to import the device types you would like without copy and pasting them into the Netbox UI.\n\n## 🚀 Getting Started\n\n1. This script is written in Python, so lets setup a virtual environment.\n\n```\ngit clone https://github.com/netbox-community/Device-Type-Library-Import.git\ncd Device-Type-Library-Import\npython3 -m venv venv\nsource venv/bin/activate\n```\n\n2. Now that we have the basics setup, we'll need to install the requirements.\n\n```\npip install -r requirements.txt\n```\n\n3. There are two variables that are required when using this script to import device types into your Netbox installation. (1) Your Netbox instance URL and (2) a token with **write rights**.\n\nCopy the existing `.env.example` to your own `.env` file, and fill in the variables.\n\n```\ncp .env.example .env\nvim .env\n```\n\nFinally, we are able to execute the script and import some device templates!\n\n## 🔌 Usage\n\nTo use the script, simply execute the script as follows. Make sure you're still in the activated virtual environment we created before.\n\n```\n./nb-dt-import.py\n```\n\nThis will clone the latest master branch from the `netbox-community/devicetype-library` from Github and install it into the `repo` subdirectory. If this directory already exists, it will perform a `git pull` to update the repository instead.\n\nNext, it will loop over every manufacturer and every device of every manufacturer and begin checking if your Netbox install already has them, and if not, creates them. It will skip preexisting manufacturers, devices, interfaces, etc. so as to not end up with duplicate entries in your Netbox instance.\n\n### 🧰 Arguments\n\nThis script currently accepts a list of vendors as an argument, so that you can selectively import devices.\n\nTo import only device by APC, for example:\n\n```\n./nb-dt-import.py --vendors apc\n```\n\n`--vendors` can also accept a comma separated list of vendors if you want to import multiple.\n\n```\n./nb-dt-import.py --vendors apc,juniper\n```\n\n## Docker build\n\nIt's possible to use this project as a docker container.\n\nTo build :\n\n```\ndocker build -t netbox-devicetype-import-library .\n```\n\nAlternatively you can pull a pre-built image from Github Container Registry (ghcr.io):\n\n```\ndocker pull ghcr.io/minitriga/netbox-device-type-library-import\n```\n\nThe container supports the following env var as configuration :\n\n- `REPO_URL`, the repo to look for device types (defaults to _https://github.com/netbox-community/devicetype-library.git_)\n- `REPO_BRANCH`, the branch to check out if appropriate, defaults to master.\n- `NETBOX_URL`, used to access netbox\n- `NETBOX_TOKEN`, token for accessing netbox\n- `VENDORS`, a comma-separated list of vendors to import (defaults to None)\n- `REQUESTS_CA_BUNDLE`, path to a CA_BUNDLE for validation if you are using self-signed certificates(file must be included in the container)\n\nTo run :\n\n```\ndocker run -e \"NETBOX_URL=http://netbox:8080/\" -e \"NETBOX_TOKEN=98765434567890\" ghcr.io/minitriga/netbox-device-type-library-import\n```\n\n## 🧑‍💻 Contributing\n\nWe're happy about any pull requests!\n\n## 📜 License\n\nMIT\n" }, { "path": "log_handler.py", "content": "from sys import exit as system_exit\n\n\nclass LogHandler:\n def __new__(cls, *args, **kwargs):\n return super().__new__(cls)\n\n def __init__(self, args):\n self.args = args\n\n def exception(self, exception_type, exception, stack_trace=None):\n exception_dict = {\n \"EnvironmentError\": f'Environment variable \"{exception}\" is not set.',\n \"SSLError\": f'SSL verification failed. IGNORE_SSL_ERRORS is {exception}. Set IGNORE_SSL_ERRORS to True if you want to ignore this error. EXITING.',\n \"GitCommandError\": f'The repo \"{exception}\" is not a valid git repo.',\n \"GitInvalidRepositoryError\": f'The repo \"{exception}\" is not a valid git repo.',\n \"Exception\": f'An unknown error occurred: \"{exception}\"'\n }\n\n if self.args.verbose and stack_trace:\n print(stack_trace)\n print(exception_dict[exception_type])\n system_exit(1)\n\n def verbose_log(self, message):\n if self.args.verbose:\n print(message)\n\n def log(self, message):\n print(message)\n\n def log_device_ports_created(self, created_ports: list = [], port_type: str = \"port\"):\n for port in created_ports:\n self.verbose_log(f'{port_type} Template Created: {port.name} - '\n + f'{port.type if hasattr(port, \"type\") else \"\"} - {port.device_type.id} - '\n + f'{port.id}')\n return len(created_ports)\n\n def log_module_ports_created(self, created_ports: list = [], port_type: str = \"port\"):\n for port in created_ports:\n self.verbose_log(f'{port_type} Template Created: {port.name} - '\n + f'{port.type if hasattr(port, \"type\") else \"\"} - {port.module_type.id} - '\n + f'{port.id}')\n return len(created_ports)\n" }, { "path": "nb-dt-import.py", "content": "#!/usr/bin/env python3\nfrom collections import Counter\nfrom datetime import datetime\nimport yaml\nimport pynetbox\nfrom glob import glob\nimport os\n\nimport settings\nfrom netbox_api import NetBox\n\n\ndef main():\n startTime = datetime.now()\n args = settings.args\n\n netbox = NetBox(settings)\n files, vendors = settings.dtl_repo.get_devices(\n f'{settings.dtl_repo.repo_path}/device-types/', args.vendors)\n\n settings.handle.log(f'{len(vendors)} Vendors Found')\n device_types = settings.dtl_repo.parse_files(files, slugs=args.slugs)\n settings.handle.log(f'{len(device_types)} Device-Types Found')\n netbox.create_manufacturers(vendors)\n netbox.create_device_types(device_types)\n\n if netbox.modules:\n settings.handle.log(\"Modules Enabled. Creating Modules...\")\n files, vendors = settings.dtl_repo.get_devices(\n f'{settings.dtl_repo.repo_path}/module-types/', args.vendors)\n settings.handle.log(f'{len(vendors)} Module Vendors Found')\n module_types = settings.dtl_repo.parse_files(files, slugs=args.slugs)\n settings.handle.log(f'{len(module_types)} Module-Types Found')\n netbox.create_manufacturers(vendors)\n netbox.create_module_types(module_types)\n\n settings.handle.log('---')\n settings.handle.verbose_log(\n f'Script took {(datetime.now() - startTime)} to run')\n settings.handle.log(f'{netbox.counter[\"added\"]} devices created')\n settings.handle.log(f'{netbox.counter[\"images\"]} images uploaded')\n settings.handle.log(\n f'{netbox.counter[\"updated\"]} interfaces/ports updated')\n settings.handle.log(\n f'{netbox.counter[\"manufacturer\"]} manufacturers created')\n if settings.NETBOX_FEATURES['modules']:\n settings.handle.log(\n f'{netbox.counter[\"module_added\"]} modules created')\n settings.handle.log(\n f'{netbox.counter[\"module_port_added\"]} module interface / ports created')\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "netbox_api.py", "content": "from collections import Counter\nimport pynetbox\nimport requests\nimport os\nimport glob\n# from pynetbox import RequestError as APIRequestError\n\nclass NetBox:\n def __new__(cls, *args, **kwargs):\n return super().__new__(cls)\n\n def __init__(self, settings):\n self.counter = Counter(\n added=0,\n updated=0,\n manufacturer=0,\n module_added=0,\n module_port_added=0,\n images=0,\n )\n self.url = settings.NETBOX_URL\n self.token = settings.NETBOX_TOKEN\n self.handle = settings.handle\n self.netbox = None\n self.ignore_ssl = settings.IGNORE_SSL_ERRORS\n self.modules = False\n self.new_filters = False\n self.connect_api()\n self.verify_compatibility()\n self.existing_manufacturers = self.get_manufacturers()\n self.device_types = DeviceTypes(self.netbox, self.handle, self.counter, self.ignore_ssl, self.new_filters)\n\n def connect_api(self):\n try:\n self.netbox = pynetbox.api(self.url, token=self.token)\n if self.ignore_ssl:\n self.handle.verbose_log(\"IGNORE_SSL_ERRORS is True, catching exception and disabling SSL verification.\")\n #requests.packages.urllib3.disable_warnings()\n self.netbox.http_session.verify = False\n except Exception as e:\n self.handle.exception(\"Exception\", 'NetBox API Error', e)\n\n def get_api(self):\n return self.netbox\n\n def get_counter(self):\n return self.counter\n\n def verify_compatibility(self):\n # nb.version should be the version in the form '3.2'\n version_split = [int(x) for x in self.netbox.version.split('.')]\n\n # Later than 3.2\n # Might want to check for the module-types entry as well?\n if version_split[0] > 3 or (version_split[0] == 3 and version_split[1] >= 2):\n self.modules = True\n\n # check if version >= 4.1 in order to use new filter names (https://github.com/netbox-community/netbox/issues/15410)\n if version_split[0] >= 4 and version_split[1] >= 1:\n self.new_filters = True\n self.handle.log(f'Netbox version {self.netbox.version} found. Using new filters.')\n \n def get_manufacturers(self):\n return {str(item): item for item in self.netbox.dcim.manufacturers.all()}\n\n def create_manufacturers(self, vendors):\n to_create = []\n self.existing_manufacturers = self.get_manufacturers()\n for vendor in vendors:\n try:\n manGet = self.existing_manufacturers[vendor[\"name\"]]\n self.handle.verbose_log(f'Manufacturer Exists: {manGet.name} - {manGet.id}')\n except KeyError:\n to_create.append(vendor)\n self.handle.verbose_log(f\"Manufacturer queued for addition: {vendor['name']}\")\n\n if to_create:\n try:\n created_manufacturers = self.netbox.dcim.manufacturers.create(to_create)\n for manufacturer in created_manufacturers:\n self.handle.verbose_log(f'Manufacturer Created: {manufacturer.name} - '\n + f'{manufacturer.id}')\n self.counter.update({'manufacturer': 1})\n except pynetbox.RequestError as request_error:\n self.handle.log(\"Error creating manufacturers\")\n self.handle.verbose_log(f\"Error during manufacturer creation. - {request_error.error}\")\n\n def create_device_types(self, device_types_to_add):\n for device_type in device_types_to_add:\n\n # Remove file base path\n src_file = device_type[\"src\"]\n del device_type[\"src\"]\n\n # Pre-process front/rear_image flag, remove it if present\n saved_images = {}\n image_base = os.path.dirname(src_file).replace(\"device-types\",\"elevation-images\")\n for i in [\"front_image\",\"rear_image\"]:\n if i in device_type:\n if device_type[i]:\n image_glob = f\"{image_base}/{device_type['slug']}.{i.split('_')[0]}.*\"\n images = glob.glob(image_glob, recursive=False)\n if images:\n saved_images[i] = images[0]\n else:\n self.handle.log(f\"Error locating image file using '{image_glob}'\")\n del device_type[i]\n\n try:\n dt = self.device_types.existing_device_types[device_type[\"model\"]]\n self.handle.verbose_log(f'Device Type Exists: {dt.manufacturer.name} - '\n + f'{dt.model} - {dt.id}')\n except KeyError:\n try:\n dt = self.netbox.dcim.device_types.create(device_type)\n self.counter.update({'added': 1})\n self.handle.verbose_log(f'Device Type Created: {dt.manufacturer.name} - '\n + f'{dt.model} - {dt.id}')\n except pynetbox.RequestError as e:\n self.handle.log(f'Error {e.error} creating device type:'\n f' {device_type[\"manufacturer\"][\"name\"]} {device_type[\"model\"]}')\n continue\n\n if \"interfaces\" in device_type:\n self.device_types.create_interfaces(device_type[\"interfaces\"], dt.id)\n if \"power-ports\" in device_type:\n self.device_types.create_power_ports(device_type[\"power-ports\"], dt.id)\n if \"power-port\" in device_type:\n self.device_types.create_power_ports(device_type[\"power-port\"], dt.id)\n if \"console-ports\" in device_type:\n self.device_types.create_console_ports(device_type[\"console-ports\"], dt.id)\n if \"power-outlets\" in device_type:\n self.device_types.create_power_outlets(device_type[\"power-outlets\"], dt.id)\n if \"console-server-ports\" in device_type:\n self.device_types.create_console_server_ports(device_type[\"console-server-ports\"], dt.id)\n if \"rear-ports\" in device_type:\n self.device_types.create_rear_ports(device_type[\"rear-ports\"], dt.id)\n if \"front-ports\" in device_type:\n self.device_types.create_front_ports(device_type[\"front-ports\"], dt.id)\n if \"device-bays\" in device_type:\n self.device_types.create_device_bays(device_type[\"device-bays\"], dt.id)\n if self.modules and 'module-bays' in device_type:\n self.device_types.create_module_bays(device_type['module-bays'], dt.id)\n\n # Finally, update images if any\n if saved_images:\n self.device_types.upload_images(self.url, self.token, saved_images, dt.id)\n\n def create_module_types(self, module_types):\n all_module_types = {}\n for curr_nb_mt in self.netbox.dcim.module_types.all():\n if curr_nb_mt.manufacturer.slug not in all_module_types:\n all_module_types[curr_nb_mt.manufacturer.slug] = {}\n\n all_module_types[curr_nb_mt.manufacturer.slug][curr_nb_mt.model] = curr_nb_mt\n\n\n for curr_mt in module_types:\n try:\n module_type_res = all_module_types[curr_mt['manufacturer']['slug']][curr_mt[\"model\"]]\n self.handle.verbose_log(f'Module Type Exists: {module_type_res.manufacturer.name} - '\n + f'{module_type_res.model} - {module_type_res.id}')\n except KeyError:\n try:\n module_type_res = self.netbox.dcim.module_types.create(curr_mt)\n self.counter.update({'module_added': 1})\n self.handle.verbose_log(f'Module Type Created: {module_type_res.manufacturer.name} - '\n + f'{module_type_res.model} - {module_type_res.id}')\n except pynetbox.RequestError as exce:\n self.handle.log(f\"Error '{exce.error}' creating module type: \" +\n f\"{curr_mt}\")\n\n if \"interfaces\" in curr_mt:\n self.device_types.create_module_interfaces(curr_mt[\"interfaces\"], module_type_res.id)\n if \"power-ports\" in curr_mt:\n self.device_types.create_module_power_ports(curr_mt[\"power-ports\"], module_type_res.id)\n if \"console-ports\" in curr_mt:\n self.device_types.create_module_console_ports(curr_mt[\"console-ports\"], module_type_res.id)\n if \"power-outlets\" in curr_mt:\n self.device_types.create_module_power_outlets(curr_mt[\"power-outlets\"], module_type_res.id)\n if \"console-server-ports\" in curr_mt:\n self.device_types.create_module_console_server_ports(curr_mt[\"console-server-ports\"], module_type_res.id)\n if \"rear-ports\" in curr_mt:\n self.device_types.create_module_rear_ports(curr_mt[\"rear-ports\"], module_type_res.id)\n if \"front-ports\" in curr_mt:\n self.device_types.create_module_front_ports(curr_mt[\"front-ports\"], module_type_res.id)\n\nclass DeviceTypes:\n def __new__(cls, *args, **kwargs):\n return super().__new__(cls)\n\n def __init__(self, netbox, handle, counter, ignore_ssl, new_filters):\n self.netbox = netbox\n self.handle = handle\n self.counter = counter\n self.existing_device_types = self.get_device_types()\n self.ignore_ssl = ignore_ssl\n self.new_filters = new_filters\n\n def get_device_types(self):\n return {str(item): item for item in self.netbox.dcim.device_types.all()}\n\n def get_power_ports(self, device_type):\n return {str(item): item for item in self.netbox.dcim.power_port_templates.filter(**{'device_type_id' if self.new_filters else 'devicetype_id': device_type})}\n \n def get_rear_ports(self, device_type):\n return {str(item): item for item in self.netbox.dcim.rear_port_templates.filter(**{'device_type_id' if self.new_filters else 'devicetype_id': device_type})}\n\n def get_module_power_ports(self, module_type):\n return {str(item): item for item in self.netbox.dcim.power_port_templates.filter(**{'module_type_id' if self.new_filters else 'moduletype_id': module_type})}\n\n def get_module_rear_ports(self, module_type):\n return {str(item): item for item in self.netbox.dcim.rear_port_templates.filter(**{'module_type_id' if self.new_filters else 'moduletype_id': module_type})}\n\n def get_device_type_ports_to_create(self, dcim_ports, device_type, existing_ports):\n to_create = [port for port in dcim_ports if port['name'] not in existing_ports]\n for port in to_create:\n port['device_type'] = device_type\n\n return to_create\n\n def get_module_type_ports_to_create(self, module_ports, module_type, existing_ports):\n to_create = [port for port in module_ports if port['name'] not in existing_ports]\n for port in to_create:\n port['module_type'] = module_type\n\n return to_create\n\n def create_interfaces(self, interfaces, device_type):\n existing_interfaces = {str(item): item for item in self.netbox.dcim.interface_templates.filter(\n **{'device_type_id' if self.new_filters else 'devicetype_id': device_type})}\n to_create = self.get_device_type_ports_to_create(\n interfaces, device_type, existing_interfaces)\n\n if to_create:\n try:\n self.counter.update({'updated':\n self.handle.log_device_ports_created(\n self.netbox.dcim.interface_templates.create(to_create), \"Interface\")\n })\n except pynetbox.RequestError as excep:\n self.handle.log(f\"Error '{excep.error}' creating Interface\")\n\n def create_power_ports(self, power_ports, device_type):\n existing_power_ports = self.get_power_ports(device_type)\n to_create = self.get_device_type_ports_to_create(power_ports, device_type, existing_power_ports)\n\n if to_create:\n try:\n self.counter.update({'updated':\n self.handle.log_device_ports_created(\n self.netbox.dcim.power_port_templates.create(to_create), \"Power Port\")\n })\n except pynetbox.RequestError as excep:\n self.handle.log(f\"Error '{excep.error}' creating Power Port\")\n\n def create_console_ports(self, console_ports, device_type):\n existing_console_ports = {str(item): item for item in self.netbox.dcim.console_port_templates.filter(**{'device_type_id' if self.new_filters else 'devicetype_id': device_type})}\n to_create = self.get_device_type_ports_to_create(console_ports, device_type, existing_console_ports)\n\n if to_create:\n try:\n self.counter.update({'updated':\n self.handle.log_device_ports_created(\n self.netbox.dcim.console_port_templates.create(to_create), \"Console Port\")\n })\n except pynetbox.RequestError as excep:\n self.handle.log(f\"Error '{excep.error}' creating Console Port\")\n\n def create_power_outlets(self, power_outlets, device_type):\n existing_power_outlets = {str(item): item for item in self.netbox.dcim.power_outlet_templates.filter(**{'device_type_id' if self.new_filters else 'devicetype_id': device_type})}\n to_create = self.get_device_type_ports_to_create(power_outlets, device_type, existing_power_outlets)\n\n if to_create:\n existing_power_ports = self.get_power_ports(device_type)\n for outlet in to_create:\n try:\n power_port = existing_power_ports[outlet[\"power_port\"]]\n outlet['power_port'] = power_port.id\n except KeyError:\n pass\n\n try:\n self.counter.update({'updated':\n self.handle.log_device_ports_created(\n self.netbox.dcim.power_outlet_templates.create(to_create), \"Power Outlet\")\n })\n except pynetbox.RequestError as excep:\n self.handle.log(f\"Error '{excep.error}' creating Power Outlet\")\n\n def create_console_server_ports(self, console_server_ports, device_type):\n existing_console_server_ports = {str(item): item for item in self.netbox.dcim.console_server_port_templates.filter(**{'device_type_id' if self.new_filters else 'devicetype_id': device_type})}\n to_create = self.get_device_type_ports_to_create(console_server_ports, device_type, existing_console_server_ports)\n\n if to_create:\n try:\n self.counter.update({'updated':\n self.handle.log_device_ports_created(\n self.netbox.dcim.console_server_port_templates.create(to_create), \"Console Server Port\")\n })\n except pynetbox.RequestError as excep:\n self.handle.log(f\"Error '{excep.error}' creating Console Server Port\")\n\n def create_rear_ports(self, rear_ports, device_type):\n existing_rear_ports = self.get_rear_ports(device_type)\n to_create = self.get_device_type_ports_to_create(rear_ports, device_type, existing_rear_ports)\n\n if to_create:\n try:\n self.counter.update({'updated':\n self.handle.log_device_ports_created(\n self.netbox.dcim.rear_port_templates.create(to_create), \"Rear Port\")\n })\n except pynetbox.RequestError as excep:\n self.handle.log(f\"Error '{excep.error}' creating Rear Port\")\n\n def create_front_ports(self, front_ports, device_type):\n existing_front_ports = {str(item): item for item in self.netbox.dcim.front_port_templates.filter(**{'device_type_id' if self.new_filters else 'devicetype_id': device_type})}\n to_create = self.get_device_type_ports_to_create(front_ports, device_type, existing_front_ports)\n\n if to_create:\n all_rearports = self.get_rear_ports(device_type)\n for port in to_create:\n try:\n rear_port = all_rearports[port[\"rear_port\"]]\n port['rear_port'] = rear_port.id\n except KeyError:\n self.handle.log(f'Could not find Rear Port for Front Port: {port[\"name\"]} - '\n + f'{port[\"type\"]} - {device_type}')\n\n try:\n self.counter.update({'updated':\n self.handle.log_device_ports_created(\n self.netbox.dcim.front_port_templates.create(to_create), \"Front Port\")\n })\n except pynetbox.RequestError as excep:\n self.handle.log(f\"Error '{excep.error}' creating Front Port\")\n\n def create_device_bays(self, device_bays, device_type):\n existing_device_bays = {str(item): item for item in self.netbox.dcim.device_bay_templates.filter(**{'device_type_id' if self.new_filters else 'devicetype_id': device_type})}\n to_create = self.get_device_type_ports_to_create(device_bays, device_type, existing_device_bays)\n\n if to_create:\n try:\n self.counter.update({'updated':\n self.handle.log_device_ports_created(\n self.netbox.dcim.device_bay_templates.create(to_create), \"Device Bay\")\n })\n except pynetbox.RequestError as excep:\n self.handle.log(f\"Error '{excep.error}' creating Device Bay\")\n\n def create_module_bays(self, module_bays, device_type):\n existing_module_bays = {str(item): item for item in self.netbox.dcim.module_bay_templates.filter(**{'device_type_id' if self.new_filters else 'devicetype_id': device_type})}\n to_create = self.get_device_type_ports_to_create(module_bays, device_type, existing_module_bays)\n\n if to_create:\n try:\n self.counter.update({'updated':\n self.handle.log_device_ports_created(\n self.netbox.dcim.module_bay_templates.create(to_create), \"Module Bay\")\n })\n except pynetbox.RequestError as excep:\n self.handle.log(f\"Error '{excep.error}' creating Module Bay\")\n\n def create_module_interfaces(self, module_interfaces, module_type):\n existing_interfaces = {str(item): item for item in self.netbox.dcim.interface_templates.filter(**{'module_type_id' if self.new_filters else 'moduletype_id': module_type})}\n to_create = self.get_module_type_ports_to_create(module_interfaces, module_type, existing_interfaces)\n\n if to_create:\n try:\n self.counter.update({'updated':\n self.handle.log_module_ports_created(\n self.netbox.dcim.interface_templates.create(to_create), \"Module Interface\")\n })\n except pynetbox.RequestError as excep:\n self.handle.log(f\"Error '{excep.error}' creating Module Interface\")\n\n def create_module_power_ports(self, power_ports, module_type):\n existing_power_ports = self.get_module_power_ports(module_type)\n to_create = self.get_module_type_ports_to_create(power_ports, module_type, existing_power_ports)\n\n if to_create:\n try:\n self.counter.update({'updated':\n self.handle.log_module_ports_created(\n self.netbox.dcim.power_port_templates.create(to_create), \"Module Power Port\")\n })\n except pynetbox.RequestError as excep:\n self.handle.log(f\"Error '{excep.error}' creating Module Power Port\")\n\n def create_module_console_ports(self, console_ports, module_type):\n existing_console_ports = {str(item): item for item in self.netbox.dcim.console_port_templates.filter(**{'module_type_id' if self.new_filters else 'moduletype_id': module_type})}\n to_create = self.get_module_type_ports_to_create(console_ports, module_type, existing_console_ports)\n\n if to_create:\n try:\n self.counter.update({'updated':\n self.handle.log_module_ports_created(\n self.netbox.dcim.console_port_templates.create(to_create), \"Module Console Port\")\n })\n except pynetbox.RequestError as excep:\n self.handle.log(f\"Error '{excep.error}' creating Module Console Port\")\n\n def create_module_power_outlets(self, power_outlets, module_type):\n existing_power_outlets = {str(item): item for item in self.netbox.dcim.power_outlet_templates.filter(**{'module_type_id' if self.new_filters else 'moduletype_id': module_type})}\n to_create = self.get_module_type_ports_to_create(power_outlets, module_type, existing_power_outlets)\n\n if to_create:\n existing_power_ports = self.get_module_power_ports(module_type)\n for outlet in to_create:\n try:\n power_port = existing_power_ports[outlet[\"power_port\"]]\n outlet['power_port'] = power_port.id\n except KeyError:\n pass\n\n try:\n self.counter.update({'updated':\n self.handle.log_module_ports_created(\n self.netbox.dcim.power_outlet_templates.create(to_create), \"Module Power Outlet\")\n })\n except pynetbox.RequestError as excep:\n self.handle.log(f\"Error '{excep.error}' creating Module Power Outlet\")\n\n def create_module_console_server_ports(self, console_server_ports, module_type):\n existing_console_server_ports = {str(item): item for item in self.netbox.dcim.console_server_port_templates.filter(**{'module_type_id' if self.new_filters else 'moduletype_id': module_type})}\n to_create = self.get_module_type_ports_to_create(console_server_ports, module_type, existing_console_server_ports)\n\n if to_create:\n try:\n self.counter.update({'updated':\n self.handle.log_module_ports_created(\n self.netbox.dcim.console_server_port_templates.create(to_create), \"Module Console Server Port\")\n })\n except pynetbox.RequestError as excep:\n self.handle.log(f\"Error '{excep.error}' creating Module Console Server Port\")\n\n def create_module_rear_ports(self, rear_ports, module_type):\n existing_rear_ports = self.get_module_rear_ports(module_type)\n to_create = self.get_module_type_ports_to_create(rear_ports, module_type, existing_rear_ports)\n\n if to_create:\n try:\n self.counter.update({'updated':\n self.handle.log_module_ports_created(\n self.netbox.dcim.rear_port_templates.create(to_create), \"Module Rear Port\")\n })\n except pynetbox.RequestError as excep:\n self.handle.log(f\"Error '{excep.error}' creating Module Rear Port\")\n\n def create_module_front_ports(self, front_ports, module_type):\n existing_front_ports = {str(item): item for item in self.netbox.dcim.front_port_templates.filter(**{'module_type_id' if self.new_filters else 'moduletype_id': module_type})}\n to_create = self.get_module_type_ports_to_create(front_ports, module_type, existing_front_ports)\n\n if to_create:\n existing_rear_ports = self.get_module_rear_ports(module_type)\n for port in to_create:\n try:\n rear_port = existing_rear_ports[port[\"rear_port\"]]\n port['rear_port'] = rear_port.id\n except KeyError:\n self.handle.log(f'Could not find Rear Port for Front Port: {port[\"name\"]} - '\n + f'{port[\"type\"]} - {module_type}')\n\n try:\n self.counter.update({'updated':\n self.handle.log_module_ports_created(\n self.netbox.dcim.front_port_templates.create(to_create), \"Module Front Port\")\n })\n except pynetbox.RequestError as excep:\n self.handle.log(f\"Error '{excep.error}' creating Module Front Port\")\n\n def upload_images(self,baseurl,token,images,device_type):\n '''Upload front_image and/or rear_image for the given device type\n\n Args:\n baseurl: URL for Netbox instance\n token: Token to access Netbox instance\n images: map of front_image and/or rear_image filename\n device_type: id for the device-type to update\n\n Returns:\n None\n '''\n url = f\"{baseurl}/api/dcim/device-types/{device_type}/\"\n headers = { \"Authorization\": f\"Token {token}\" }\n\n files = { i: (os.path.basename(f), open(f,\"rb\") ) for i,f in images.items() }\n response = requests.patch(url, headers=headers, files=files, verify=(not self.ignore_ssl))\n\n self.handle.log( f'Images {images} updated at {url}: {response}' )\n self.counter[\"images\"] += len(images)\n" }, { "path": "repo.py", "content": "import os\nfrom glob import glob\nfrom re import sub as re_sub\nfrom git import Repo, exc\nimport yaml\n\n\nclass DTLRepo:\n def __new__(cls, *args, **kwargs):\n return super().__new__(cls)\n\n def __init__(self, args, repo_path, exception_handler):\n self.handle = exception_handler\n self.yaml_extensions = ['yaml', 'yml']\n self.url = args.url\n self.repo_path = repo_path\n self.branch = args.branch\n self.repo = None\n self.cwd = os.getcwd()\n\n if os.path.isdir(self.repo_path):\n self.pull_repo()\n else:\n self.clone_repo()\n\n def get_relative_path(self):\n return self.repo_path\n\n def get_absolute_path(self):\n return os.path.join(self.cwd, self.repo_path)\n\n def get_devices_path(self):\n return os.path.join(self.get_absolute_path(), 'device-types')\n\n def get_modules_path(self):\n return os.path.join(self.get_absolute_path(), 'module-types')\n\n def slug_format(self, name):\n return re_sub('\\W+', '-', name.lower())\n\n def pull_repo(self):\n try:\n self.handle.log(\"Package devicetype-library is already installed, \"\n + f\"updating {self.get_absolute_path()}\")\n self.repo = Repo(self.repo_path)\n if not self.repo.remotes.origin.url.endswith('.git'):\n self.handle.exception(\"GitInvalidRepositoryError\", self.repo.remotes.origin.url,\n f\"Origin URL {self.repo.remotes.origin.url} does not end with .git\")\n self.repo.remotes.origin.pull()\n self.repo.git.checkout(self.branch)\n self.handle.verbose_log(\n f\"Pulled Repo {self.repo.remotes.origin.url}\")\n except exc.GitCommandError as git_error:\n self.handle.exception(\n \"GitCommandError\", self.repo.remotes.origin.url, git_error)\n except Exception as git_error:\n self.handle.exception(\n \"Exception\", 'Git Repository Error', git_error)\n\n def clone_repo(self):\n try:\n self.repo = Repo.clone_from(\n self.url, self.get_absolute_path(), branch=self.branch)\n self.handle.log(\n f\"Package Installed {self.repo.remotes.origin.url}\")\n except exc.GitCommandError as git_error:\n self.handle.exception(\"GitCommandError\", self.url, git_error)\n except Exception as git_error:\n self.handle.exception(\n \"Exception\", 'Git Repository Error', git_error)\n\n def get_devices(self, base_path, vendors: list = None):\n files = []\n discovered_vendors = []\n vendor_dirs = os.listdir(base_path)\n\n for folder in [vendor for vendor in vendor_dirs if not vendors or vendor.casefold() in vendors]:\n if folder.casefold() != \"testing\":\n discovered_vendors.append({'name': folder,\n 'slug': self.slug_format(folder)})\n for extension in self.yaml_extensions:\n files.extend(glob(base_path + folder + f'/*.{extension}'))\n return files, discovered_vendors\n\n def parse_files(self, files: list, slugs: list = None):\n deviceTypes = []\n for file in files:\n with open(file, 'r') as stream:\n try:\n data = yaml.safe_load(stream)\n except yaml.YAMLError as excep:\n self.handle.verbose_log(excep)\n continue\n manufacturer = data['manufacturer']\n data['manufacturer'] = {\n 'name': manufacturer, 'slug': self.slug_format(manufacturer)}\n\n # Save file location to resolve any relative paths for images\n data['src'] = file\n\n if slugs and True not in [True if s.casefold() in data['slug'].casefold() else False for s in slugs]:\n self.handle.verbose_log(f\"Skipping {data['model']}\")\n continue\n\n deviceTypes.append(data)\n return deviceTypes\n" }, { "path": "requirements.txt", "content": "GitPython==3.1.32\npynetbox==7.4.0\npython-dotenv==1.0.0\nPyYAML==6.0.1\n" }, { "path": "settings.py", "content": "from argparse import ArgumentParser\nimport os\nfrom log_handler import LogHandler\nfrom repo import DTLRepo\nfrom dotenv import load_dotenv\nload_dotenv()\n\nREPO_URL = os.getenv(\"REPO_URL\",\n default=\"https://github.com/netbox-community/devicetype-library.git\")\nREPO_BRANCH = os.getenv(\"REPO_BRANCH\", default=\"master\")\nNETBOX_URL = os.getenv(\"NETBOX_URL\")\nNETBOX_TOKEN = os.getenv(\"NETBOX_TOKEN\")\nIGNORE_SSL_ERRORS = (os.getenv(\"IGNORE_SSL_ERRORS\", default=\"False\") == \"True\")\nREPO_PATH = f\"{os.path.dirname(os.path.realpath(__file__))}/repo\"\n\n# optionally load vendors through a comma separated list as env var\nVENDORS = list(filter(None, os.getenv(\"VENDORS\", \"\").split(\",\")))\n\n# optionally load device types through a space separated list as env var\nSLUGS = os.getenv(\"SLUGS\", \"\").split()\n\nNETBOX_FEATURES = {\n 'modules': False,\n}\n\nparser = ArgumentParser(description='Import Netbox Device Types')\nparser.add_argument('--vendors', nargs='+', default=VENDORS,\n help=\"List of vendors to import eg. apc cisco\")\nparser.add_argument('--url', '--git', default=REPO_URL,\n help=\"Git URL with valid Device Type YAML files\")\nparser.add_argument('--slugs', nargs='+', default=SLUGS,\n help=\"List of device-type slugs to import eg. ap4431 ws-c3850-24t-l\")\nparser.add_argument('--branch', default=REPO_BRANCH,\n help=\"Git branch to use from repo\")\nparser.add_argument('--verbose', action='store_true', default=False,\n help=\"Print verbose output\")\n\nargs = parser.parse_args()\n\nargs.vendors = [v.casefold()\n for vendor in args.vendors for v in vendor.split(\",\") if v.strip()]\nargs.slugs = [s for slug in args.slugs for s in slug.split(\",\") if s.strip()]\n\nhandle = LogHandler(args)\n# Evaluate environment variables and exit if one of the mandatory ones are not set\nMANDATORY_ENV_VARS = [\"REPO_URL\", \"NETBOX_URL\", \"NETBOX_TOKEN\"]\nfor var in MANDATORY_ENV_VARS:\n if var not in os.environ:\n handle.exception(\"EnvironmentError\", var,\n f'Environment variable \"{var}\" is not set.\\n\\nMANDATORY_ENV_VARS: {str(MANDATORY_ENV_VARS)}.\\n\\nCURRENT_ENV_VARS: {str(os.environ)}')\n\ndtl_repo = DTLRepo(args, REPO_PATH, handle)\n" } ]