Repository: WindRiver-Labs/wrlinux-x Branch: WRLINUX_10_21_BASE Commit: 1bccf8827e6b Files: 89 Total size: 460.4 KB Directory structure: gitextract_d8ghdmvu/ ├── .gitignore ├── EULA ├── LICENSE ├── README.md ├── bin/ │ ├── .pylintrc │ ├── LICENSE.texttable │ ├── Makefile │ ├── argparse_setup.py │ ├── argparse_wrl.py │ ├── branch_mirror.py │ ├── dump_layer_dependencies.py │ ├── dump_layer_rev.py │ ├── flatten_mirror.py │ ├── layer_index.py │ ├── logger_setup.py │ ├── sanity.py │ ├── settings.py │ ├── setup.py │ ├── test-network.py │ ├── texttable.py │ ├── toaster_fixture.py │ ├── transform_index.py │ ├── utils_setup.py │ └── windshare.py ├── data/ │ ├── environment.d/ │ │ ├── 00_check_repo.sh │ │ ├── 00_wrl_eula.sh │ │ ├── 01_wrl_buildtools.sh │ │ ├── 02_wrl_anspass.sh │ │ ├── 03_wrl_askpass.sh │ │ ├── 04_wrl_buildtools.sh │ │ ├── 05_wrl_anspass.sh │ │ ├── 06_wrl_repo.sh │ │ ├── 07_toaster_update.sh │ │ ├── check_update.sh │ │ ├── kernel_type.sh │ │ ├── setup_anspass │ │ ├── setup_askpass │ │ ├── setup_ssh │ │ └── setup_utils │ ├── local_layer/ │ │ ├── README │ │ ├── classes/ │ │ │ └── .keepme │ │ ├── conf/ │ │ │ └── layer.conf │ │ ├── downloads/ │ │ │ └── .keepme │ │ ├── git/ │ │ │ └── .keepme │ │ ├── recipes-sample/ │ │ │ └── hello/ │ │ │ ├── hello/ │ │ │ │ └── hello.c │ │ │ └── hello_1.0.bb │ │ └── scripts/ │ │ └── .keepme │ ├── samples/ │ │ ├── README-MIRROR.sample │ │ ├── README.sample │ │ ├── bblayers.conf.sample │ │ ├── conf-notes.sample │ │ ├── local.conf.sample │ │ └── site.conf.sample │ └── xml/ │ ├── axxiaarm-dl.xml │ ├── bitbake.inc │ ├── intel-socfpga-dl.xml │ ├── meta-anaconda-dl.xml │ ├── meta-cgl-common-dl.xml │ ├── meta-clang-dl.xml │ ├── meta-cloud-services-dl-3-3.xml │ ├── meta-dpdk-dl.xml │ ├── meta-efi-secure-boot-dl.xml │ ├── meta-encrypted-storage-dl.xml │ ├── meta-freescale-dl.xml │ ├── meta-ids-dl.xml │ ├── meta-integrity-dl.xml │ ├── meta-intel-dl.xml │ ├── meta-iot-cloud-dl.xml │ ├── meta-openembedded-dl-3-3.xml │ ├── meta-qt5-dl.xml │ ├── meta-raspberrypi-dl.xml │ ├── meta-realtime-dl-3-3.xml │ ├── meta-security-compliance-dl.xml │ ├── meta-selinux-dl.xml │ ├── meta-signing-key-dl.xml │ ├── meta-tensorflow-dl-3-3.xml │ ├── meta-tpm-dl.xml │ ├── meta-tpm2-dl.xml │ ├── meta-virtualization-dl-3-3.xml │ ├── nxp-imx7.xml │ ├── nxp-s32g2xx-dl.xml │ ├── oe-core-dl-3-3.xml │ ├── openembedded-core.inc │ ├── ti-j72xx-dl.xml │ ├── wr-ostree-dl.xml │ ├── wrlinux-dl.xml │ ├── wrlinux.xml │ └── xilinx-zynqmp-dl.xml └── setup.sh ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ buildtools.git *.pyc bin/.venv *.pyo /*.patch *.swp *.orig *.rej *~ ================================================ FILE: EULA ================================================ WIND RIVER LINUX PLATFORM SOFTWARE LICENSE AGREEMENT This Wind River Linux Platform Software License Agreement (the "Agreement"), is made between Wind River Systems, Inc. ("Wind River"), on the one hand, and the individual accessing or using the Software or any single legal entity on behalf of which such individual is acting ("Customer"), on the other hand. The effective date of this Agreement is the earlier of the date that you accept this Agreement or the date that Wind River makes the Products available to you ("Effective Date"). PLEASE READ THIS AGREEMENT CAREFULLY BEFORE DOWNLOADING, ACCESSING, INSTALLING OR USING THE PRODUCTS. BY CLICKING ON THE "I ACCEPT" BUTTON YOU: ACKNOWLEDGE THAT YOU HAVE READ AND UNDERSTAND THIS AGREEMENT, AGREE TO BE BOUND BY ITS TERMS AND CONDITIONS, AND WARRANT YOU HAVE AUTHORITY TO ENTER INTO THIS AGREEMENT. IF YOU ARE ACTING ON BEHALF OF AN ENTITY, YOU REPRESENT THAT YOU HAVE THE AUTHORITY TO ENTER INTO THIS AGREEMENT ON BEHALF OF THAT ENTITY. IF YOU DO NOT ACCEPT THE TERMS OF THIS AGREEMENT, THEN YOU MUST NOT USE THE PRODUCTS AND SERVICES AND YOU MUST UNINSTALL AND RETURN THE PRODUCT TO WIND RIVER WITHIN TEN (10) DAYS. IF YOU ARE EVALUATING THE PRODUCT, THE TERMS SPECIFIED IN APPENDIX B SHALL APPLY IN ADDITION OF THE TERMS OF THE AGREEMENT AND APPENDIX A. ________________________________________________________________________ The parties agree as follows: 1. SCOPE. This framework Agreement governs Wind River's provision to Customer of the Products and Services as detailed in the Quote or Exhibit. Terms in initial capitalized letters, not defined elsewhere in this Agreement, have the meanings set forth in Section 12 (Definitions). 2. LICENSE. Each program, component or application included in the Software may be governed by one or more Open Source License(s), as specified or referenced in the respective Source Code files. Without limiting the foregoing, for the convenience of Customer, Wind River will make available, upon request, applicable licensing information in the Linux Foundation's SPDX(R) format. Tools provided to Customer shall be licensed under the terms of the attached Appendix A, which is hereby incorporated by reference into this Agreement. 3. SERVICES. 3.1. Support. Customer's payment of the applicable fees entitles Customer to Support during the License Term. Support is provided in accordance with the terms of Wind River's then current Software Support and Maintenance Agreement, which terms are hereby incorporated by reference into this Agreement. A copy of the curr ent Software Support and Maintenance Agreement can be accessed through Wind River's website: http://www.windriver.com/support/SupportTerms.pdf 3.2. Professional Services. Professional services, if included in a signed statement of work, will be provided at additional cost and subject to Wind River's then current Professional Services Terms and Conditions, which terms are hereby incorporated by reference into this Agreement. A copy of the current Professional Services Terms and Conditions can be accessed through Wind River's website: http://windriver.com/services/ProductAgreementServicesAttachment.pdf 3.3. Education and Training. Education and training services (e.g., mentoring, live or remote classes, etc.), if included in the Quote or Exhibit will be provided pursuant to Wind River's then current Education Services terms and conditions, which terms are hereby incorporated by reference into this Agreement. A copy of the current Education Services terms and conditions can be accessed through Wind River's website: http://education.windriver.com/content/public/resources/Supplemental_Education_Services_Terms_and_Conditions.pdf 4. CONFIDENTIAL INFORMATION. Customer will not use or disclose any Confidential Information, except as expressly authorized by this Agreement, and will protect all such Confidential Information using the same degree of care that Customer uses with respect to its own proprietary information, but in no event less than reasonable care. If Confidential Information must be disclosed to a third party by reason of legal, accounting or regulatory requirements beyond the reasonable control of Customer, Customer will: (i) assert the confidential nature of the information in the action or proceeding; (ii) promptly notify Wind River of the order or request; and (iii) permit Wind River (at its own expense) to seek an appropriate protective order. 5. TERM AND TERMINATION. 5.1. Term. The term of this Agreement will commence upon the Effective Date and continue until expiration of the License Term unless earlier terminated as set forth herein. The renewal date for all Products and Services hereunder will be the last day of the applicable License Term, regardless of whether Customer chooses to license additional Products during such License Term. Customer must notify Wind River at least thirty (30) days prior to the end of the License Term of its request to renew its subscription. Unless the parties mutually agree to renew the subscription, and Customer pays the then-applicable fees, Customer's license to the Tools and entitlement to Support will terminate on the last day of the then-current License Term. 5.2. Termination. Either party may terminate this Agreement immediately upon notice for the material breach of the other party, which breach is curable and has remained uncured for a period of thirty (30) days from the date of notice. In addition, Wind River may immediately terminate this Agreement if Customer is delinquent on its payment obligations for thirty (30) days or more. 5.3. Consequences of Expiration or Termination. Upon expiration of the License Term or termination of this Agreement, Customer will: (a) not use the Tools for any purpose whatsoever; (b) no longer be entitled to receive or have access to Support or other Services; (c) immediately destroy or return to Wind River all material belonging to Wind River or its licensors, including, without limitation, all Confidential Information of Wind River then in Customer's possession; and (d) promptly certify to Wind River in writing that Customer has completed the actions described in the foregoing subsections. These remedies will be cumulative and in addition to any other remedies available to Wind River. Sections 4, 5, 6, 7, 8, 9, 10, 11 and 12 will survive expiration or termination of this Agreement. 6. DELIVERY, PAYMENT AND TAXES. 6.1. Delivery and Payment Terms. 6.1.1. The Product will be deemed irrevocably accepted upon shipment. When Product is made available to Customer for electronic download through access to a website and/or server, shipment will be deemed to have occurred at the time of such availability. 6.1.2. Customer will pay Wind River the fees set forth in the applicable Exhibit or Quote. Wind River will invoice Customer the applicable fees upon receipt of a correctly issued purchase order from Customer, which may coincide with shipment of the Software or Tools. Payment terms herein are subject to prior credit approval by Wind River. Customer will pay all invoices within thirty (30) days of the date of such invoice in U.S. dollars unless otherwise set forth in the applicable Exhibit or Quote. Interest on late payments will accrue at the rate of one and one-half percent (1.5%) per month or partial month, or, if lower, the highest rate permitted by law. Customer is responsible for reasonable costs associated with collection of overdue amounts, including without limitation, reasonable attorneys' fees and court costs. Failure to pay any invoice in the manner described in this Section 6 may, at Wind River's discretion, be deemed a material breach of this Agreement. 6.1.3. If Products or Services are purchased through an authorized Wind River distributor, Customer will be invoiced directly by such distributor (and not Wind River), and Customer will pay the distributor the applicable fees in accordance with the payment terms agreed to between Customer and the distributor. 6.2. Purchase Orders. Unless Products or Services are purchased through an authorized Wind River distributor, Customer will submit to Wind River a purchase order in the amount specified in the corresponding Exhibit or Quote. Any additional or different terms and conditions on the purchase order will be of no force or effect. Wind River may reject any purchase order that Wind River determines to be outside the scope of this Agreement. 6.3. Taxes. All amounts due hereunder will be paid without deduction, set-off or counter claim, free and clear of any restrictions or conditions, and without deduction for any taxes, levies, imposts, duties, fees, deductions, withholdings or other governmental charges. If any deduction is required to be made by law, Customer will pay the full amount owed to Wind River plus the amount to be deducted, so that Wind River will receive the same amount as it would have received had the deduction not been required. If Customer is claiming sales or use tax exemption, a valid and fully executed Tax Exempt Certificate must be attached to this Agreement or the applicable Quote or Exhibit, or the applicable purchase order. Customer will promptly pay or reimburse all taxes (exclusive of taxes on Wind River's net income), duties and assessments arising from amounts payable to Wind River under this Agreement, or furnish Wind River with evidence acceptable to the taxing authority to sustain exemption therefrom. 7. CUSTOMER COMPLIANCE. Wind River or its designated representative will have the right to: (a) require that Customer send a written certification of compliance with the terms of this Agreement within fifteen (15) days of Wind River's request; and (b) conduct an inspection and audit ("Audit") upon reasonable notice of the relevant operational, accounting and sales books and records of Customer and any other information within Customer's possession or control that is reasonably necessary to determine whether Customer has complied with this Agreement, and obtain true and correct photocopies of the foregoing materials. Such Audit will be conducted during regular business hours at Customer's offices and so as not to interfere unreasonably with Customer's normal business activities. Customer will permit or provide for completion of any Audit within forty-five (45) days of notice. If an Audit discloses any underpayment of fees, then Wind River will invoice Customer for, and Customer will promptly pay Wind River, the underpaid fee amount based on the higher of the price specified in this Agreement or Wind River's price list in effect at the time the audit is completed, together with late payment interest in accordance with Section 6.1 (Delivery and Payment Terms). If an Audit discloses underpayment of five percent (5%) or more, then Wind River will also invoice Customer for, and Customer will promptly pay, Wind River's reasonable costs of conducting the Audit. 8. LIMITED WARRANTY. 8.1. Limited Warranty. The Software is provided "AS IS". In no event does Wind River warrant that the Software is error free or that the Software will satisfy Customer's own specific requirements. Customer assumes full responsibility for: (a) the selection of the Product; (b) the proper installation and use of the Products; (c) verifying the results obtained from the use of the Products; and (d) taking appropriate measures to prevent loss of data. 8.2. Warranty Disclaimer. EXCEPT AS EXPRESSLY SET FORTH IN SECTION 8.1 ABOVE, WIND RIVER AND ITS LICENSORS DISCLAIM ALL WARRANTIES, EXPRESS, IMPLIED AND STATUTORY INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT OF THIRD PARTY RIGHTS WITH RESPECT TO THE PRODUCTS AND SERVICES PROVIDED UNDER THIS AGREEMENT. NO ORAL OR WRITTEN INFORMATION OR ADVICE GIVEN BY WIND RIVER, ITS DEALERS, DISTRIBUTORS, AGENTS OR EMPLOYEES WILL IN ANY WAY INCREASE THE SCOPE OF THE WARRANTIES PROVIDED BY WIND RIVER IN THIS AGREEMENT OR THE APPLICABLE OPEN SOURCE LICENSES. WIND RIVER DOES NOT WARRANT THAT THE QUALITY OR PERFORMANCE OF ANY PRODUCT AND SERVICES WILL MEET CUSTOMER'S REQUIREMENTS OR THAT CUSTOMER WILL BE ABLE TO ACHIEVE ANY PARTICULAR RESULTS FROM USE OR MODIFICATION OF THE PRODUCT, OR THAT THE PRODUCT WILL OPERATE FREE FROM ERROR. WIND RIVER MAKES NO WARRANTY WITH RESPECT TO ANY MALFUNCTIONS OR OTHER ERRORS IN ITS PRODUCTS CAUSED BY VIRUS, INFECTION, WORM OR SIMILAR MALICIOUS CODE NOT DEVELOPED BY WIND RIVER. WIND RIVER MAKES NO WARRANTY THAT ANY PRODUCTS OF WIND RIVER WILL PROTECT AGAINST ALL POSSIBLE SECURITY THREATS, INCLUDING INTENTIONAL MISCONDUCT BY THIRD PARTIES. WIND RIVER IS NOT LIABLE FOR ANY DOWNTIME OR SERVICE INTERRUPTION, FOR ANY LOST OR STOLEN DATA OR SYSTEMS, OR FOR ANY OTHER DAMAGES ARISING OUT OF OR RELATING TO ANY SUCH ACTIONS OR INTRUSIONS. Some jurisdictions do not allow the limitation or exclusion of implied warranties or how long an implied warranty may last, so the above limitations may not apply to Customer. This warranty gives Customer specific legal rights and Customer may have other rights that vary from jurisdiction to jurisdiction. 9. CUSTOMER INDEMNITY. Customer will defend at its expense Wind River against any claims, suits, or proceedings brought against Wind River by third parties resulting from, in connection with, attributable to any Customer product or service incorporating or using the Software, or by Customer's direct or indirect end users, and will indemnify Wind River for any costs, losses, damages, and expenses (including reasonable legal fees) resulting from or in connection with such claim, suit, or proceeding. Customer's obligations under this Section are conditioned on Wind River: (a) providing Customer prompt notice of such claim, suit, or proceeding; (b) tendering to Customer the defense or settlement of any such claim, suit, or proceeding at Customer's expense; and (c) cooperating with Customer, at Customer's expense, in defending or settling such claim, suit, or proceeding. 10. LIMITATION OF LIABILITY. WIND RIVER AND ITS LICENSORS WILL NOT BE LIABLE TO CUSTOMER OR ANY THIRD PARTY FOR ANY LOSS OF PROFITS, REVENUE OR GOODWILL, COSTS OF PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES, LOSS OR INTERRUPTION OF BUSINESS, LOSS OF ANTICIPATED SAVINGS, OR LOSS OF DATA, OR ANY INDIRECT, EXEMPLARY, PUNITIVE, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES OF ANY KIND, HOWEVER CAUSED AND REGARDLESS OF THE FORM OF ACTION, WHETHER IN CONTRACT, TORT (INCLUDING NEGLIGENCE), STRICT PRODUCT LIABILITY OR ANY OTHER LEGAL OR EQUITABLE THEORY, EVEN IF WIND RIVER HAS BEEN ADVISED OR SHOULD HAVE KNOWN OF THE POSSIBILITY OF SUCH DAMAGES. IN NO EVENT WILL WIND RIVER'S AGGREGATE CUMULATIVE LIABILITY FOR ANY CLAIMS ARISING OUT OF OR RELATED TO THIS AGREEMENT EXCEED THE AMOUNTS PAID TO WIND RIVER BY CUSTOMER FOR THE AFFECTED PRODUCT OR SERVICES PURSUANT TO THIS AGREEMENT IN THE TWELVE (12) MONTHS PRECEDING THE CLAIM. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages so this limitation and exclusion may not apply to Customer. THE LIMITED WARRANTY, LIMITED REMEDIES, WARRANTY DISCLAIMER AND LIMITED LIABILITY ARE FUNDAMENTAL ELEMENTS OF THE BASIS OF THE BARGAIN BETWEEN WIND RIVER AND CUSTOMER. WIND RIVER WOULD NOT BE ABLE TO PROVIDE THE PRODUCTS OR SERVICES WITHOUT SUCH LIMITATIONS. 11. GENERAL. 11.1. Governing Law and Venue. This Agreement will be governed in all respects by the laws of the United States of America and the State of Delaware, without reference to conflict of laws principles. The United Nations Convention on Contracts for the International Sale of Goods is specifically excluded from application to to this Agreement. All disputes arising under this Agreement will be brought exclusively in the State of Delaware or of the Federal courts sitting therein, provided, however, that the parties will be entitled to seek injunctive relief in the appropriate forum. Customer consents to the personal jurisdiction of the above courts. 11.2. Export Control. All software and technical information delivered under this Agreement are subject to U.S. export controls under the Export Administration Regulations (the "EAR") or the International Traffic in Arms Regulations (the "ITAR") and may be subject to export, re-export or import regulations in other countries. Customer agrees to strictly comply with all such laws and regulations. Customer will not export or re-export the software and technical information, directly or indirectly, to: (a) any countries that are subject to US export restrictions (currently including, but not necessarily limited to, Cuba, Iran, North Korea, Sudan, and Syria); (b) any end user who Customer knows or has reason to know will utilize them in the design, development or production of nuclear, chemical or biological weapons, or rocket systems, space launch vehicles, and sounding rockets, or unmanned air vehicle systems; or (c) any end user who has been prohibited from participating in US export transactions by any federal agency of the US government. Some of Wind River's products are classified as "restricted" encryption products under Section 740.17(b)(2) of the EAR and may not be exported or re-exported to government end-users (as defined in Section 772 of the EAR) outside the countries listed in Supplement No. 3 to Part 740 of the EAR without authorization from the U.S. government. 11.3. U.S. Government End-Users. All software and any data relating thereto or derived therefrom are "commercial items" as defined in 48 C.F.R. 2.101, consisting of "commercial computer software" and "commercial computer software documentation" as such terms are used in 48 C.F.R. 12.212. If the end User is a U.S. Government agency, department, or instrumentality, then the use, duplication, reproduction, release, modification, disclosure or transfer of the Software and any data relating thereto or derived therefrom, is restricted in accordance with 48 C.F.R. 12.211, 48 C.F.R. 12.212, 48 C.F.R. 227.7102-2, and 48 C.F.R. 227.7202, as applicable. Customer will provide the Software to the U.S. Government or to U.S. Government end Users only pursuant to an end user license agreement in accordance with the terms of this Agreement. This U.S. Government end users clause is in lieu of, and supersedes, any Federal Acquisition Regulations ("FAR"), the Defense FAR Supplement ("DFARS"), or other clause or provision that addresses Government rights in computer software or technical data. 11.4. Notices. All notices under this Agreement will be: (a) in writing; (b) delivered by personal delivery or certified or registered mail, return receipt requested, and deemed given upon personal delivery or five (5) days after deposit in the mail. Notices to Wind River will be sent to Wind River Systems, Inc., General Counsel, Legal Affairs, 500 Wind River Way, Alameda, CA 94501, USA (or such other address designated in writing by Wind River), and notices to Customer will be sent to the address identified in Customer's purchase order. 11.5. Force Majeure. Neither party will be liable for any failure or delay (except for the payment of money) on account of strikes, shortages, riots, insurrection, fires, explosions, acts of God, war, governmental action, labor conditions, material shortages or any other cause which is beyond the reasonable control of such party. 11.6. Use of Customer's Name. Subject to Customer's approval, Wind River may identify Customer as a customer of Wind River products or services. 11.7. Assignment. Customer will not assign, transfer or delegate any right, license or obligation under this Agreement to a third party, directly or indirectly, including by operation of law or through bankruptcy, merger, acquisition, sale or transfer of all, substantially all or any part of the business or assets of Customer, or undergo a change of Control, without first obtaining Wind River's prior written consent, which Wind River may withhold in Wind River's sole discretion. Any purported assignment, transfer, delegation or change of Control in violation of this Section 11.7 (Assignment) is null and void. Wind River may assign, transfer or delegate this Agreement or any right, license or obligation hereunder in its sole discretion. 11.8. Counterparts. In the event th is Agreement is executed with signatures, this Agreement may be executed in one or more counterparts, each of which will be deemed an original, but all of which together will constitute one and the same instrument. Counterparts to this Agreement transmitted by facsimile transmission, by electronic mail in "portable document format" (".pdf") form, or by any other electronic means intended to preserve the original graphic and pictorial appearance of a document, will have the same effect as a signed original of this Agreement. 11.9. Waiver. The failure of either party to require performance by the other party of any provision hereof will not affect the full right to require such performance at any time thereafter; nor will the waiver by either party of a breach of any provision hereof be taken or held to be a waiver of the provision itself. 11.10. Miscellaneous. This Agreement will not create any agency, employment relationship, partnership or other form of joint enterprise between the parties. This Agreement and its attachments and exhibits constitute the entire agreement between Customer and Wind River and supersedes all prior oral or written agreements between the parties with respect to the subject matter hereof. The terms and conditions of any purchase order or other instrument issued by Customer in connection with this Agreement will be of no force or effect. This Agreement may only be amended by a writing signed by the parties that refers explicitly to this Agreement. If a provision of this Agreement is unenforceable or invalid, the provision will be revised so as to best accomplish the objectives of the parties. 12. DEFINITIONS. In addition to the terms in initial capitalized letters defined elsewhere in this Agreement, the following terms in initial capitalized letters have the respective meanings set forth below: 12.1. "Confidential Information" means information disclosed by Wind River to Customer, including without limitation, intellectual property, software programs, documentation, pricing, current, future, and proposed products and services, which is (a) in tangible form and bears a "confidential", "proprietary" or similar legend; and (b) discussions relating to that information whether those discussion occur prior to, concurrent with, or following disclosure of the information. Wind River shall make reasonable efforts to mark its confidential information in tangible form with any of the aforementioned legends prior to disclosure. However, Wind River's information in tangible form that does not bear any of these legends, and discussions relating to that information, shall nevertheless be protected hereunder as Confidential Information, if Customer knew, or should have reasonably known under the circumstances, that the information was confidential and had been communicated to it in confidence. Confidential Information does not include any information that Customer reasonably demonstrates is: (i) published or otherwise available to the public other than by breach of this Agreement by Customer; (ii) rightfully received by Customer from a third party without confidentiality limitations; (iii) independently developed by Customer as evidenced by appropriate records; (iv) known to Customer prior to its first receipt from Wind River as evidenced by appropriate records; (v) hereinafter disclosed by Wind River to a third party without restriction on disclosure; or (vi) approved for public release by written authorization of Wind River. 12.2. "Control" means: (a) ownership of more than fifty percent (50%) of the outstanding stock or securities entitled to vote for the election of directors or similar managing authority of the subject entity; (b) ownership of more than fifty percent (50%) of the ownership interest that represents the right to make decisions for the subject entity; (c) any other ability to elect more than half of the board of directors or similar managing authority of the subject entity, whether by contract or otherwise; or (d) the possession, directly or indirectly, of the power to direct or cause the direction of the management or policies of the subject entity whether through the ownership of voting securities, through other voting rights, by contract or otherwise. 12.3. "Exhibit" means an attachment to this Agreement that the parties may elect to simultaneously or separately execute from time to time, in addition to the Quote, detailing Products and Services to be provided to Customer, as well as other business restrictions associated with such Products and Services (e.g., Project and Development Locations). 12.4. "License Term" means the term set forth in the Quote or Exhibit or one (1) year commencing on the Effective Date if no term is specified. 12.5. "Object Code" means computer programming code in a form not readily perceivable by humans and suitable for machine execution without the intervening steps of interpretation or compilation. 1 2.6. "Open Source License" means a software license under which the Soft ware Object Code and/or Source Code is made available under terms that allow any licensee to copy, create derivative works and distribute the software without any fee or cost, including but not limited to any open source license listed by the Open Source Initiative (http://www.opensource.org). 12.7. "Product" means Software and Tools. 12.8. "Quote" means the Wind River final sales quote detailing the Products and Services to be provided to Customer, as well as other business restrictions associated with such Products and Services (e.g., Project and Development Locations). 12.9. "Services" means those services provided by Wind River as detailed in the Quote, statement of work or Exhibit, and includes, but is not limited to Support, professional services and education services. 12.10. "Software" means the Linux operating system components (such as the kernel, BSPs and drivers) and any add-on profiles and other related software components, as specified in the Quote or Exhibit and made available by Wind River to Customer in Source Code or Object Code form, including any updates thereto and all accompanying documentation governed by an Open Source License. Software does not include Tools. 12.11. "Source Code" means computer programming code in human readable form that is not suitable for machine execution without the intervening steps of interpretation or compilation. 12.12. "SPDX" means the Software Package Data Exchange specification for communicating the components, licenses and copyrights associated with a software package, as developed by the Linux Foundation. 12.13. "Support" means the annual support and maintenance services specified in the Quote or Exhibit to be provided by Wind River in accordance with Section 3.1 (Support & Maintenance). 12.14. "Tools" means the proprietary Source Code and Object Code and any accompanying documentation provided by Wind River under this Agreement which is intended for Customer's internal use in accordance with the terms of the Tools Agreement in Addendum 1 (e.g., Wind River Workbench). Tools shall be deemed "Confidential Information". APPENDIX A - WIND RIVER LINUX PLATFORM TOOLS ADDITIONAL TERMS 1. SCOPE. These terms ("Tools Agreement") govern Customer's rights and obligations with respect to the Tools, as specified in the Quote or Exhibit. Terms in initial capitalized letters, not defined in the Agreement or in this Tools Agreement, have the meanings set forth in Section 5 (Definitions). The terms of the Agreement hereby incorporated by reference into this Tools Agreement. The terms of this Tools Agreement shall supersede any conflicting provisions of the Agreement for purposes of such Tools. 2. LICENSE GRANTS. Subject to Customer's compliance with the terms and conditions of this Tools Agreement and any Quote or Exhibit, and payment of the applicable fees, Wind River grants to Customer, during the License Term, a restricted, personal, non-transferable, non-exclusive, internal-use license: 2.1. for Floating User Development Licenses, to reproduce and use one (1) copy of the Tools per Development License on the Supported Architecture Family, solely for the number of licensed concurrent Floating Users located at each Development Location and specified as licensed for such Tools in such Exhibit or Quote; OR 2.2. for Named User Development Licenses, to reproduce and use one (1) copy of the Tools per Development License on the Supported Architecture Family, solely for the licensed Named Users located at each Development Location and specified as licensed for such Tools in such Exhibit or Quote. Customer will use license management technology in the Tools to assign each Development License to a Named User and no other User may use the copy of the Tools designated for that Development License without the prior written consent of Wind River; except, however, that Customer may (a) transfer a Named User Development License from one Named User to another individual if the Named User has left Customer's business, and is replaced by another Named User, or (b) make legal name changes or network UserID changes to the Named User profile information, subject to notification to Wind River submitted to Wind River's email alias (CustomerReports@windriver.com); OR 2.3. for Node-Lock User Development Licenses, to reproduce, access and use one (1) copy of the Tools per Development License, solely on the number of "Node-locked hosts" licensed and located at each Development Location and specified as licensed for such Tools in such Exhibit A or Quote. For purposes of this Agreement, the "Node" means the host computer where both the Tools and license management file are physically installed. 3. GENERAL RESTRICTIONS. 3.1. General. Tools are provided to Customer in Object Code and are intended for Customer's internal use in developing applications in accordance with the Development License and not for distribution. Except as expressly permitted by this Tools Agreement, Customer will not, nor permit any third party to: (a) for Floating User Development Licenses: (i) permit access to or use of the Tools in excess of the quantity of concurrent Floating Users set forth in the applicable Exhibit or Quote; (ii) transfer any Floating User Development License from a Development Location in one country to a Development Location in another country; or (iii) allow one Floating User in one country to access the Tools from a licensed server in another country; (b) for Named User Development License, permit concurrent access to or use of a single copy of the Tools for any Named User Development License, or access or use by anyone other than the Named User assigned to the Development License, as set forth in the applicable Exhibit or Quote; (c) for Node-Lock User Development Licenses, install the Tools on more than the quantity of licensed Nodes set forth in the applicable Exhibit or Quote; (d) translate, reverse engineer, decompile, disassemble (except to the extent applicable laws specifically prohibit such restriction) or attempt to derive the Source Code of any Tools Object Code provided to Customer; (e) reproduce the Tools other than as specified in the applicable license grant set forth in Section 2 (License Grants); (f) sublicense, rent, lease, loan, timeshare, sell, distribute, disclose, publish, assign or transfer any rights, grant a security interest in, transfer possession of the Tools, or electronically transfer the Tools from one computer to another except over Customer's internal network pursuant to Section 3.2 (Network Access); (g) modify, distribute or otherwise use the Tools in any manner that causes any portion of the Tools that is not already subject to Open Source License to become subject to the terms of any Open Source License; or (h) alter or remove any of Wind River's or its licensors' copyright or proprietary rights notices or legends appearing on or in the Tools. Customer will reproduce such notices on any copies of the Tools Customer is permitted to make. Customer will inform all Users of the restrictions set forth in this Tools Agreement with respect to use of the Tools. 3.2. Network Access. Floating and Named Users who have a unique network log-in (e.g., NT or Unix log-in) may access the Tools from a secure network server located at the Development Location over a secure VPN or equivalent secure network. Within thirty (30) days following Wind River's written request, Customer will disclose to Wind River the Development Location of each network from which Tools will be accessed. Node-Lock Users will not remotely access the licensed node, where "remote access" means any access to the node other than being physically present at the host computer. 3.3. Third Party Software. The Tools licensed under this Tools Agreement may contain or be derived from materials of third party licensors such as materials subject to an Open Source Licenses ("Third Party Software"). Such third party materials may be subject to restrictions in addition to those listed in this Section 3 (General Restrictions), which restrictions, if any, are set forth in the third party notice file that accompanies the Tools. Customer acknowledges and agrees that its use of Third Party Software is subject to its compliance with any such additional terms. 3.4 Copies. Customer may make a reasonable number of copies of the Tools for archival purposes or for use as a back-up. Customer must copy all copyright legends, trademarks, trade names, and other legends on any copies of the Tools. 4. KEYS AND ACCESS. Wind River will provide Customer those keys that are reasonably necessary to permit Customer to gain access to the licensed Tools. Any technology, ideas, know how, documentation, processes, algorithms, and trade secrets disclosed to Customer by Wind River in connection with or as embodied in the Tools and keys shall be considered "Confidential Information" as defined in the Agreement. Customer will disclose such keys solely to authorized Users of the Tools. The Tools also contains license management technology intended to cause the Tools to cease operating upon expiration or termination of the license. Customer will not circumvent the license management technology, or any other security devices, access logs, or other protective measures provided with the Tools or permit or assist any User or any third party to do the same. The license management technology and any other protective measures in the Tools are included solely as a matter of administrative convenience, and Customer has no right or license in or to such protective measures. Customer will be invoiced for any additional Users who access the Tools and will pay any such invoice in accordance with Section 6 (Delivery, Payment and Taxes) of the Agreement. Customer will follow the recommended installation procedures for each Tools product licensed under this Tools Agreement. 5. DEFINITIONS. In addition to the terms in initial capitalized letters defined elsewhere in this Agreement, the following terms in initial capitalized letters have the respective meanings set forth below: 5.1. "Development License" means the right of a User to use certain Tools pursuant to Section 2 (License Grants) of this Tools Agreement. The Development License type and quantity will be specified in an Exhibit or Quote. 5.2. "Development Location" means the geographic address of each Customer site or facility where the Tools are authorized to be installed and used in accordance with the terms of this Agreement. Customer's Development Location(s) will be specified in an Exhibit A or Quote. 5.3. "Floating User" means any User who may access and use the Tools pursuant to Section 2.1 (Floating User Development License). 5.4. "Named User" means any User who uses the Tools pursuant to Section 2.2 (Named User Development License). 5.5. "Node-Lock User" means any User who uses the Tools pursuant to Section 2.3 (Node-Lock User Development License). 5.6. "Supported Architecture Family" means the target architecture family or families then-supported by Wind River on which Customer may use the Tools pursuant to this Agreement, as specified in an Exhibit or Quote. 5.7. "User" means a Customer employee or independent contractor who (a) is based at a Development Location, (b) will utilize the Tools on behalf of Customer, and (c) has signed a confidentiality agreement with Customer in which such employee or independent contractor agrees to protect third party confidential information with terms no less stringent than those set forth herein. A User may be either: a Floating User, Named User or Node-Lock User, as such terms are further defined in Section 2 (License Grants). For the avoidance of doubt, a User will be deemed to have utilized the Tools if such User has accessed or used any portion of the Tools, including without limitation, to accomplish the goal of, or which results in, any of the following: (a) developing Customer products, including without limitation, software systems, devices, applications or API code, as further specified in Section 2.1(License Grants); (b) using Tools to compile any code; or other similar engineering activity or result. 5.8. "Externally downloadable 3rd party components" means third party software that we don't include with our code when we ship, however we do include recipes/hooks etc that enable downloading from the third party website. By selecting this component, you understand that it will be downloaded and installed from a third party website. Such software may involve additional licensing terms. Please contact your legal department should you have any questions regarding licensing terms and obligations. APPENDIX B - WIND RIVER LINUX PLATFORM PRODUCT EVALUATION TERMS 1. SCOPE. These terms govern Customer's rights and obligations with respect to Products which are for Customer's limited evaluation purposes ("Evaluation Software"). These terms shall supersede any conflicting provisions of the Agreement with respect to the Evaluation Software. 2. LICENSE GRANTS. The following license terms and conditions set forth in this Section 2 (and no other terms) will solely apply to Customer's use of Evaluation Software: 2.1. Evaluation License. Subject to Customer's compliance with the terms and conditions of the Agreement, Appendix A and this Appendix B, Wind River hereby grants to Customer a restricted, personal, non-transferable, non-exclusive, internal-use license, for a thirty (30) day period from the Effective Date ("Evaluation Term") to: (i) use the Tools solely at the Development Location agreed to in writing by the parties and solely for the purpose of internally evaluating the Products and not for any other purpose (including without limitation, commercial or production purposes); and (ii) reproduce the Tools for archive purposes, consistent with Customer's standard archive procedures. 2.2. Feedback. Customer may disclose any results of any evaluation of the Products including, without limitation, Customer's opinions, observations, comments, criticisms and suggested improvements, whether in written or oral form (herein, "Feedback") from its evaluations only to Wind River. Should Customer provide Wind River with Feedback, Wind River will have the right to use such Feedback and related information in any manner it deems appropriate. 2.3. LIMITED OBLIGATIONS. CUSTOMER ACKNOWLEDGES THAT WIND RIVER HAS NO OBLIGATION TO PROVIDE SUPPORT OR MAINTENANCE FOR THE EVALUATION SOFTWARE. IN NO EVENT WILL WIND RIVER'S AGGREGATE CUMULATIVE LIABILITY FOR ANY CLAIMS ARISING OUT OF OR RELATED TO EVALUATION OF THE PRODUCTS EXCEED $50.00. --- Linux Click Wrap Rev Feb 2020 ================================================ FILE: LICENSE ================================================ GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Lesser General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. ================================================ FILE: README.md ================================================ wrlinux setup ============= With this tool you can either create a new distribution builder platform project, or create a mirror that other projects can be based on. In addition, you can use the tool to obtain information about available project configuration options. The tool uses a layer index (such as layers.openembedded.org), as specified in the bin/settings.py file, to determine what layers are required to construct a distribution builder project. The tool relies on 'repo' from the Android 'git-repo' project. It produces a repo style 'default.xml' file, and then calls repo to download all of the layer components necessary. The tool also configures various sample files that are used by oe-init-build-env to construct your build directory. Documentation ------------- For documentation, see the Wind River Knowledge Library: https://knowledge.windriver.com Select "Products > Operating Systems > Linux > Linux LTS" to view all documentation categories. It is suggested that you start with: _The Wind River Linux Platform Development Quick Start_. When following the quick start and other examples, use: branchName: WRLINUX_10_21_BASE repoURL: git:///git/wrlinux-x Workflows --------- Basic setup/usage workflow: The setup program is expected to have been cloned inside of a project directory, such as: $ mkdir my-project $ cd my-project $ git clone --branch wrlinux-x Once cloned, simply run the setup.sh (./wrlinux-x/setup.sh) to get a list of options. The setup program will construct a new git repository in the current working directory. This repository is used to manage the output of the setup program. You may re-run the setup program at any time to update/change the project settings. However, your build directory will not be touched. You will have to resync it with the updated project. (Specifically bblayers.conf and local.conf to the config/*.sample versions.) To update your project, re-run the setup program with the same arguments. Note the original arguments are available in the generated README file. Mirror workflow: $ mkdir my-mirror $ cd my-mirror $ git clone --branch wrlinux-x $ ./wrlinux-x/setup.sh --all-layers --mirror The above command will mirror all layers, including download layers into the current location, for the branch specified. To update the mirror, re-run the setup.sh command with the same arguments. To add additional branches to your mirror, change the branch in the wrlinux-x, and re-run the setup.sh with the same arguments. A user can reference this mirror by doing: $ mkdir my-project $ cd my-project $ git clone --branch /wrlinux-x and then run the wrlinux-x/setup.sh program as described above. License ------- Copyright (C) 2016-2017 Wind River Systems, Inc. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License version 2 as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ================================================ FILE: bin/.pylintrc ================================================ [MASTER] # Specify a configuration file. #rcfile= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= # Add files or directories to the blacklist. They should be base names, not # paths. ignore=CVS # Add files or directories matching the regex patterns to the blacklist. The # regex matches against base names, not paths. ignore-patterns= # Pickle collected data for later comparisons. persistent=yes # List of plugins (as comma separated values of python modules names) to load, # usually to register additional checkers. load-plugins= # Use multiple processes to speed up Pylint. jobs=2 # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code extension-pkg-whitelist= # Allow optimization of some AST trees. This will activate a peephole AST # optimizer, which will apply various small optimizations. For instance, it can # be used to obtain the result of joining multiple strings with the addition # operator. Joining a lot of strings can lead to a maximum recursion error in # Pylint and this flag can prevent that. It has one side effect, the resulting # AST will be different than the one from reality. This option is deprecated # and it will be removed in Pylint 2.0. optimize-ast=no [MESSAGES CONTROL] # Only show warnings with the listed confidence levels. Leave empty to show # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED confidence= # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). See also the "--disable" option for examples. #enable= # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this # option multiple times (only on the command line, not in the configuration # file where it should appear only once).You can also use "--disable=all" to # disable everything first and then reenable specific checks. For example, if # you want to run only the similarities checker, you can use "--disable=all # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" disable=long-builtin,coerce-builtin,apply-builtin,raising-string,parameter-unpacking,reduce-builtin,old-raise-syntax,unpacking-in-except,nonzero-method,dict-iter-method,getslice-method,cmp-builtin,round-builtin,range-builtin-not-iterating,basestring-builtin,standarderror-builtin,import-star-module-level,coerce-method,old-octal-literal,file-builtin,dict-view-method,old-ne-operator,indexing-exception,no-absolute-import,execfile-builtin,oct-method,zip-builtin-not-iterating,buffer-builtin,delslice-method,raw_input-builtin,metaclass-assignment,xrange-builtin,cmp-method,intern-builtin,print-statement,useless-suppression,filter-builtin-not-iterating,unicode-builtin,using-cmp-argument,map-builtin-not-iterating,backtick,input-builtin,old-division,setslice-method,next-method-called,long-suffix,suppressed-message,hex-method,unichr-builtin,reload-builtin,bad-whitespace,missing-docstring,logging-not-lazy,line-too-long,invalid-name,too-few-public-methods,trailing-newlines,bad-continuation [REPORTS] # Set the output format. Available formats are text, parseable, colorized, msvs # (visual studio) and html. You can also give a reporter class, eg # mypackage.mymodule.MyReporterClass. output-format=text # Put messages in a separate file for each module / package specified on the # command line instead of printing them on stdout. Reports (if any) will be # written in a file name "pylint_global.[txt|html]". This option is deprecated # and it will be removed in Pylint 2.0. files-output=no # Tells whether to display a full report or only the messages reports=no # Python expression which should return a note less than 10 (10 is the highest # note). You have access to the variables errors warning, statement which # respectively contain the number of errors / warnings messages and the total # number of statements analyzed. This is used by the global evaluation report # (RP0004). evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) # Template used to display messages. This is a python new-style format string # used to format the message information. See doc for all details #msg-template= [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. notes=FIXME,XXX,TODO [SPELLING] # Spelling dictionary name. Available dictionaries: none. To make it working # install python-enchant package. spelling-dict= # List of comma separated words that should not be checked. spelling-ignore-words= # A path to a file that contains private dictionary; one word per line. spelling-private-dict-file= # Tells whether to store unknown words to indicated private dictionary in # --spelling-private-dict-file option instead of raising a message. spelling-store-unknown-words=no [LOGGING] # Logging modules to check that the string format arguments are in logging # function parameter format #logging-modules=logging [SIMILARITIES] # Minimum lines number of a similarity. min-similarity-lines=4 # Ignore comments when computing similarities. ignore-comments=yes # Ignore docstrings when computing similarities. ignore-docstrings=yes # Ignore imports when computing similarities. ignore-imports=no [TYPECHECK] # Tells whether missing members accessed in mixin class should be ignored. A # mixin class is detected if its name ends with "mixin" (case insensitive). ignore-mixin-members=yes # List of module names for which member attributes should not be checked # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis. It # supports qualified module names, as well as Unix pattern matching. ignored-modules= # List of class names for which member attributes should not be checked (useful # for classes with dynamically set attributes). This supports the use of # qualified names. ignored-classes=optparse.Values,thread._local,_thread._local # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. generated-members= # List of decorators that produce context managers, such as # contextlib.contextmanager. Add to this list to register other decorators that # produce valid context managers. contextmanager-decorators=contextlib.contextmanager [VARIABLES] # Tells whether we should check for unused import in __init__ files. init-import=no # A regular expression matching the name of dummy variables (i.e. expectedly # not used). dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy # List of additional names supposed to be defined in builtins. Remember that # you should avoid to define new builtins when possible. additional-builtins= # List of strings which can identify a callback function by name. A callback # name must start or end with one of those strings. callbacks=cb_,_cb # List of qualified module names which can have objects that can redefine # builtins. redefining-builtins-modules=six.moves,future.builtins [FORMAT] # Maximum number of characters on a single line. max-line-length=100 # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ # Allow the body of an if to be on the same line as the test if there is no # else. single-line-if-stmt=no # List of optional constructs for which whitespace checking is disabled. `dict- # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. # `trailing-comma` allows a space between comma and closing bracket: (a, ). # `empty-line` allows space-only lines. no-space-check=trailing-comma,dict-separator # Maximum number of lines in a module max-module-lines=1000 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). indent-string=' ' # Number of spaces of indent required inside a hanging or continued line. indent-after-paren=4 # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. expected-line-ending-format= [BASIC] # Good variable names which should always be accepted, separated by a comma good-names=i,j,k,ex,Run,_ # Bad variable names which should always be refused, separated by a comma bad-names=foo,bar,baz,toto,tutu,tata # Colon-delimited sets of names that determine each other's naming style when # the name regexes allow several styles. name-group= # Include a hint for the correct naming format with invalid-name include-naming-hint=no # List of decorators that produce properties, such as abc.abstractproperty. Add # to this list to register other decorators that produce valid properties. property-classes=abc.abstractproperty # Regular expression matching correct constant names const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ # Naming hint for constant names const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ # Regular expression matching correct class attribute names class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ # Naming hint for class attribute names class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ # Regular expression matching correct inline iteration names inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ # Naming hint for inline iteration names inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ # Regular expression matching correct argument names argument-rgx=[a-z_][a-z0-9_]{2,30}$ # Naming hint for argument names argument-name-hint=[a-z_][a-z0-9_]{2,30}$ # Regular expression matching correct variable names variable-rgx=[a-z_][a-z0-9_]{2,30}$ # Naming hint for variable names variable-name-hint=[a-z_][a-z0-9_]{2,30}$ # Regular expression matching correct method names method-rgx=[a-z_][a-z0-9_]{2,30}$ # Naming hint for method names method-name-hint=[a-z_][a-z0-9_]{2,30}$ # Regular expression matching correct module names module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ # Naming hint for module names module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ # Regular expression matching correct function names function-rgx=[a-z_][a-z0-9_]{2,30}$ # Naming hint for function names function-name-hint=[a-z_][a-z0-9_]{2,30}$ # Regular expression matching correct class names class-rgx=[A-Z_][a-zA-Z0-9]+$ # Naming hint for class names class-name-hint=[A-Z_][a-zA-Z0-9]+$ # Regular expression matching correct attribute names attr-rgx=[a-z_][a-z0-9_]{2,30}$ # Naming hint for attribute names attr-name-hint=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match function or class names that do # not require a docstring. no-docstring-rgx=^_ # Minimum line length for functions/classes that require docstrings, shorter # ones are exempt. docstring-min-length=-1 [ELIF] # Maximum number of nested blocks for function / method body max-nested-blocks=5 [DESIGN] # Maximum number of arguments for function / method max-args=10 # Argument names that match this expression will be ignored. Default to name # with leading underscore ignored-argument-names=_.* # Maximum number of locals for function / method body max-locals=50 # Maximum number of return / yield for function / method body max-returns=6 # Maximum number of branch for function / method body max-branches=30 # Maximum number of statements in function / method body max-statements=50 # Maximum number of parents for a class (see R0901). max-parents=7 # Maximum number of attributes for a class (see R0902). max-attributes=7 # Minimum number of public methods for a class (see R0903). min-public-methods=2 # Maximum number of public methods for a class (see R0904). max-public-methods=20 # Maximum number of boolean expressions in a if statement max-bool-expr=5 [CLASSES] # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__,__new__,setUp # List of valid names for the first argument in a class method. valid-classmethod-first-arg=cls # List of valid names for the first argument in a metaclass class method. valid-metaclass-classmethod-first-arg=mcs # List of member names, which should be excluded from the protected access # warning. exclude-protected=_asdict,_fields,_replace,_source,_make [IMPORTS] # Deprecated modules which should not be used, separated by a comma deprecated-modules=optparse # Create a graph of every (i.e. internal and external) dependencies in the # given file (report RP0402 must not be disabled) import-graph= # Create a graph of external dependencies in the given file (report RP0402 must # not be disabled) ext-import-graph= # Create a graph of internal dependencies in the given file (report RP0402 must # not be disabled) int-import-graph= # Force import order to recognize a module as part of the standard # compatibility libraries. known-standard-library= # Force import order to recognize a module as part of a third party library. known-third-party=enchant # Analyse import fallback blocks. This can be used to support both Python 2 and # 3 compatible code, which means that the block might have code that exists # only in one or another interpreter, leading to false positives when analysed. analyse-fallback-blocks=no [EXCEPTIONS] # Exceptions that will emit a warning when being caught. Defaults to # "Exception" overgeneral-exceptions=Exception ================================================ FILE: bin/LICENSE.texttable ================================================ The MIT License (MIT) Copyright (c) 2018 Gerome Fournier Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: bin/Makefile ================================================ SHELL = /bin/bash #requires bash VENV = $(PWD)/.venv/ DEPS = $(wildcard *.py) TMP_PIP = $(VENV)/get-pip.py PYTHON3 := $(shell command -v python3 2> /dev/null) WGET := $(shell command -v wget 2> /dev/null) .PHONY: setup lint clean test help .DEFAULT_GOAL := build help: @echo "Make options for oe-chef development" @echo @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-10s\033[0m %s\n", $$1, $$2}' $(VENV): ifndef PYTHON3 $(error "python3 is not installed.") endif ifndef WGET $(error "wget is not installed.") endif test -d $(VENV) || python3 -m venv --without-pip $(VENV); \ touch $(VENV); \ . $(VENV)/bin/activate; \ wget -O $(TMP_PIP) https://bootstrap.pypa.io/get-pip.py; \ python3 $(TMP_PIP); \ pip3 install pylint nose flake8; setup: $(VENV) ## Install all python dependencies in virtualenv clean: ## Delete virtualenv and all build directories rm -rf $(VENV) test: setup ## Run tests . $(VENV)/bin/activate; python3 test/test_oe_chef.py lint: setup ## Run pylint . $(VENV)/bin/activate; pylint $(DEPS) ================================================ FILE: bin/argparse_setup.py ================================================ # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Note this class MUST run in both python2 and python3 import argparse import sys import os from urllib.parse import urlparse import logger_setup logger = logger_setup.setup_logging() class Argparse_Setup: def __init__(self, setup, parser=None): if not parser: parser = argparse.ArgumentParser(description='setup.py: Application to fetch & setup a distribution project.') self.layer_select = False self.parser = parser self.setup = setup # Each group's layername must have a group keyword, for example: # - meta-examplekey # - meta-examplekey-layer1 # - meta-examplekey-layer2 # The examplekey is the keyword for these layers. It will be used to # check against the layername from layerindex (not layer's directory # name). Any layername contains the keyword is treated as an extra # group layer, and they should only be enabled when # --use-layer-groups=keyword or --layers=layer is specified. # Add the new keyword here when there is a new extra group. self.extra_group_keys=['ccm'] def evaluate_args(self, args): self.add_options() parsed_args = self.parser.parse_args(args) self.handle_setup_args(parsed_args, args) def handle_setup_args(self, parsed_args, args): # Parse setup options if (parsed_args.verbose): if self.setup: self.setup.set_debug() del parsed_args.verbose if (parsed_args.base_url): if self.setup: self.setup.set_base_url(parsed_args.base_url) del parsed_args.base_url if (parsed_args.base_branch): if self.setup: self.setup.set_base_branch(parsed_args.base_branch) del parsed_args.base_branch # Parse repo option if (parsed_args.repo_verbose): if self.setup: self.setup.set_repo_verbose(parsed_args.repo_verbose) del parsed_args.repo_verbose if (parsed_args.repo_jobs): if self.setup: self.setup.set_jobs(parsed_args.repo_jobs) del parsed_args.repo_jobs if (parsed_args.repo_depth): if self.setup: self.setup.set_depth(parsed_args.repo_depth) del parsed_args.repo_depth if (parsed_args.repo_force_sync): if self.setup: self.setup.set_force_sync(parsed_args.repo_force_sync) del parsed_args.repo_force_sync if (parsed_args.repo_url): if self.setup: self.setup.set_repo_url(parsed_args.repo_url) del parsed_args.repo_url if (parsed_args.repo_branch): if self.setup: self.setup.set_repo_rev(parsed_args.repo_branch) del parsed_args.repo_branch # Look for list options if parsed_args.list_distros: if self.setup: self.setup.list_distros = parsed_args.list_distros if parsed_args.list_machines: if self.setup: self.setup.list_machines = parsed_args.list_machines if parsed_args.list_layers: if self.setup: self.setup.list_layers = True if parsed_args.list_recipes: if self.setup: self.setup.list_recipes = True if parsed_args.repo_no_fetch: if self.setup: self.setup.repo_no_fetch = True del parsed_args.repo_no_fetch if (parsed_args.list_distros or parsed_args.list_machines or parsed_args.list_layers or parsed_args.list_recipes): return # Parse layer selection options if parsed_args.distros: self.layer_select = True if self.setup: self.setup.distros = [] for d in parsed_args.distros: for distro in d.split(','): self.setup.distros.append(distro) if parsed_args.machines: self.layer_select = True if self.setup: self.setup.machines = [] for m in parsed_args.machines: for machine in m.split(','): self.setup.machines.append(machine) if parsed_args.layers: self.layer_select = True if self.setup: self.setup.layers = [] for l in parsed_args.layers: for layer in l.split(','): if '://' in layer: # check if layer has a valid url scheme url = urlparse(layer) if url.scheme and url.scheme != 'file': remote_layer = {} remote_layer['branch'] = 'master' remote = layer.split('+') remote_layer['url'] = remote[0] remote_layer['path'] = 'layers/' + remote[0].split('/')[-1] for arg in remote[1:]: if arg.startswith('branch='): remote_layer['branch'] = arg[7:] self.setup.remote_layers.append(remote_layer) else: logger.warning("Skipping invalid remote url: %s" % (layer)) elif '/' in layer or os.path.exists(layer): layer = os.path.realpath(layer) if not os.path.isdir(layer): logger.warning("Skipping invalid local layer %s" % (layer)) else: self.setup.local_layers.append(layer) else: self.setup.layers.append(layer) if parsed_args.recipes: self.layer_select = True if self.setup: self.setup.recipes = [] for r in parsed_args.recipes: for recipe in r.split(','): self.setup.recipes.append(recipe) if parsed_args.all_layers: self.layer_select = True if self.setup: self.setup.all_layers = parsed_args.all_layers if parsed_args.no_recommend: self.layer_select = True if self.setup: self.setup.no_recommend = parsed_args.no_recommend if (parsed_args.mirror): if self.layer_select is not True: print('ERROR: The --mirror option requires at least one Layer Section argument, see --help.') sys.exit(1) if self.setup: self.setup.mirror = parsed_args.mirror if parsed_args.use_layer_groups: if self.setup: self.setup.use_layer_groups = parsed_args.use_layer_groups if self.layer_select is not True: print('ERROR: You must include at least one Layer Selection argument, see --help.') sys.exit(1) def add_setup_options(self): # Setup options self.parser.add_argument('-v', '--verbose', help='Set the verbosity to debug', action="store_true") self.base_args = self.parser.add_argument_group('Base Settings') setup_base_url = "" if self.setup and self.setup.base_url: setup_base_url = '(default %s)' % (self.setup.base_url) self.base_args.add_argument('--base-url', metavar="URL", help='URL to fetch from %s' % (setup_base_url)) setup_base_branch = "" if self.setup and self.setup.base_branch: setup_base_branch = '(default %s)' % (self.setup.base_branch) self.base_args.add_argument('--base-branch', metavar="BRANCH", help='Base branch identifier %s' % (setup_base_branch)) self.parser.add_argument('--mirror', help='Do not construct a project, instead construct a mirror of the repositories that would have been used to construct a project (requires a Layer Selection argument)', action='store_true') def add_repo_options(self): self.repo_args = self.parser.add_argument_group('repo Settings') # Repo options setup_jobs = "" self.repo_args.add_argument('-rv', '--repo-verbose', action='store_true', help='Disables use of --quiet with repo commands') if self.setup and self.setup.jobs: setup_jobs = '(default %s)' % (self.setup.jobs) self.repo_args.add_argument('-rj', '--repo-jobs', metavar='JOBS', help='Sets repo project to fetch simultaneously %s' % (setup_jobs)) self.repo_args.add_argument('--repo-depth', metavar='DEPTH', help='Sets repo --depth; see repo init --help (note: if set, a value of >= 2 is required)') self.repo_args.add_argument('--repo-force-sync', action='store_true', help='Sets repo --force-sync; see repo sync --help') self.repo_args.add_argument('--repo-no-fetch', help='Do all the setup but do not call repo sync', action="store_true") repo_url = "" if 'REPO_URL' in os.environ: repo_url = '(default %s)' % os.environ['REPO_URL'] repo_rev = "" if 'REPO_REV' in os.environ: repo_rev = '(default %s)' % os.environ['REPO_REV'] self.repo_args.add_argument('--repo-url', metavar="URL", help='Url for git-repo %s' % (repo_url)) self.repo_args.add_argument('--repo-branch', metavar="REV", help='Url for git-repo %s' % (repo_rev)) def add_list_options(self): self.list_args = self.parser.add_argument_group('Layer Listings') # List options self.list_args.add_argument('--list-distros', metavar='all', nargs='?', const='default', help='List available distro values') self.list_args.add_argument('--list-machines', metavar='all', nargs='?', const='default', help='List available machine values') self.list_args.add_argument('--list-layers', action='store_true', help='List all available layers') self.list_args.add_argument('--list-recipes', action='store_true', help='List all available recipes') def add_layer_options(self): self.layer_args = self.parser.add_argument_group('Layer Selection') # Layer selection and local.conf setup setup_distro = "" setup_distro_str = "" if self.setup and self.setup.distros: setup_distro = self.setup.distros[0] setup_distro_str = '(default %s)' % setup_distro self.layer_args.add_argument('--distros', metavar='DISTRO', help='Select layer(s) based on required distribution and set the default DISTRO= value %s' % setup_distro_str, nargs="+") setup_machine = "" setup_machine_str = "" if self.setup and self.setup.machines: setup_machine = self.setup.machines[0] setup_machine_str = '(default %s)' % setup_machine self.layer_args.add_argument('--machines', metavar='MACHINE', help='Select layer(s) based on required machine(s) and set the default MACHINE= value %s' % setup_machine_str, nargs='+') self.layer_args.add_argument('--layers', metavar='LAYER', help='Select layer(s) to include in the project and add to the default bblayers.conf. Can accept the name of a layer in the layerindex, a path to a layer on local storage or a remote url that will be cloned by git-repo. (||:///(+branch=)) ', nargs='+') self.layer_args.add_argument('--recipes', metavar='RECIPE', help='Select layers(s) based on recipe(s)', nargs='+') self.layer_args.add_argument('--all-layers', help='Select all available layers', action='store_true') self.layer_args.add_argument('--no-recommend', help='Disable recommended layers during layer resolution', action='store_true') self.layer_args.add_argument('--use-layer-groups', metavar='EXTRA_GROUP', help="Specify extra layer groups to use. Make sure you have permissions to access these groups before you use it.", action='store', nargs='+', choices=self.extra_group_keys) def add_other_options(self): pass def add_options(self): self.add_setup_options() self.add_repo_options() self.add_list_options() self.add_layer_options() self.add_other_options() ================================================ FILE: bin/argparse_wrl.py ================================================ # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Note this class must run in both python2 and python3 import argparse import logging import string import sys from argparse_setup import Argparse_Setup class Argparse_Wrl(Argparse_Setup): def __init__(self, setup, parser=None): self.parser = argparse.ArgumentParser(description='setup.py: Application to fetch & setup a WindRiver Linux project.') Argparse_Setup.__init__(self, setup, parser) def handle_setup_args(self, parsed_args, args): if (parsed_args.buildtools_branch): # ignore (handled by setup.sh) del parsed_args.buildtools_branch if (parsed_args.buildtools_type): # ignore (handled by setup.sh) del parsed_args.buildtools_type if (parsed_args.user): # ignore (handled by setup.sh) del parsed_args.user if (parsed_args.password): # ignore (handled by setup.sh) del parsed_args.password if parsed_args.no_anspass: # ignore (handled by setup.sh) del parsed_args.no_anspass if (parsed_args.accept_eula): # ignore (handled by setup.sh) del parsed_args.accept_eula if parsed_args.list_templates: if self.setup: self.setup.list_wrtemplates = parsed_args.list_templates if (parsed_args.list_templates): return if parsed_args.templates: self.layer_select = True if self.setup: self.setup.wrtemplates = [] for t in parsed_args.templates: for wrtemplate in t.split(','): self.setup.wrtemplates.append(wrtemplate) if parsed_args.dl_layers: self.layer_select = True if self.setup: self.setup.dl_layers = parsed_args.dl_layers Argparse_Setup.handle_setup_args(self, parsed_args, args) def add_setup_options(self): Argparse_Setup.add_setup_options(self) setup_buildtools_branch = "" if self.setup and self.setup.buildtools_branch: setup_buildtools_branch = '(default %s)' % (self.setup.buildtools_branch) self.base_args.add_argument('--buildtools-branch', metavar="BRANCH", help='Buildtools branch %s' % (setup_buildtools_branch)) self.base_args.add_argument('--buildtools-type', metavar="TYPE", help="Buildtools type: basic or extended (default basic), install extended buildtools with 'extended'") self.base_args.add_argument('--user', help='Specify default user for download') self.base_args.add_argument('--password', help='Specify default password for download') self.base_args.add_argument('--accept-eula', metavar="ACCEPT", help='Accept End User License Agreement (yes|no)') self.base_args.add_argument('--no-anspass', action='store_true', help='Do not use anspass to save the user name and password') def add_list_options(self): Argparse_Setup.add_list_options(self) self.list_args.add_argument('--list-templates', metavar='all', nargs='?', const='default', help='List available templates') def add_layer_options(self): Argparse_Setup.add_layer_options(self) self.layer_args.add_argument('--templates', metavar='TEMPLATE', help='Select layers(s) based on template(s) and add them by default to the builds', nargs='+') self.layer_args.add_argument('--dl-layers', help='Enable download layers; these layers include predownloaded items', action='store_true') ================================================ FILE: bin/branch_mirror.py ================================================ #!/usr/bin/env python3 # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # This program will identify the layers, and their current branches, # that need to be branched as part of the release process. # # The program will branch them and provide a list that can be used # to push the items. # # The program will also update the mirror index entries with the # branch names. import os import sys import subprocess import xml.etree.ElementTree as ET import utils_setup from layer_index import Layer_Index import logger_setup import settings completed = [] def usage(): print("usage: %s [--force]" % sys.argv[0]) sys.exit(1) if len(sys.argv) < 2: usage() logger = logger_setup.setup_logging() dest_branch = sys.argv[1] force = False if len(sys.argv) > 2: if sys.argv[2] != '--force': usage() force = True mirror_path = 'mirror-index' if not os.path.exists(mirror_path): logger.critical('No mirror-index found. %s' % mirror_path) sys.exit(1) cmd = ['git', 'rev-parse', '--abbrev-ref', 'HEAD'] ret = subprocess.Popen(cmd, cwd=mirror_path, close_fds=True, stdout=subprocess.PIPE) branch = "" output = "" while True: output = ret.stdout.readline() if not output and ret.poll() is not None: break branch += output.decode('utf-8') ret.wait() branch = branch.strip() if branch == dest_branch: logger.warning('Branch is already configured!') work_list = [] def git_branch(_dst, _orig_branch, _branch): logger.info('Branching %s: %s -> %s' % (_dst, _orig_branch, _branch)) # Break this into two commands, so it will work with tags as well... _cmd = [ 'git', 'fetch', '.', '%s' % (_orig_branch) ] utils_setup.run_cmd(_cmd, cwd=_dst) _cmd = [ 'git', 'branch', _branch, 'FETCH_HEAD' ] if force: _cmd.append('-f') utils_setup.run_cmd(_cmd, cwd=_dst) work_list.append('%s %s' % (_dst, _branch)) # We assume this program is located in the bin directory dst = os.path.dirname(os.path.dirname(sys.argv[0])) # Branch the setup program.... git_branch(_dst=dst, _orig_branch=branch, _branch=dest_branch) completed.append(dst) # Transform and export the mirror index git_branch(mirror_path, branch, dest_branch) completed.append(mirror_path) index = Layer_Index(indexcfg=settings.INDEXES, base_branch=branch, replace=settings.REPLACE, mirror=mirror_path) cmd = ['git', 'checkout', dest_branch] utils_setup.run_cmd(cmd, cwd=mirror_path) logger.info('Loading default.xml') tree = ET.parse('default.xml') root = tree.getroot() logger.info('Branching based on default.xml') default_revision = None base_url = None for child in root: if child.tag == 'remote': if 'fetch' in child.attrib: base_url = child.attrib['fetch'] if child.tag == 'default': if 'revision' in child.attrib: default_revision = child.attrib['revision'] if child.tag != 'project': continue src = child.attrib['name'] if not os.path.exists(src): if os.path.exists(src + '.git'): src += '.git' else: logger.warning('Unable to find %s' % src) continue revision = None if not ('bare' in child.attrib and child.attrib['bare'] == 'True'): revision = default_revision if 'revision' in child.attrib: revision = child.attrib['revision'] if revision: git_branch(src, revision, dest_branch) completed.append(src) logger.info('Transforming default.xml') for child in root: if 'revision' in child.attrib: child.attrib['revision'] = dest_branch open('default.xml', 'wt').write(ET.tostring(root, encoding='unicode')) logger.info('Transforming index...') bitbake_branch = branch branchid = None for lindex in index.index: for branches in lindex['branches']: if 'name' in branches and branches['name'] == branch: branches['name'] = dest_branch if 'bitbake_branch' in branches and branches['bitbake_branch'] != '': bitbake_branch = branches['bitbake_branch'] branches['bitbake_branch'] = dest_branch branchid = branches['id'] for layer in lindex['layerItems']: if layer['vcs_url']: for lb in lindex['layerBranches']: if layer['id'] == lb['layer'] and lb['branch'] == branchid: if 'actual_branch' in lb and lb['actual_branch'] != "": lb['actual_branch'] = '' # Remove older entries for (dirpath, dirnames, filenames) in os.walk(mirror_path): if dirpath.endswith('/.git') or '/.git/' in dirpath or dirpath.endswith('/xml') or '/xml/' in dirpath: continue for filename in filenames: if filename.startswith(lindex['CFG']['DESCRIPTION'].translate(str.maketrans('/ ', '__'))): os.remove(os.path.join(dirpath, filename)) index.serialize_index(lindex, os.path.join(mirror_path, lindex['CFG']['DESCRIPTION']), split=True, IncludeCFG=True, mirror=True, base_url=base_url) # git add file. cmd = ['git', 'add', '-A', '.'] utils_setup.run_cmd(cmd, cwd=mirror_path) cmd = ['git', 'diff-index', '--quiet', 'HEAD', '--'] try: utils_setup.run_cmd(cmd, cwd=mirror_path) except: logger.debug('Updating mirror-index') cmd = ['git', 'commit', '-m', 'Branch (%s) and adjust index entries' % (dest_branch)] utils_setup.run_cmd(cmd, cwd=mirror_path) logger.info('Done') logger.plain('Writing branched-layer.list...') output = open('branched-layer.list', 'wt') for item in work_list: output.write("%s\n" % item) output.close() ================================================ FILE: bin/dump_layer_dependencies.py ================================================ #!/usr/bin/env python3 # Copyright (C) 2016-2017 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # This program will dump the dependencies for one or more layers in a # given branch. It can be used to verify that when a branch is created # that the dependencies have been scanned properly. # Adjust the standard WR urls to make comparisons easier. REPLACE = [ ( 'git://git.wrs.com/', '#BASE_URL#' ), ( 'http://git.wrs.com/cgit/', '#BASE_WEB#' ), ] # Note the branch can be hard coded. This is required only when you want # to limit the branch from a restapi-web import. (This does not do anything # on other input formats.) INDEXES = [ { 'DESCRIPTION' : 'Wind River Developer Layer Index', 'TYPE' : 'restapi-web', 'URL' : 'http://layers.wrs.com/layerindex/api/', 'CACHE' : None, #'BRANCH' : 'master-wr', }, ] import os import sys from layer_index import Layer_Index def usage(): print("usage: %s " % sys.argv[0]) sys.exit(1) if len(sys.argv) < 2: usage() base_branch=sys.argv[1] index = Layer_Index(INDEXES, base_branch=base_branch, replace=REPLACE) import unicodedata for lindex in index.index: dep_out = [] branchid = index.getBranchId(lindex, index.getIndexBranch(default=base_branch, lindex=lindex)) if branchid: for lb in lindex['layerBranches']: if lb['branch'] == branchid: for layer in index.find_layer(lindex, layerBranch=lb): name = layer['name'] (required, recommended) = index.getDependencies(lindex, lb) reqnames = [] recnames = [] for req in required: for layer in index.find_layer(lindex, layerBranch=req): reqnames.append(layer['name']) for rec in recommended: for layer in index.find_layer(lindex, layerBranch=rec): recnames.append(layer['name']) dep_out.append((name, ' '.join(sorted(reqnames)), ' '.join(sorted(recnames)))) for (name, reqs, recs) in sorted(dep_out, key=lambda t: t[0]): print('%s %s (%s)' % (name, reqs, recs)) ================================================ FILE: bin/dump_layer_rev.py ================================================ #!/usr/bin/env python3 # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # This program will load the index and for each layer list the layer, # branch and current commit. This can be used to verify that the # index and layers are in sync. import os import sys import settings from layer_index import Layer_Index mirror_path = 'mirror-index' if not os.path.exists(mirror_path): if not os.path.exists(mirror_path + '.git'): print('No %s found. Is this a mirror?' % mirror_path) sys.exit(1) else: mirror_path = mirror_path + '.git' index = Layer_Index(indexcfg=settings.INDEXES, base_branch=None, replace=settings.REPLACE, mirror=mirror_path) for lindex in index.index: for branch in lindex['branches']: basebranch = branch['name'] for litem in lindex['layerItems']: for lbranch in lindex['layerBranches']: if lbranch['layer'] == litem['id']: branch = basebranch if lbranch['actual_branch'] != "": branch = lbranch['actual_branch'] print('%s %s %s %s' % (litem['name'], litem['vcs_url'], branch, lbranch['vcs_last_rev'])) break ================================================ FILE: bin/flatten_mirror.py ================================================ #!/usr/bin/env python3 # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # This program will flatten a mirror and copy the result into a target dir. # # Many git servers require a single flat level of git trees in order to # work properly. Flattening the tree will require both renaming directories # as well as updating the mirror-index to match. # Format of the subset folder file is: # # # is the name in the layer index of that layer # is a directory to generate inside of the # # Note, if this is used, an error will be generated for any known layers # that are not assigned to a folder. # # A special folder name '[SKIP]' is defined when you are intentionally # skipping a layer. import argparse import os import sys import shutil import subprocess import xml.etree.ElementTree as ET import utils_setup from layer_index import Layer_Index import logger_setup import settings def config_args(args): parser = argparse.ArgumentParser(description='flatten_mirror.py: Flatten a mirror.') parser.add_argument('dest', help='Destination Directory') parser.add_argument('--push-not-copy', help='Push non-bare layers, don\'t copy them. This allows the flattened version to only have one branch.', action='store_true') parser.add_argument('--subset-mirror', metavar='FILE', help='Use a file that will allow the system to subset the mirror into specific subdirectories.') parser.add_argument('--strip-git', help='Strip the .git suffix from paths when copying. This is needed when copying to an http server, vs a git server.', action='store_true') parsed_args = parser.parse_args(args) return (parsed_args.dest, parsed_args.push_not_copy, parsed_args.subset_mirror, parsed_args.strip_git) def push_or_copy(_layer, _src, _dst, _branch=None): if subset_folders: if _layer not in subset_folders: logger.critical("Layer %s (%s) not in SUBSET_FOLDERS" % (_layer, _src)) raise elif subset_folders[_layer] == "[SKIP]": return else: dstdir = os.path.dirname(_dst) dstbase = os.path.basename(_dst) _dst = os.path.join(dstdir, subset_folders[_layer], dstbase) if not os.path.exists(_src): _src += '.git' if not os.path.exists(_src): logger.critical("Unable to find %s!" % _src) raise # Make output consistent if not strip_git and not _dst.endswith('.git'): _dst += '.git' if strip_git and _dst.endswith('.git'): _dst = _dst[:-4] if not git_push or not _branch: logger.plain('cp %s -> %s' % (_src, _dst)) shutil.copytree(_src, _dst, symlinks=True, ignore_dangling_symlinks=True) else: logger.plain('push %s -> %s (%s)' % (_src, _dst, _branch)) if os.path.exists(_dst): logger.critical('Destination %s already exists!' % _dst) raise os.makedirs(_dst, exist_ok=True) # New bare repo _cmd = [ 'git', 'init', '--bare' ] utils_setup.run_cmd(_cmd, cwd=_dst) # Push just the one branch _cmd = [ 'git', 'push', os.path.abspath(_dst), _branch ] utils_setup.run_cmd(_cmd, cwd=_src) # Used to subset target mirror path, if necessary # Return the mirror_directory target for a given layer entry. # If the target directory does not exist, we create it (git repository) # # This function will return None if the layer is to be skipped. def get_mirror_dir(_layer, _dst): if subset_folders: if _layer in subset_folders: if subset_folders[_layer] == "[SKIP]": return None dstdir = os.path.dirname(_dst) dstbase = os.path.basename(_dst) _dst = os.path.join(dstdir, subset_folders[_layer], dstbase) else: logger.error("Layer %s not in SUBSET_FOLDERS" % _layer) # Use default directory # If the target directory does not exist, create it if not os.path.exists(_dst): cmd = ['git', 'init', _dst] utils_setup.run_cmd(cmd) if not branch or branch == "": logger.critical('Where did the branch val go!? %s' % branch) raise cmd = ['git', 'checkout', '-b', branch] try: utils_setup.run_cmd(cmd, cwd=_dst) except: # if we failed, then simply try to switch branches cmd = ['git', 'checkout', branch] utils_setup.run_cmd(cmd, cwd=_dst) return _dst # Return the layer specific XML directory. The XML directory # lives inside of the mirror path. If the mirror directory does # not already exist it is created by callign get_mirror_dir # # This function will return None if the layer is to be skipped. def get_xml_dir(_layer, _mirror): _mirror = get_mirror_dir(_layer, _mirror) if _mirror: return os.path.join(_mirror, 'xml') return _mirror # Process and transform the specified XML files. Return a list of # project names referenced by this XML file. # # If the _dest is None, we still process the XML layer, but we never # write out a result. This is necessary to parse the components owned # by the xml file so we know to skip them... def transform_xml(_src, _dest): logger.plain('Processing %s' % _src) if not os.path.exists(_src): logger.warning('Not found %s' % _src) return [] if _dest: os.makedirs(os.path.dirname(_dest), exist_ok=True) with open(_src, 'rt') as fin, open(_dest, 'wt') as fout: return transform_xml_inside(fin, fout) else: with open(_src, 'rt') as fin: return transform_xml_inside(fin, None) def transform_xml_inside(_fin, _fout): result = [] for _line in _fin: modified = False try: _root = ET.fromstring(_line) except: logger.warning('exception on: %s' % _line) if _fout: _fout.write('%s' % _line) continue # Skip linkfiles, don't warn.. we know these are valid if _root.tag == 'linkfile': if _fout: _fout.write('%s' % _line) continue if _root.tag != 'project': logger.warning('Not project: %s' % _line) if _fout: _fout.write('%s' % _line) continue for attrib in _root.attrib: if attrib == 'name': result.append(_root.attrib['name']) if _root.attrib['name'] != _root.attrib['name'].split('/')[-1]: _root.attrib['name'] = _root.attrib['name'].split('/')[-1] modified = True for _child in _root: for attrib in _child.attrib: if attrib == 'name': result.append(_child.attrib['name']) if _child.attrib['name'] != _child.attrib['name'].split('/')[-1]: _child.attrib['name'] = _child.attrib['name'].split('/')[-1] modified = True if _fout: if modified: _fout.write(' %s\n' % (ET.tostring(_root, encoding='unicode'))) else: _fout.write('%s' % _line) return result # Process the mirrors and add any new files to the git repo def update_mirror(_dst_mirror): cmd = ['git', 'add', '-A', '.'] utils_setup.run_cmd(cmd, cwd=_dst_mirror) logger.debug('Updating mirror-index') cmd = ['git', 'commit', '-m', 'Updated index - Flatten Mirror'] try: utils_setup.run_cmd(cmd, cwd=_dst_mirror, log=2) except: # Nothing changed... pass def copy_premirrors_dl(dest): premirrors_dl = "premirrors-dl/downloads/" if os.path.exists(premirrors_dl): logger.plain("Copying %s" % premirrors_dl) else: return cmd = "cp --parent -a".split() if os.stat(premirrors_dl).st_dev == os.stat(dest).st_dev: # Hard link when possible cmd.append("-l") cmd += [premirrors_dl, dest] utils_setup.run_cmd(cmd) # This assumes variables 'dest', 'git_push', 'subset_file' and # 'setup_dir' is globally set. def main(): global subset_folders global branch if subset_file: subset_folders = {} with open(subset_file, 'rt') as f: for line in f: if line.startswith('#'): continue lsplit = line.split() if len(lsplit) == 0: continue if len(lsplit) != 2: logger.critical("Subset Folders, invalid line: %s" % (line)) return 1 subset_folders[lsplit[0]] = lsplit[1] if os.path.exists(dest): logger.critical('Destination directory %s already exists. Please choose a different destination.' % (dest)) return 1 # We have to run this against a mirror, check for a mirror-index mirror_path = 'mirror-index' if not os.path.exists(mirror_path): logger.critical('No %s found. Is this a mirror?' % mirror_path) return 1 dst_base_mirror = os.path.join(dest, mirror_path) # Find the base branch of the mirror-index to set a default cmd = ['git', 'rev-parse', '--abbrev-ref', 'HEAD'] _ret = subprocess.Popen(cmd, cwd=mirror_path, close_fds=True, stdout=subprocess.PIPE) branch = "" output = "" while True: output = _ret.stdout.readline() if not output and _ret.poll() is not None: break branch += output.decode('utf-8') _ret.wait() branch = branch.strip() if not branch or branch == "": logger.critical('Unable to determine base branch.') return 1 # Create the destination os.makedirs(dest, exist_ok=False) #### Load the index and create a list of things we need to parse # this is the list of things that MAY be in the default.xml file, we need # to have a list to later process that file and exclude things we've already # done. processed_list = [] ### Create the target mirror-index... # Transform and export the mirror index logger.plain('Transforming index...') index = Layer_Index(indexcfg=settings.INDEXES, base_branch=branch, replace=settings.REPLACE, mirror=mirror_path) branchid = -1 base_branch = branch bitbake_branch = branch for lindex in index.index: if 'CFG' in lindex: base_branch = lindex['CFG']['BRANCH'] bitbake_branch = branch for b in lindex['branches']: if 'name' in b and b['name'] == base_branch: branchid = b['id'] if 'bitbake_branch' in b and b['bitbake_branch'] != "": bitbake_branch = b['bitbake_branch'] break logger.info('Discovered base_branch: %s (%s)' % (base_branch, branchid)) logger.info('Discovered bitbake_branch: %s' % bitbake_branch) for layer in lindex['layerItems']: logger.info('Processing layer %s...' % layer['name']) # Identify, manipulate and copy the layer... if 'vcs_url' in layer: full_url = layer['vcs_url'].replace('#BASE_URL#/', '') base_url = layer['vcs_url'].split('/')[-1] layer['vcs_url'] = '#BASE_URL#' + '/' + base_url layer['vcs_web_url'] = '' layer['vcs_web_tree_base_url'] = '' layer['vcs_web_file_base_url'] = '' layer['mailing_list_url'] = '' # Find actual_branch if one is there revision = base_branch for lb in lindex['layerBranches']: if lb['branch'] == branchid and lb['layer'] == layer['id']: if lb['actual_branch'] != "": revision = lb['actual_branch'] src = full_url dst = os.path.join(dest, os.path.basename(src)) if src not in processed_list: push_or_copy(layer['name'], src, dst, revision) processed_list.append(src) xml_dir = get_xml_dir(layer['name'], dst_base_mirror) def xml_dest_dir(_xml_dir, _name): if not _xml_dir: return None return os.path.join(_xml_dir, _name) src = os.path.join(mirror_path, 'xml', '%s.inc' % layer['name']) if os.path.exists(src): xml_dst = xml_dest_dir(xml_dir, '%s.inc' % layer['name']) for name in transform_xml(src, xml_dst): dst = os.path.join(dest, os.path.basename(name)) if name not in processed_list: push_or_copy(layer['name'], name, dst) processed_list.append(name) src = os.path.join(mirror_path, 'xml', '%s.xml' % layer['name']) if os.path.exists(src): xml_dst = xml_dest_dir(xml_dir, '%s.xml' % layer['name']) for name in transform_xml(src, xml_dst): dst = os.path.join(dest, os.path.basename(name)) if name not in processed_list: push_or_copy(layer['name'], name, dst) processed_list.append(name) # OpenEmbedded-Core is a bit unique. There are a few items # that need to be grouped by this subset entry, these are # items NOT included in the index or default.xml # # wrlinux-x # git-repo # bitbake if layer['name'] == 'openembedded-core': # wrlinux-x (or whatever it's called) convert to bare using .git src = os.path.join(setup_dir, '.git') dst = os.path.join(dest, os.path.basename(setup_dir)) if src not in processed_list: push_or_copy(layer['name'], src, dst, branch) processed_list.append(src) # git-repo src = os.path.join(os.path.dirname(full_url), 'git-repo') dst = os.path.join(dest, os.path.basename(src)) if src not in processed_list: push_or_copy(layer['name'], src, dst) processed_list.append(src) # bitbake src = os.path.join(os.path.dirname(full_url), 'bitbake') dst = os.path.join(dest, os.path.basename(src)) if src not in processed_list: push_or_copy(layer['name'], src, dst, bitbake_branch) processed_list.append(src) src = os.path.join(mirror_path, 'xml', 'bitbake.inc') if os.path.exists(src): xml_dst = xml_dest_dir(xml_dir, 'bitbake.inc') for name in transform_xml(src, xml_dst): dst = os.path.join(dest, os.path.basename(name)) if name not in processed_list: push_or_copy(layer['name'], name, dst) processed_list.append(name) src = os.path.join(mirror_path, 'xml', 'bitbake.xml') if os.path.exists(src): xml_dst = xml_dest_dir(xml_dir, 'bitbake.xml') for name in transform_xml(src, xml_dst): dst = os.path.join(dest, os.path.basename(name)) if name not in processed_list: push_or_copy(layer['name'], name, dst) processed_list.append(name) # dst_base_mirror may not exist if we're subsetting... os.makedirs(dst_base_mirror, exist_ok=True) index.serialize_index(lindex, os.path.join(dst_base_mirror, lindex['CFG']['DESCRIPTION']), split=True, IncludeCFG=True, mirror=True, base_url=base_url) # Since serialize can't subset, we do it manually... # if the rules change in layer_index.py, adjust them here.. if subset_folders: base_branch = branch if 'CFG' in lindex: base_branch = lindex['CFG']['BRANCH'] for layer in lindex['layerItems']: json = "%s__%s__%s.json" % (lindex['CFG']['DESCRIPTION'], base_branch, layer['name']) json = json.translate(str.maketrans('/ ', '__')) src = os.path.join(dst_base_mirror, json) mirror_dir = get_mirror_dir(layer['name'], dst_base_mirror) if not mirror_dir: # Skipped item, remove it and continue logger.plain('rm %s' % src) os.remove(src) continue dst = os.path.join(mirror_dir, json) os.makedirs(os.path.dirname(dst), exist_ok=True) logger.plain('mv %s -> %s' % (src, dst)) os.rename(src, dst) # Directory is expected to be empty, remove it. os.rmdir(dst_base_mirror) #### Now process anythign else we've not yet processed logger.info('Processing left-overs...') # Now process the default.xml, and process anything not previous processed... tree = ET.parse('default.xml') root = tree.getroot() default_revision = None base_url = None for child in root: if child.tag == 'remote': if 'fetch' in child.attrib: base_url = child.attrib['fetch'] if child.tag == 'default': if 'revision' in child.attrib: default_revision = child.attrib['revision'] if child.tag != 'project': continue src = child.attrib['name'] if src in processed_list or src + '.git' in processed_list: continue dst = os.path.join(dest, os.path.basename(src)) revision = None if not ('bare' in child.attrib and child.attrib['bare'] == 'True'): revision = default_revision if 'revision' in child.attrib: revision = child.attrib['revision'] push_or_copy(os.path.basename(src), src, dst, revision) #### Update the mirror-index repositories (git add/git commit) logger.plain('Updating mirror-index repositories...') # git add file. if subset_folders: index_list = [] for layer in subset_folders: if subset_folders[layer] == "[SKIP]": continue dst_mirror = get_mirror_dir(layer, dst_base_mirror) if dst_mirror not in index_list: index_list.append(dst_mirror) for dst_mirror in index_list: update_mirror(dst_mirror) else: update_mirror(dst_base_mirror) copy_premirrors_dl(dest) logger.plain('Done') return 0 # Define globals if __name__ == '__main__': logger = logger_setup.setup_logging() dest, git_push, subset_file, strip_git = config_args(sys.argv[1:]) subset_folders = None branch = None setup_dir = os.path.dirname(os.path.dirname(sys.argv[0])) ret = main() sys.exit(ret) ================================================ FILE: bin/layer_index.py ================================================ # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import json import sys import os import difflib from collections import OrderedDict import logger_setup import utils_setup import texttable as tt # type, url/path, description, cache # type: restapi-web - REST API from a LayerIndex-web # restapi-files - REST API, but only from files # export - Exported DB from a LayerIndex-web -- reads file(s) logger = logger_setup.setup_logging() class Layer_Index(): # Index in REST-API format... This is used by external items. index = [] def __init__(self, indexcfg=[], base_branch=None, replace=[], mirror=None): self.index = [] # Do we have local mirror entries to load? m_index = {} if mirror: for (dirpath, dirnames, filenames) in os.walk(mirror): if dirpath.endswith('/.git') or '/.git/' in dirpath or dirpath.endswith('/xml') or '/xml/' in dirpath: continue for filename in filenames: # Serialize function, ALWAYS writes out w/ .json extension if not filename.endswith('.json'): continue pindex = self.load_serialized_index(os.path.join(dirpath, filename), name='Mirrored Index') # A mirror can be made up of multiple indexes, so we need to identify which one they belong to if pindex and pindex['CFG']['DESCRIPTION'] in m_index: lindex = m_index[pindex['CFG']['DESCRIPTION']] for entry in pindex: if 'apilinks' == entry: continue if 'CFG' == entry: # Conflicts don't matter here, just accept it lindex[entry] = pindex[entry] continue if entry not in lindex: lindex[entry] = [] try: lindex[entry] = self.__add_cmp_lists(pindex[entry], lindex[entry]) except TypeError as error: raise TypeError('Merge failed of pindex[%s] and lindex[%s]: %s' % (entry, entry, error)) else: # Not already know m_index[pindex['CFG']['DESCRIPTION']] = pindex for cfg in indexcfg: lindex = None branch = base_branch indextype = None indexurl = None indexname = None indexcache = None if 'TYPE' in cfg: indextype = cfg['TYPE'] if 'URL' in cfg: indexurl = cfg['URL'] if 'DESCRIPTION' in cfg: indexname = cfg['DESCRIPTION'] if 'CACHE' in cfg: indexcache = cfg['CACHE'] if 'BRANCH' in cfg: branch = cfg['BRANCH'] # Replace magic values with real values... for (find, rep) in replace: indexurl = indexurl.replace(find, rep) if branch: branch = branch.replace(find, rep) # Do we have an mirrored version? If so use it, skip regular processing if indexname in m_index: logger.plain('Using index %s from the mirror index...' % (indexname)) lindex = m_index[indexname] else: logger.plain('Loading index %s from %s...' % (indexname, indexurl)) # If not previously loaded from the mirror, attempt to load... if not lindex: if indextype == 'restapi-web': lindex = self.load_API_Index(indexurl, indexname, branches=branch) elif indextype == 'restapi-files': lindex = self.load_serialized_index(indexurl, name=indexname) elif indextype == 'export': lindex = self.load_django_export(indexurl, name=indexname) elif not lindex: # Unknown index type... logger.error('Unknown index type %s' % indextype) raise SyntaxError('Unknown index type %s' % indextype) # If we couldn't pull from the regular location, pull from the cache! if lindex is None and indexcache and os.path.exists(indexcache + '.json'): logger.plain('Falling back to the index cache %s...' % (indexcache)) lindex = self.load_serialized_index(indexcache + '.json', name=indexname, branches=[branch]) if not lindex or 'branches' not in lindex or 'layerItems' not in lindex or 'layerBranches' not in lindex: logger.warning('Index %s was empty... Ignoring.' % indexname) continue # Start data transforms... for entry in lindex['layerItems']: for obj in entry: # Run replace on any 'url' items. if 'url' in obj: vcs_url = entry[obj] for (find, rep) in replace: vcs_url = vcs_url.replace(find, rep) entry[obj] = vcs_url # Cache the data we loaded... (after replacements) if we loaded data. if lindex and indexcache: dir = os.path.dirname(indexcache) if dir: os.makedirs(dir, exist_ok=True) self.serialize_index(lindex, indexcache, split=False) lindex['CFG'] = cfg lindex['CFG']['BRANCH'] = branch if lindex and 'distros' in lindex: # Default setup is actually implemented as 'nodistro' for (idx, dist) in enumerate(lindex['distros']): if dist['name'] == "defaultsetup": dist['name'] = 'nodistro' lindex['distros'][idx] = dist # Everything works off layerBranches, so make sure to keep it sorted! lindex['layerBranches'] = self.sortEntry(lindex['layerBranches']) if lindex: self.index.append(lindex) def load_API_Index(self, url, name=None, branches=None): """ Fetches layer information from a remote layer index. The return value is a dictionary containing API, layer, branch, dependency, recipe, machine, distro, and template information. url is the url to the rest api of the layer index, such as: http://layers.openembedded.org/layerindex/api/ branches is a str or list of branches to filter on """ lindex = {} assert url is not None logger.debug('Loading %s from url %s...' % (name, url)) def _get_json_response(apiurl=None, retry=True): assert apiurl is not None res = utils_setup.fetch_url(apiurl) try: parsed = json.loads(res.read().decode('utf-8')) except ConnectionResetError: if retry: logger.debug("%s: Connection reset by peer. Retrying..." % url) parsed = _get_json_response(apiurl=apiurl, retry=False) logger.debug("%s: retry successful.") else: logger.critical("%s: Connection reset by peer." % url) logger.critical("Is there a firewall blocking your connection?") sys.exit(1) except: if retry: logger.debug("%s: get response failed. Retrying..." % url) parsed = _get_json_response(apiurl=apiurl, retry=False) logger.debug("%s: retry successful.") else: logger.critical("%s: get response failed" % url) sys.exit(1) return parsed from urllib.request import URLError try: lindex['apilinks'] = _get_json_response(url) except URLError as e: logger.warning("Index %s: could not connect to %s: %s" % (name, url, e.reason)) return None filter = "" # If it's list, keep it, if it's a string change it to a list if branches and type(branches) == type(str()): branches = [branches] if branches: filter = "?filter=name:%s" \ % "OR".join(branches) lindex['branches'] = _get_json_response(lindex['apilinks']['branches'] + filter) if not lindex['branches']: logger.warning("No valid branches (%s) found at url %s." % (branches or "*", url)) return lindex filter = "" if branches: filter = "?filter=branch__name:%s" \ % "OR".join(branches) lindex['layerBranches'] = _get_json_response(lindex['apilinks']['layerBranches'] + filter) if not lindex['layerBranches']: logger.warning("No layers on branches (%s) found at url %s." % (branches or "*", url)) return lindex layerids = [] for i, layerBranch in enumerate(lindex['layerBranches']): layerids.append("%s" % layerBranch['layer']) lindex['layerItems'] = _get_json_response(lindex['apilinks']['layerItems']) filter = "" if branches: filter = "?filter=layerbranch__branch__name:%s" \ % "OR".join(branches) lindex['layerDependencies'] = _get_json_response(lindex['apilinks']['layerDependencies'] + filter) filter = "" if branches: filter = "?filter=layerbranch__branch__name:%s" \ % "OR".join(branches) lindex['machines'] = _get_json_response(lindex['apilinks']['machines'] + filter) filter = "" if branches: filter = "?filter=layerbranch__branch__name:%s" \ % "OR".join(branches) lindex['recipes'] = _get_json_response(lindex['apilinks']['recipes'] + filter) filter = "" if branches: filter = "?filter=layerbranch__branch__name:%s" \ % "OR".join(branches) if 'distros' in lindex['apilinks']: lindex['distros'] = _get_json_response(lindex['apilinks']['distros'] + filter) else: # Not all layer indexes have a distribution API. If not we need to emulate nodistro. lindex['distros'] = [] idx = 1 for branch in lindex['branches']: for lb in self.getLayerBranch(lindex, branch['id'], name='openembedded-core'): lindex['distros'].append({"layerbranch": lb['id'], "id": idx, "description": "default", "updated": "2016-01-01T00:00:00+0000", "name": "nodistro"}) idx = idx + 1 filter = "" if branches: filter = "?filter=layerbranch__branch__name:%s" \ % "OR".join(branches) if 'wrtemplates' in lindex['apilinks']: lindex['wrtemplates'] = _get_json_response(lindex['apilinks']['wrtemplates'] + filter) else: lindex['wrtemplates'] = [] if 'YPCompatibleVersions' in lindex['apilinks']: lindex['YPCompatibleVersions'] = _get_json_response(lindex['apilinks']['YPCompatibleVersions']) else: lindex['YPCompatibleVersions'] = [] logger.debug('...loading %s from url %s, done.' % (name, url)) return lindex # Merge listone and listtwo, returning listtwo def __add_cmp_lists(self, listone, listtwo): # Copy the items from listone, into listtwo -- if it isn't already # there.. if it is there, verify it's the same or raise an error... if not listone: # List one is empty, just return listtwo return listtwo if not listtwo: # List two is empty, just return listone return listone for one in reversed(listone): found = False for two in listtwo: if 'id' in one and 'id' in two: if one['id'] == two['id']: found = True # Contents need to be the same! if one != two: # Something is out of sync here... raise TypeError('Cannot merge two objects with the same id %s, but different contents:\n%s\n%s' % (one['id'], one, two)) break else: # This is not a valid object if not 'id' in one and 'id' in two: raise TypeError('No id in object from the first parameter list:\n%s' % (one)) if 'id' in one and not 'id' in two: raise TypeError('No id in object from the second parameter list:\n%s' % (two)) if not 'id' in one and not 'id' in two: raise TypeError('No id in object in either parameter list:\n%s\n%s' % (one, two)) if not found: listtwo.append(one) return listtwo def load_serialized_index(self, path, name=None, branches=None): lindex = {} lindex['branches'] = [] lindex['layerItems'] = [] lindex['layerBranches'] = [] lindex['layerDependencies'] = [] lindex['recipes'] = [] lindex['machines'] = [] lindex['distros'] = [] lindex['wrtemplates'] = [] lindex['YPCompatibleVersions'] = [] assert path is not None def loadCache(path): logger.debug('Loading json file %s' % path) pindex = json.load(open(path, 'rt', encoding='utf-8')) for entry in pindex: if 'apilinks' == entry: continue if 'CFG' == entry: # Conflicts don't matter here, just accept it lindex[entry] = pindex[entry] continue if entry not in lindex: lindex[entry] = [] try: lindex[entry] = self.__add_cmp_lists(pindex[entry], lindex[entry]) except TypeError as error: raise TypeError('Merge failed of pindex[%s] and lindex[%s]: %s' % (entry, entry, error)) logger.debug('...loading json file %s, done.' % path) if os.path.exists(path) and os.path.isdir(path): logger.debug('Loading %s from directory %s...' % (name, path)) for (dirpath, dirnames, filenames) in os.walk(path): for filename in filenames: if not filename.endswith('.json'): continue fpath = os.path.join(dirpath, filename) loadCache(fpath) logger.debug('...loading %s from path %s, done.' % (name, path)) elif os.path.exists(path): logger.debug('Loading %s from path %s...' % (name, path)) loadCache(path) logger.debug('...loading %s from path %s, done.' % (name, path)) else: logger.error("Index %s: could not find path %s" % (name, path)) return None return lindex def load_django_export(self, path, name=None, branches=None): lindex = {} lindex['branches'] = [] lindex['layerItems'] = [] lindex['layerBranches'] = [] lindex['layerDependencies'] = [] lindex['recipes'] = [] lindex['machines'] = [] lindex['distros'] = [] lindex['wrtemplates'] = [] lindex['YPCompatibleVersions'] = [] assert path is not None def loadDB(path): def constructObject(entry): obj = entry['fields'].copy() obj['id'] = entry['pk'] return obj pindex = {} logger.debug('Loading json file %s' % path) dbindex = json.load(open(path, 'rt', encoding='utf-8')) # We discard anything that doesn't start with 'layerindex.' # the other data is adminstrative and stuff we should not mess with for entry in dbindex: if 'model' in entry: model = entry['model'] if model.startswith('layerindex.'): if 'branch' == model[11:]: name = 'branches' elif 'layeritem' == model[11:]: name = 'layerItems' elif 'layerbranch' == model[11:]: name = 'layerBranches' elif 'layerdependency' == model[11:]: name = 'layerDependencies' elif 'recipe' == model[11:]: name = 'recipes' elif 'machine' == model[11:]: name = 'machines' elif 'distro' == model[11:]: name = 'distros' elif 'wrtemplate' == model[11:]: name = 'wrtemplates' elif 'ypcompatibleversion' == model[11:]: name = 'YPCompatibleVersions' else: name = model[11:] if name not in pindex: pindex[name] = [] pindex[name].append(constructObject(entry)) for entry in pindex: if entry not in lindex: lindex[entry] = [] try: lindex[entry] = self.__add_cmp_lists(pindex[entry], lindex[entry]) except TypeError as error: raise TypeError('Merge failed of pindex[%s] and lindex[%s]: %s' % (entry, entry, error)) logger.debug('...loading json file %s, done.' % path) if os.path.exists(path) and os.path.isdir(path): logger.debug('Loading %s from path %s...' % (name, path)) for (dirpath, dirnames, filenames) in os.walk(path): for filename in filenames: if not filename.endswith('.json'): continue fpath = os.path.join(dirpath, filename) loadDB(fpath) logger.debug('...loading %s from path %s, done.' % (name, path)) elif os.path.exists(path): logger.debug('Loading %s from path %s...' % (name, path)) loadDB(path) logger.debug('...loading %s from path %s, done.' % (name, path)) else: logger.error("Index %s: could not find path %s" % (name, path)) return None return lindex # Provide a function to sort layer index content (restapi format) # When serializing the data this is import to limit # changes to the files... def sortEntry(self, item): newitem = item try: if type(newitem) == type(dict()): newitem = OrderedDict(sorted(newitem.items(), key=lambda t: t[0])) elif type(newitem) == type(list()): newitem.sort(key=lambda obj: obj['id']) for index, entry in enumerate(newitem): newitem[index] = self.sortEntry(newitem[index]) except: pass return newitem # Sort and return a new restapi style index def sortRestApi(self, index): lindex = self.sortEntry(index) for entry in lindex: lindex[entry] = self.sortEntry(lindex[entry]) return lindex # layerBranches must be a list of layerBranch entries to parse, it only affects # output when 'split' is True. def serialize_index(self, lindex, path, split=False, layerBranches=None, IncludeCFG=False, mirror=False, base_url=None): # If we're not splitting, we must be caching... if not split: dir = os.path.dirname(path) base = os.path.basename(path) fname = base.translate(str.maketrans('/ ', '__')) fpath = os.path.join(dir, fname) # Need to filter out local information pindex = {} for entry in lindex: if (IncludeCFG == False and 'CFG' == entry) or 'apilinks' == entry: continue pindex[entry] = lindex[entry] json.dump(self.sortRestApi(pindex), open(fpath + '.json', 'wt'), indent=4) return # We serialize based on the layerBranches, this allows us to subset # everything in a logical way... if not layerBranches: layerBranches = lindex['layerBranches'] for lb in layerBranches: pindex = {} def filter_item(lb, objects): filtered = [] for obj in lindex[objects]: if 'layerbranch' in obj: if obj['layerbranch'] == lb['id']: filtered.append(obj) elif 'layer' in obj: for layer in self.find_layer(lindex, layerBranch=lb): if obj['layer'] == layer['id']: filtered.append(obj) else: # No simple filter method, just include it... filtered.append(obj) return filtered for entry in lindex: if (IncludeCFG == False and 'CFG' == entry) or 'apilinks' == entry or 'branches' == entry or 'layerBranches' == entry or 'layerItems' == entry: continue elif (IncludeCFG == True and 'CFG' == entry): pindex[entry] = lindex[entry] continue pindex[entry] = filter_item(lb, entry) for branch in lindex['branches']: if branch['id'] == lb['branch']: pindex['branches'] = [branch] # We must include the layerbranch for what we are processing... pindex['layerBranches'] = [lb] # We also need to include the layerbranch for any required dependencies... (required, recommended) = self.getDependencies(lindex, lb) for req_lb in required: found = False for p_lb in pindex['layerBranches']: if p_lb['id'] == req_lb['id']: found = True break if found == False: pindex['layerBranches'].append(req_lb) # We need to include the layerItems for each layerBranch pindex['layerItems'] = [] for p_lb in pindex['layerBranches']: for li in self.find_layer(lindex, layerBranch=p_lb): found = False for p_li in pindex['layerItems']: if p_li['id'] == li['id']: found = True break if found == False: pindex['layerItems'].append(li) # If we're mirroring, we need to adjust the URL for the # mirror to work properly. Replace remote with BASE_URL. # (This uses the same logic as the default.xml construction) if mirror == True: from urllib.parse import urlparse from copy import deepcopy pindex['layerItems'] = deepcopy(pindex['layerItems']) for layer in pindex['layerItems']: vcs_url = layer['vcs_url'] if base_url and vcs_url.startswith(base_url): layer['vcs_url'] = layer['vcs_url'].replace(base_url, '#BASE_URL#') else: url = urlparse(vcs_url) if url.scheme: layer['vcs_url'] = layer['vcs_url'].replace(url.scheme + '://' + url.netloc, '#BASE_URL#') dir = os.path.dirname(path) base = os.path.basename(path) fname = base + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name'] fname = fname.translate(str.maketrans('/ ', '__')) fpath = os.path.join(dir, fname) json.dump(self.sortRestApi(pindex), open(fpath + '.json', 'wt'), indent=4) # layerBranches must be a list of layerBranch entries to parse, it only affects # output when 'split' is True. def serialize_django_export(self, lindex, path, split=False, layerBranches=None, IncludeCFG=False): def convertToDjango(restindex): dbindex = [] def constructObject(entry, model): obj = OrderedDict() obj['pk'] = entry['id'] obj['model'] = model obj['fields'] = OrderedDict(sorted(entry.items(), key=lambda t: t[0])) del obj['fields']['id'] if model == 'layerindex.branch' and 'update_environment' in obj['fields']: if 'pythonenvironment' not in restindex: # We have 'lost' the environment, so workaround it being missing... obj['fields']['update_environment'] = None return obj # Convert the restindex to a dbindex for entry in restindex: if (IncludeCFG == False and 'CFG' == entry) or 'apilinks' == entry: continue elif 'branches' == entry: model = 'layerindex.branch' elif 'layerItems' == entry: model = 'layerindex.layeritem' elif 'layerBranches' == entry: model = 'layerindex.layerbranch' elif 'layerDependencies' == entry: model = 'layerindex.layerdependency' elif 'recipes' == entry: model = 'layerindex.recipe' elif 'machines' == entry: model = 'layerindex.machine' elif 'distros' == entry: model = 'layerindex.distro' elif 'wrtemplates' == entry: model = 'layerindex.wrtemplate' else: model = 'layerindex.' + entry for item in restindex[entry]: dbindex.append(constructObject(item, model)) return dbindex # Just write out a single master file.. if not split: dir = os.path.dirname(path) base = os.path.basename(path) fname = base.translate(str.maketrans('/ ', '__')) fpath = os.path.join(dir, fname) # Need to filter out local information pindex = {} for entry in lindex: if (IncludeCFG == False and 'CFG' == entry) or 'apilinks' == entry: continue pindex[entry] = lindex[entry] json.dump(convertToDjango(self.sortRestApi(pindex)), open(fpath + '.json', 'wt'), indent=4) return # We serialize based on the layerBranches, this allows us to subset # everything in a logical way... if not layerBranches: layerBranches = lindex['layerBranches'] for lb in layerBranches: pindex = {} def filter_item(lb, objects): filtered = [] for obj in lindex[objects]: if 'layerbranch' in obj: if obj['layerbranch'] == lb['id']: filtered.append(obj) elif 'layer' in obj: for layer in self.find_layer(lindex, layerBranch=lb): if obj['layer'] == layer['id']: filtered.append(obj) else: # No simple filter method, just include it... filtered.append(obj) return filtered for entry in lindex: if (IncludeCFG == False and 'CFG' == entry) or 'apilinks' == entry or 'branches' == entry or 'layerBranches' == entry or 'layerItems' == entry: continue pindex[entry] = filter_item(lb, entry) pindex['layerBranches'] = [lb] pindex['layerItems'] = self.find_layer(lindex, layerBranch=lb) for branch in lindex['branches']: if branch['id'] == lb['branch']: pindex['branches'] = [branch] dir = os.path.dirname(path) base = os.path.basename(path) fname = base + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name'] fname = fname.translate(str.maketrans('/ ', '__')) fpath = os.path.join(dir, fname) json.dump(convertToDjango(self.sortRestApi(pindex)), open(fpath + '.json', 'wt'), indent=4) def print_close_matches(self, key, value, full_list): msg = '%s "%s" not found' % (key, value) close_matches = difflib.get_close_matches(value, full_list) if close_matches: msg += ". Close matches:\n %s" % '\n '.join(list(set(close_matches))) logger.critical(msg + '\n') def find_layer(self, lindex, id=None, name=None, layerBranch=None, layerBranchId=None, distro=None, machine=None, recipe=None, wrtemplate=None): result = [] if layerBranch: id = layerBranch['layer'] # Only one layerItem per lindex, so break once we find it if id: for layer in lindex['layerItems']: if layer['id'] == id: result.append(layer) if layerBranch: result[-1]['collection'] = layerBranch['collection'] else: for branch in lindex['layerBranches']: if branch['layer'] == id: result[-1]['collection'] = branch['collection'] break break return result if name: full_list = [] found = False for layer in lindex['layerItems']: value_from_index = layer['name'] full_list.append(value_from_index) if value_from_index == name: result.append(layer) for branch in lindex['layerBranches']: if branch['layer'] == layer['id']: result[-1]['collection'] = branch['collection'] break found = True break if not found: self.print_close_matches('layer', name, full_list) return result layerBranchIds = [] if layerBranchId: layerBranchIds.append(layerBranchId) args = { 'distros': distro, 'machines': machine, 'recipes': recipe, 'wrtemplates': wrtemplate } for k, v in args.items(): if v: full_list = [] found = False for index_dict in lindex[k]: if k == 'recipes': value_from_index = index_dict['pn'] else: value_from_index = index_dict['name'] full_list.append(value_from_index) if value_from_index == v: found = True layerBranchIds.append(index_dict['layerbranch']) if not found: self.print_close_matches(k.rstrip('s'), v, full_list) if layerBranchIds: for layerBranch in lindex['layerBranches']: for layerBranchId in layerBranchIds: if layerBranch['id'] == layerBranchId: for layer in lindex['layerItems']: if layer['id'] == layerBranch['layer']: result.append(layer) result[-1]['collection'] = layerBranch['collection'] return result return None def get_index_layers(self, base_branch): index_layers = {} for lindex in self.index: index = lindex['CFG']['DESCRIPTION'] or lindex['CFG']['URL'] index_layers[index] = [] branchid = self.getBranchId(lindex, self.getIndexBranch(default=base_branch, lindex=lindex)) if branchid: for lb in lindex['layerBranches']: if lb['branch'] == branchid: index_layers[index].extend(self.find_layer(lindex, layerBranch=lb)) return index_layers def list_layers(self, base_branch): index_layers = self.get_index_layers(base_branch) for index, layers in index_layers.items(): logger.plain ('Index: %s' % index) table = tt.Texttable() table.set_deco(tt.Texttable.HEADER) table.set_header_align(['l', 'l']) table.set_cols_align(['l', 'l']) table.header(['layer', 'summary']) table.set_cols_dtype(['t', 't']) for layer in layers: name = layer['name'] summary = layer['summary'] or name table.add_row([name, summary]) s = table.draw() logger.plain (s) logger.plain ('') def getYPCompatibleVersion(self, lindex, id): if not id: return [] for vers in lindex['YPCompatibleVersions']: if vers['id'] == id: return vers['name'].split() return [] def list_obj(self, base_branch, object, display, compat='all'): for lindex in self.index: logger.plain ('Index: %s' % (lindex['CFG']['DESCRIPTION'] or lindex['CFG']['URL'])) table = tt.Texttable() table.set_deco(tt.Texttable.HEADER) table.set_header_align(['l', 'l', 'l']) table.set_cols_align(['l', 'l', 'l']) table.header(['display', 'description', 'layer']) table.set_cols_dtype(['t', 't', 't']) table.set_max_width(100) branchid = self.getBranchId(lindex, self.getIndexBranch(default=base_branch, lindex=lindex)) if branchid: # there are more layerBranches then objects (usually)... for lb in lindex['layerBranches']: if compat != 'all': if compat not in self.getYPCompatibleVersion(lindex, lb['yp_compatible_version']): continue for layer in self.find_layer(lindex, layerBranch=lb): for obj in lindex[object]: if obj['layerbranch'] == lb['id'] and lb['branch'] == branchid: lname = layer['name'] name = obj['name'] description = (obj['description'] or name).strip() table.add_row([name, description, lname]) s = table.draw() logger.plain(s) logger.plain ('') def get_machines(self, base_branch, compat='all'): machines = [] for lindex in self.index: branchid = self.getBranchId(lindex, self.getIndexBranch(default=base_branch, lindex=lindex)) if branchid: for lb in lindex['layerBranches']: if compat != 'all': if compat not in self.getYPCompatibleVersion(lindex, lb['yp_compatible_version']): continue for layer in self.find_layer(lindex, layerBranch=lb): for obj in lindex['machines']: if obj['layerbranch'] == lb['id'] and lb['branch'] == branchid: machines.append(obj['name']) return machines def list_distros(self, base_branch, compat): self.list_obj(base_branch, 'distros', 'distro', compat) def list_machines(self, base_branch, compat): self.list_obj(base_branch, 'machines', 'machine', compat) def list_wrtemplates(self, base_branch, compat): self.list_obj(base_branch, 'wrtemplates', 'templates', compat) def list_recipes(self, base_branch): for lindex in self.index: logger.plain ('Index: %s' % (lindex['CFG']['DESCRIPTION'] or lindex['CFG']['URL'])) logger.plain ('%s %s %s' % (('{:15}'.format('recipe'), '{:9}'.format('version'), 'summary'))) logger.plain ('{:-^80}'.format("")) branchid = self.getBranchId(lindex, self.getIndexBranch(default=base_branch, lindex=lindex)) if branchid: # there are more layerBranches then objects (usually)... for lb in lindex['layerBranches']: for layer in self.find_layer(lindex, layerBranch=lb): for obj in lindex['recipes']: if obj['layerbranch'] == lb['id'] and lb['branch'] == branchid: lname = layer['name'] pn = obj['pn'] pv = obj['pv'] summary = (obj['summary'] or pn).strip() logger.plain('%s %s %s' % ('{:15}'.format(pn), '{:9}'.format(pv), summary[:50])) logger.plain ('') def getBranchId(self, lindex, name): for branch in lindex['branches']: if branch['name'] == name: return branch['id'] return None def getLayerBranch(self, lindex, branchid, layerBranchId=None, collection=None, name=None, distro=None, machine=None, recipe=None, wrtemplate=None, layerItem=None): result = [] if layerBranchId: for lb in lindex['layerBranches']: if branchid == lb['branch'] and layerBranchId == lb['id']: result.append(lb) break return result if collection: for lb in lindex['layerBranches']: if 'collection' in lb and collection == lb['collection']: result.append(lb) break return result layerItems = [] if layerItem: layerItems.append(layerItem) args = { 'name': name, 'distro': distro, 'machine': machine, 'recipe': recipe, 'wrtemplate': wrtemplate } for k, v in args.items(): if v: if k == 'name': layerItem = self.find_layer(lindex, name=v) elif k == 'distro': layerItem = self.find_layer(lindex, distro=v) elif k == 'machine': layerItem = self.find_layer(lindex, machine=v) elif k == 'recipe': layerItem = self.find_layer(lindex, recipe=v) elif k == 'wrtemplate': layerItem = self.find_layer(lindex, wrtemplate=v) if layerItem: layerItems = layerItems + layerItem if layerItems: for layerItem in layerItems: for lb in lindex['layerBranches']: if branchid == lb['branch'] and layerItem['id'] == lb['layer']: result.append(lb) return result return None def getDependencies(self, lindex, layerBranch): required = [] recommended = [] for ld in lindex['layerDependencies']: if layerBranch['id'] == ld['layerbranch']: layers = self.find_layer(lindex, id=ld['dependency']) if (not layers or layers == []) and ld['required'] == True: logger.warning('%s: Unable to find dependency %s -- Skipping' % (self.find_layer(lindex, layerBranch=layerBranch)[0]['name'], ld['dependency'])) for layer in layers: for lb in self.getLayerBranch(lindex, layerBranch['branch'], layerItem=layer): if not lb: continue if not 'required' in ld or ld['required'] == True: #print('li_getdep_dep: %s (%s) req %s (%s)' % (self.find_layer(lindex, layerBranch=layerBranch)['name'], layerBranch['id'], self.find_layer(lindex, layerBranch=lb)['name'], ld['dependency']) ) required.append(lb) else: #print('li_getdep_dep: %s (%s) rec %s (%s)' % (self.find_layer(lindex, layerBranch=layerBranch)['name'], layerBranch['id'], self.find_layer(lindex, layerBranch=lb)['name'], ld['dependency']) ) recommended.append(lb) return (required, recommended) def getLayerInfo(self, lindex, layerBranch): collection = None name = None vcs_url = None if 'collection' in layerBranch: collection = layerBranch['collection'] for layer in self.find_layer(lindex, layerBranch=layerBranch): if layer: name = layer['name'] vcs_url = layer['vcs_url'] break return (collection, name, vcs_url) def getBranch(self, lindex, branchid): for branch in lindex['branches']: if branch['id'] == branchid: return branch return None def getBitbakeBranch(self, lindex, branchid): branch = self.getBranch(lindex, branchid) if branch: return branch['bitbake_branch'] or branch['name'] return None def getIndexBranch(self, default=None, lindex=None): if lindex and 'CFG' in lindex and 'BRANCH' in lindex['CFG']: return lindex['CFG']['BRANCH'] return default ================================================ FILE: bin/logger_setup.py ================================================ # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # """ logger_setup is the infrastructure for setup.py to log both to the screen and to a file. Up on initialisation, the logger will only log to the screen and have two new log levels: plain = 15 to_file = 1 plain logs will not be prefixed with the logging type when outputting to the output passed to the setup_logging function (which defaults to sys.stdout). """ import logging import sys import time FILE_LOG_FORMAT = '%(asctime)s %(levelname)8s [%(filename)s:%(lineno)s' \ + ' - %(funcName)20s(): %(message)s' # INFO = 20 PLAIN_LOG_LEVEL = 15 # DEBUG = 10 TO_FILE_LOG_LEVEL = 1 # NOTSET = 0 logger = None def plain(self, message, *args, **kws): """Function to be added to the logger for plain log level support""" if self.isEnabledFor(PLAIN_LOG_LEVEL): self._log(PLAIN_LOG_LEVEL, message, args, **kws) def to_file(self, message, *args, **kws): """Function to be added to the logger for to_file log level support""" if self.isEnabledFor(TO_FILE_LOG_LEVEL): self._log(TO_FILE_LOG_LEVEL, message, args, **kws) def setup_logging(level=PLAIN_LOG_LEVEL, output=sys.stdout): """create the setup.py singleton logger.""" global logger if logger: return logger logger = logging.getLogger('setup.py') formatter = ScreenFormatter("%(levelname)s: %(message)s") stream_h = logging.StreamHandler(output) stream_h.setFormatter(formatter) # Logging timezone is UTC stream_h.converter = time.gmtime logger.setLevel(level) logger.addHandler(stream_h) logging.addLevelName(PLAIN_LOG_LEVEL, "PLAIN") logging.Logger.plain = plain return logger class FileFormatter(logging.Formatter): def format(self, record): # FileHandler doesn't need color record.levelname = record.levelname_orig return logging.Formatter.format(self, record) def setup_logging_file(log_file): """Add a logging.FileHandler to the logger""" global logger logger.debug("Logging to %s" % log_file) formatter = FileFormatter(FILE_LOG_FORMAT) file_h = logging.FileHandler(log_file) # Logging timezone is UTC file_h.converter = time.gmtime file_h.setFormatter(formatter) file_h.setLevel(TO_FILE_LOG_LEVEL) logger.addHandler(file_h) logging.addLevelName(TO_FILE_LOG_LEVEL, 'TO_FILE') logging.Logger.to_file = to_file class ScreenFormatter(logging.Formatter): """ScreenFormatter exists to allow printing plain messages with a different format than other log levels. """ RED, GREEN, YELLOW, BLUE = [1, 2, 3, 4] RESET_SEQ = "\033[0m" COLOR_SEQ = "\033[1;%dm" COLORS = { 'INFO': GREEN, 'DEBUG': BLUE, 'WARNING': YELLOW, 'ERROR': RED, 'CRITICAL': RED } def format(self, record): levelname = record.levelname # Save it for FileHandler record.levelname_orig = levelname if levelname == "PLAIN": msg = record.getMessage() else: if sys.stdout.isatty(): fore_color = 30 + self.COLORS[levelname] levelname_color = self.COLOR_SEQ % fore_color + levelname + self.RESET_SEQ record.levelname = levelname_color msg = logging.Formatter.format(self, record) msg = logging.Formatter.format(self, record) return msg # Add a class to emulate stdout/stderr class LoggerOut: def __init__(self, logger, isatty): self.logger = logger self.tty = isatty def write(self, message): # We skip any lines that are simply a '\n'. # The logger always ends in the equivalent of a \n, and many programs # seem to like to insert blank lines using '\n' which makes the # logging confusing. if message != '\n': self.logger(message) def flush(self): # We print all messages immediately, so flush is a no-op. pass def isatty(self): return self.tty ================================================ FILE: bin/sanity.py ================================================ #!/usr/bin/env python3 # Copyright (C) 2019 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Please keep these sorted. """ sanity.py module to help setup program to do sanity checks. """ import os import sys import logger_setup logger = logger_setup.setup_logging() # Fixed list of tools needed. fixed_hosttools = """ [ ar as awk basename bash bzip2 cat chgrp chmod chown chrpath cmp comm cp cpio cpp cut date dd diff diffstat dirname du echo egrep env expand expr false fgrep file find flock g++ gawk gcc getconf getopt git grep gunzip gzip head hostname iconv id install ld ldd ln ls make makeinfo md5sum mkdir mknod mktemp mv nm objcopy objdump od patch perl pod2man pr printf pwd python2 python2.7 python3 ranlib readelf readlink realpath rm rmdir rpcgen sed seq sh sha256sum sleep sort split stat strings strip tail tar tee test touch tr true uname uniq wc wget which xargs """ def which(path, item, direction = 0, executable=False): """ Locate `item` in the list of paths `path` (colon separated string like $PATH). If `direction` is non-zero then the list is reversed. If `executable` is True then the candidate has to be an executable file, otherwise the candidate simply has to exist. """ if executable: is_candidate = lambda p: os.path.isfile(p) and os.access(p, os.X_OK) else: is_candidate = lambda p: os.path.exists(p) paths = (path or "").split(':') if direction != 0: paths.reverse() for p in paths: next = os.path.join(p, item) if is_candidate(next): if not os.path.isabs(next): next = os.path.abspath(next) return next return "" def check_hosttools(additions = None): """ Check tools on host. Error out if some tool is missing. """ host_tools = [] notfound = [] try: import settings host_tools = settings.REQUIRED_HOSTTOOLS.split() if additions: for tool in additions: if tool not in host_tools: host_tools.append(tool) except Exception as e: logger.warn("Failed to get host tools from setting.py: %s" % e) logger.warn("Use a fixed list of host tools for checking.") host_tools = [] if not host_tools: # Fall back to a fixed list of host tools host_tools = fixed_hosttools.split() path = os.environ['PATH'] for tool in host_tools: srctool = which(path, tool, executable=True) # gcc/g++ may link to ccache on some hosts, e.g., # /usr/local/bin/ccache/gcc -> /usr/bin/ccache, then which(gcc) # would return /usr/local/bin/ccache/gcc, but what we need is # /usr/bin/gcc, this code can check and fix that. if "ccache" in srctool: srctool = which(path, tool, executable=True, direction=1) if not srctool: notfound.append(tool) if notfound: logger.error("Required host tools not available: %s" % notfound) sys.exit(1) else: logger.info("All required host tools are available.") # allow running sanity checks individually if __name__ == '__main__': sys.exit(check_hosttools(sys.argv[1:])) ================================================ FILE: bin/settings.py ================================================ # Settings for the installer # type: restapi-web - REST API from a LayerIndex-web # restapi-files - REST API, but only from files # export - Exported DB from a LayerIndex-web -- reads file(s) # url/path may contain: # #INSTALL_DIR# which is replaced by the setup directory # #BASE_URL# which is replaced by the base url for setup directory # #BASE_BRANCH# which is replaced by the base branch for the setup directory # Since we know the url below will be mirrored to our # internal git mirrors, change format so it can be replaced # with the proper value(s) REPLACE = [ ( 'git://lxgit.wrs.com', '#BASE_URL#' ) ] INDEXES = [ { 'DESCRIPTION' : 'Wind River Developer Layer Index', 'TYPE' : 'restapi-web', 'URL' : 'http://layers.wrs.com/layerindex/api/', 'CACHE' : 'config/index-cache/layers_wrs_com', }, # { # 'DESCRIPTION' : 'OpenEmbedded Layer Index', # 'TYPE' : 'restapi-web', # 'URL' : 'http://layers.openembedded.org/layerindex/api/', # 'CACHE' : 'config/index-cache/layers_openembedded_org', # 'BRANCH' : 'morty' # }, ] # Bitbake URL on the same server at openembedded-core # bitbake is assumed to be at the same basepath as OpenEmbedded-Core BITBAKE = "bitbake" # Base Layers (these layers and their dependencies are -always- included) BASE_LAYERS = "wrlinux" DEFAULT_DISTRO = "wrlinux" DEFAULT_MACHINE = "qemux86-64" # Default number of repo jobs REPO_JOBS = 4 # Repo remote name list REMOTES = [ ( 'git://git.openembedded.org', 'openembedded' ), ( 'git://git.yoctoproject.org', 'yoctoproject' ), ( 'http://git.yoctoproject.org', 'http_yoctoproject' ), ( 'git://github.com', 'github' ), ( 'https://github.com', 'https_github' ), ] # The default tag used to filter the output of --list actions. 'all' means no # filter DEFAULT_LAYER_COMPAT_TAG = "wrl" # required host tools to build out project # Do not edit this manually, it is updated automatically. REQUIRED_HOSTTOOLS = """ [ ar as awk basename bash bzip2 cat chgrp chmod chown chrpath cmp comm cp cpio cpp cut date dd diff diffstat dirname du echo egrep env expand expr false fgrep file find flock g++ gawk gcc getconf getopt git grep gunzip gzip head hostname iconv id install ld ldd ln ls make md5sum mkdir mkfifo mknod mktemp mv nm objcopy objdump od patch perl pr printf pwd python3 ranlib readelf readlink realpath rm rmdir rpcgen sed seq sh sha1sum sha224sum sha256sum sha384sum sha512sum sleep sort split stat strings strip tail tar tee test touch tr true uname uniq wc wget which xargs """ ================================================ FILE: bin/setup.py ================================================ #!/usr/bin/env python3 # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Please keep these sorted. import logging import os import shutil import subprocess import sys import time import re from urllib.parse import urlparse import utils_setup # Setup-specific modules import logger_setup from argparse_wrl import Argparse_Wrl from layer_index import Layer_Index import settings import sanity import xml.etree.ElementTree as ET logger = logger_setup.setup_logging() # Redirect stdout and stderr to the custom logger. This allows us to use # python modules that may output only via stdout/stderr. isatty = sys.stdout.isatty() sys.stdout = logger_setup.LoggerOut(logger.info, isatty) sys.stderr = logger_setup.LoggerOut(logger.error, isatty) class Setup(): tool_list = ['repo', 'git'] default_xml = 'default.xml' default_repo_quiet = '--quiet' class_config_dir = 'config' class_log_dir = 'log' check_repo_install_dir = '.repo/repo/.git' check_repo_sync_file = '.repo/projects/' replacement = {} BINTOOLS_SSL_DIR="/bin/buildtools/sysroots/x86_64-wrlinuxsdk-linux/usr/share/ca-certificates/mozilla" BINTOOLS_SSL_CERT= "/bin/buildtools/sysroots/x86_64-wrlinuxsdk-linux/etc/ssl/certs/ca-certificates.crt" def __init__(self): # Set various default values # Default -j for repo init self.jobs = str(settings.REPO_JOBS) # Pull in the defaults from the environment (set by setup.sh) self.base_url = os.getenv('OE_BASEURL') self.base_branch = os.getenv('OE_BASEBRANCH') self.buildtools_branch = os.getenv('OE_BUILDTOOLS_BRANCH') self.buildtools_remote = os.getenv('OE_BUILDTOOLS_REMOTE') self.another_buildtools_remote = os.getenv('OE_ANOTHER_BUILDTOOLS_REMOTE') # Real project or a mirror? self.mirror = False self.mirror_index_path = None # Make/Use the project mirror as PREMIRRORS for do_fetch self.mirror_as_premirrors = False # Default configuration self.distros = [ settings.DEFAULT_DISTRO ] self.machines = [ settings.DEFAULT_MACHINE ] self.layers = [] self.recipes = [] self.wrtemplates = [] self.all_layers = False self.dl_layers = False self.local_layers = [] self.remote_layers = [] # The extra layer groups enabled by user self.use_layer_groups = [] self.no_recommend = False self.no_network = False self.allowed_network = None self.remotes = {} self.requiredlayers = [] self.recommendedlayers = [] # Default quiet: self.quiet = self.default_repo_quiet self.repo_verbose = False # Default depth self.depth = None # Default to NOT force-sync self.force_sync = None self.repo_url = None if 'REPO_URL' in os.environ: self.repo_url = os.environ['REPO_URL'] self.repo_rev = None if 'REPO_REV' in os.environ: self.repo_rev = os.environ['REPO_REV'] self.debug_lvl = 0 self.repo_no_fetch = False # Set the install_dir # Use the path from this file. Note bin has to be dropped. self.install_dir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + '/../') # Default location for the related XML files self.xml_dir = os.path.join(self.install_dir, 'data/xml') # Set the directory where we're running. self.project_dir = os.getcwd() self.conf_dir = os.path.join(self.project_dir, self.class_config_dir) # Save current base_branch to compare with next one self.saved_base_branch = os.path.join(self.conf_dir, 'saved_base_branch') # Environment setup self.env = os.environ.copy() # We do NOT want to inherit python home from the environment # See Issue: LIN1018-2934 # python3 wrapper from the buildtools sets this, which causes host # python tools to fail if 'PYTHONHOME' in self.env: del self.env['PYTHONHOME'] self.setup_env() # Config flags self.list_distros = False self.list_machines = None self.list_layers = False self.list_recipes = False self.list_wrtemplates = False self.premirrors_dl = os.path.join(self.project_dir, 'premirrors-dl') self.premirrors_dl_downloads = os.path.join(self.premirrors_dl, 'downloads') def exit(self, ret=0): logger.debug("setup.py finished (ret=%s)" % (ret)) sys.exit(ret) def start_file_logging(self): log_dir = os.path.join(self.conf_dir, self.class_log_dir) if not os.path.exists(log_dir): os.makedirs(log_dir) log_file = '%s/%s.log' % (log_dir, time.strftime('%Y-%m-%d-%H:%M:%S+0000', time.gmtime())) logger_setup.setup_logging_file(log_file) def main(self, orig_args): parser = Argparse_Wrl(self) # We want to default to help mode lacking any args. if not orig_args or not orig_args[1:]: orig_args.append('--help') parser.evaluate_args(orig_args[1:]) self.setup_args = " ".join(orig_args[1:]) self.extra_group_keys = parser.extra_group_keys self.start_file_logging() logger.debug('REPO_URL = %s' % self.repo_url) logger.debug('REPO_BRANCH = %s' % self.repo_rev) if not self.base_url: logger.error('Unable to determine base url, you may need to specify --base-url=') if not self.base_branch: logger.error('Unable to determine base branch, you may need to specify --base-branch=') # Check for require host tools for real project if not self.mirror: sanity.check_hosttools(self.tool_list) if not self.base_url or not self.base_branch: self.exit(1) # Check for all the tools and create a dictionary of the path # This shouldn't fail, because sanity.check_hosttools already checked for these... self.tools = {i : self.get_path(i) for i in self.tool_list} if None in self.tools.values(): sys.exit(1) self.load_layer_index() if len(self.index.index) == 0: logger.critical('No indexes could be loaded. This could be due to an invalid branch or tag. Exiting...') sys.exit(1) if self.list_distros: compat = self.list_distros if compat == 'default': compat = settings.DEFAULT_LAYER_COMPAT_TAG self.index.list_distros(self.base_branch, compat) if self.list_machines: compat = self.list_machines if compat == 'default': compat = settings.DEFAULT_LAYER_COMPAT_TAG self.index.list_machines(self.base_branch, compat) if self.list_layers: self.index.list_layers(self.base_branch) if self.list_recipes: self.index.list_recipes(self.base_branch) if self.list_wrtemplates: compat = self.list_wrtemplates if compat == 'default': compat = settings.DEFAULT_LAYER_COMPAT_TAG self.index.list_wrtemplates(self.base_branch, compat) if self.list_distros or self.list_machines or self.list_layers or self.list_recipes or self.list_wrtemplates: sys.exit(0) logger.debug('setup.py started') logger.debug('Calling setup main with arguments %s' % str(orig_args)) # Log debug which may have been missed due to log level. logger.debug("PATH=%s" % self.env["PATH"]) # Check saved_base_branch vs current base_branch self.check_base_branch() logger.debug("Tools are:") for key in self.tools: logger.debug("%s -> %s" % (key, self.tools[key])) logger.plain('Setting distro to "%s"' % (",".join(self.distros))) logger.plain('Setting machine to "%s"' % (",".join(self.machines))) if self.layers != []: logger.plain('Setting layers to "%s"' % (",".join(self.layers))) if self.recipes != []: logger.plain('Setting recipes to "%s"' % (",".join(self.recipes))) if self.wrtemplates != []: logger.plain('Setting templates to "%s"' % (",".join(self.wrtemplates))) self.process_layers() self.project_setup() self.__prep_replacements() if self.mirror != True: # We only want to do this if we're not mirroring... self.update_project() else: # Setup an index for others to use if we're mirroring... self.update_mirror() self.update_mirror_index() self.update_manifest() self.check_default_xml() self.update_gitignore() self.commit_files() self.check_project_path() self.repo_sync() if self.mirror_as_premirrors: if self.mirror: self.make_mirror_as_premirrors() else: self.use_mirror_as_premirrors() self.exit(0) def check_project_path(self): self.manifest_dir = os.path.join(self.project_dir, '.repo/manifests.git') project_dir_last = "" if os.path.exists(self.manifest_dir): cmd = [self.tools['git'], 'config', '--get', 'remote.origin.url'] try: p = subprocess.run(cmd, check=True, cwd=self.manifest_dir, stdout=subprocess.PIPE) project_dir_last = p.stdout.decode('utf-8').strip() except Exception as e: logger.warning('Failed to run "%s": %s' % (' '.join(cmd), e)) logger.debug('last project dir: %s' % project_dir_last) if project_dir_last and project_dir_last != self.project_dir: logger.info('project dir has been changed from %s to %s' % (project_dir_last, self.project_dir)) logger.info('Updating config files for new project dir...') cmd = [self.tools['git'], 'config', 'remote.origin.url', self.project_dir] subprocess.run(cmd, cwd=self.manifest_dir) def load_mirror_index(self, remote_mirror, folder=""): # See if there is a mirror index available from the BASE_URL mirror_index = os.path.join(self.conf_dir, 'mirror-index') try: cmd = [self.tools['git'], 'ls-remote', remote_mirror, self.base_branch] utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=self.project_dir, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) except: try: remote_mirror += "/.git" cmd = [self.tools['git'], 'ls-remote', remote_mirror, self.base_branch] utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=self.project_dir, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) except: # No mirror, return return None logger.plain('Loading the mirror index from %s (%s)...' % (remote_mirror, self.base_branch)) # This MIGHT be a valid mirror.. if not os.path.exists(mirror_index): os.makedirs(mirror_index) cmd = [self.tools['git'], 'init' ] utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=mirror_index) try: # We don't know if we're fetching a branch or tag, if it's a tag we have to do this # in two steps anyway, so always go to 'FETCH_HEAD' and then branch it. cmd = [self.tools['git'], 'fetch', '-n', '-u', remote_mirror, self.base_branch] utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=mirror_index) cmd = [self.tools['git'], 'checkout', '-B', folder + self.base_branch, 'FETCH_HEAD'] utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=mirror_index) except: # Could not fetch, return return None logger.debug('Found mirrored index.') cmd = [self.tools['git'], 'checkout', folder + self.base_branch ] utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=mirror_index) cmd = [self.tools['git'], 'reset', '--hard' ] utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=mirror_index) return mirror_index def check_base_branch(self): logger.debug('Checking saved_base_branch vs current base_branch') sync_file = os.path.join(self.project_dir, self.check_repo_sync_file) if not os.path.exists(sync_file): logger.debug("Skip the check since %s doesn't exist" % sync_file) return if not os.path.exists(self.saved_base_branch): logger.debug("Skip the check since %s doesn't exist" % self.saved_base_branch) return with open(self.saved_base_branch, 'r') as f: saved_base_branch = f.read().rstrip('\n') if saved_base_branch == self.base_branch: logger.debug('saved_base_branch and current base_branch are the same') return repo = self.tools['repo'] cmd = [repo, 'status'] with subprocess.Popen(cmd, stdout=subprocess.PIPE, env=self.env, cwd=self.project_dir) as proc: output = proc.stdout.read().decode('utf-8') if re.search('^project\s+layers/.*\s+branch', output, flags=re.M): logger.error("Found checked out branches, here is the output from 'repo status':\n%s" % output) logger.error('Switching base branch with checked out projects is not allowed,') logger.error('but you are switching from "%s" to "%s".'% (saved_base_branch, self.base_branch)) logger.error('Consider using either of the following ways to fix the problem:') logger.error('- Run "git checkout m/master" in the projects') logger.error('- Use "repo abandon [--all | ] [...]" to abandon the branch\n') self.exit(1) def load_layer_index(self): # Load Layer_Index if not (self.base_branch == "master" or self.base_branch == "master-wr"): from windshare import Windshare ws = Windshare(debug=self.debug_lvl) # Determine if this is a windshare install (ws_base_url, ws_base_folder, ws_entitlement_url) = ws.get_windshare_urls(self.base_url) if ws_base_url and ws_base_url != "" and ws.load_folders(ws_entitlement_url): logger.plain('Detected Windshare configuration. Processing entitlements and indexes.') for folder in ws.folders: self.mirror_index_path = ws.load_mirror_index(self, ws_base_url, folder) ws.write_local_mirror_index(self, self.mirror_index_path) # We need to adjust the base_url so everything works properly... self.base_url = ws_base_url # Adjust the location of the buildtools (was based on the original base_url) if self.buildtools_remote: self.buildtools_remote = ws_base_folder + '/' + self.buildtools_remote if self.another_buildtools_remote: self.another_buildtools_remote = ws_base_folder + '/' + self.another_buildtools_remote else: logger.debug('No Windshare configuration detected.') else: logger.debug('Windshare configuration disabled, building %s.' % self.base_branch) # Check if we have a mirror-index, and load it if we do... if not self.mirror_index_path: self.mirror_index_path = self.load_mirror_index(self.base_url + '/mirror-index') # Is this a tag? if so... we only pull from mirror-indexes if not self.mirror_index_path and self.base_branch.startswith('refs/tags/'): logger.error("An install from a repository tag (%s) can only be installed with a corresponding layerindex snapshot." % self.base_branch) logger.error("Unable to find %s" % self.base_url + '/mirror-index') sys.exit(1) # Mirror also has a copy of the associated XML bits if self.mirror_index_path: self.xml_dir = os.path.join(self.mirror_index_path, 'xml') # Setup replace strings as late as possible. The various self.* values # may be modified prior to this place. replace = [] replace = replace + settings.REPLACE replace = replace + [ ( '#INSTALL_DIR#', self.install_dir ), ( '#BASE_URL#', self.base_url ), ( '#BASE_BRANCH#', self.base_branch ), ] self.index = Layer_Index(indexcfg=settings.INDEXES, base_branch=self.base_branch, replace=replace, mirror=self.mirror_index_path) # Is this a Wind River tag? if so... we need to modify the 'branches' entries to be the same as the tag if self.base_branch.startswith('refs/tags/vWRLINUX'): if '_RCPL' in self.base_branch or '_UPDATE' in self.base_branch: # vWRLINUX__UPDATExxxx # vWRLINUX__RCPLxxxx new_base_branch = self.base_branch[11:] else: # vWRLINUX__10.19.25.0 new_base_branch_pos = self.base_branch.find('10.') new_base_branch = self.base_branch[:(new_base_branch_pos-1)][11:] logger.debug('Detected a WR tag, replacing %s with %s...' % (self.base_branch[11:], new_base_branch)) for lindex in self.index.index: for branch in lindex['branches']: if branch and branch['bitbake_branch'] == new_base_branch: branch['bitbake_branch'] = self.base_branch if branch and branch['name'] == new_base_branch: branch['name'] = self.base_branch def is_group_layer(self, layer_name): """ Is it an extra group layer? """ for group_key in self.extra_group_keys: if '-%s-' % group_key in layer_name or layer_name.endswith('-%s' % group_key): return True return False def is_enabled_group(self, layer_name): """ Is it an extra group layer which is enabled by user? """ if layer_name in self.layers: return True for group_key in self.extra_group_keys: if group_key in self.use_layer_groups: return True return False def process_layers(self): from collections import deque # We allow duplicates in the queue, they're filtered later # Queue of required objects requiredQueue = deque([]) # Queue of recommended objects recommendedQueue = deque([]) logger.debug('Starting') # if this switches to false, we have to exit at the end of this function allfound = True # It all starts with BASE_LAYERS, so always include this. (only from index 0) lindex = self.index.index[0] branchid = self.index.getBranchId(lindex, self.get_branch(lindex=lindex)) if branchid: for lname in settings.BASE_LAYERS.split(): base_layerBranch = self.index.getLayerBranch(lindex, branchid, name=lname) if not base_layerBranch or not branchid: raise Exception('Unable to find base layer: %s in the Layer_Index' % (lname)) for lb in base_layerBranch: requiredQueue.append( (lindex, lb) ) # process the configuration arguments (find the layers we need for the project) # if an item is 'layer:item', then the 'foo' part must match a layer name. def procConfig(layer=None, distro=None, machine=None, recipe=None, wrtemplate=None): item = ["", layer][layer != None] item = item + ["", distro][distro != None] item = item + ["", machine][machine != None] item = item + ["", recipe][recipe != None] item = item + ["", wrtemplate][wrtemplate != None] type = ["", 'layer'][layer != None] type = type + ["", 'distro'][distro != None] type = type + ["", 'machine'][machine != None] type = type + ["", 'recipe'][recipe != None] type = type + ["", 'template'][wrtemplate != None] if (':' in item): # User told us which layer, so ignore the other bits -- they can be used later... layer = item.split(':')[0] distro = None machine = None recipe = None wrtemplate = None # TODO: We do not actually verify the item we asked for (if a layer was specified) is available found = False for lindex in self.index.index: branchid = self.index.getBranchId(lindex, self.get_branch(lindex=lindex)) if not branchid: continue for layerBranch in self.index.getLayerBranch(lindex, branchid, name=layer, distro=distro, machine=machine, recipe=recipe, wrtemplate=wrtemplate) or []: requiredQueue.append( (lindex, layerBranch) ) found = True if found: break if not found: return False return True for l in self.layers: if not procConfig(layer=l): allfound = False for l in self.distros: if not procConfig(distro=l): allfound = False for l in self.machines: if not procConfig(machine=l): if self.local_layers or self.remote_layers: logger.info("Machine %s not found but validation disabled due to local or remote layers specified.", l) else: allfound = False else: if l not in self.index.get_machines(self.base_branch, settings.DEFAULT_LAYER_COMPAT_TAG): logger.critical('Unsupported machine: %s' % l) self.exit(1) for l in self.recipes: if not procConfig(recipe=l): allfound = False for l in self.wrtemplates: if not procConfig(wrtemplate=l): allfound = False if not allfound: logger.critical('Please correct the missing items, exiting.') self.exit(1) # Add all layers -- if necessary if self.all_layers == True: for lindex in self.index.index: branchid = self.index.getBranchId(lindex, self.get_branch(lindex=lindex)) if not branchid: continue for l in lindex['layerItems']: l_name = l['name'] if self.is_group_layer(l_name) and not self.is_enabled_group(l_name): logger.info('Skipping extra group layer %s' % l_name) continue for layerBranch in self.index.getLayerBranch(lindex, branchid, layerItem=l) or []: # dl layers are always added as recommends in an --all-layers mode if '-dl-' in l_name or l_name.endswith('-dl'): recommendedQueue.append( (lindex, layerBranch) ) else: requiredQueue.append( (lindex, layerBranch) ) # Compute requires and recommended layers... # List of 'collection' and layer 'name'. This list is used to avoid # including duplicates. Collection is matched first, then name -- as not # all layer indexes may contain 'collection' depCacheCol = [] depCacheName = [] def checkCache(lindex, layerBranch, addCache=False): (collection, name, vcs_url) = self.index.getLayerInfo(lindex, layerBranch=layerBranch) if collection in depCacheCol or name in depCacheName: return True if addCache: if collection: depCacheCol.append(collection) if name: depCacheName.append(name) return False def resolveIndexOrder(lindex, layerBranch, Queue): # We want to recompute the dependency in INDEXES order... (collection, name, vcs_url) = self.index.getLayerInfo(lindex, layerBranch) found = False for pindex in self.index.index: # We already know it'll be in this index, so we just use it as-is... if pindex == lindex: break # Look for the collection (or name if no collection) in the indexes in # priority order... pbranchid = self.index.getBranchId(pindex, self.get_branch(lindex=pindex)) if collection: new_layerBranches = self.index.getLayerBranch(pindex, pbranchid, collection=collection) if new_layerBranches and new_layerBranches != []: for lb in new_layerBranches: logger.info('Resolving dependency %s from %s to %s from %s' % (name, lindex['CFG']['DESCRIPTION'], name, pindex['CFG']['DESCRIPTION'])) Queue.append( (pindex, lb) ) lindex = None layerBranch = None break if name: new_layerBranches = self.index.getLayerBranch(pindex, pbranchid, name=name) if new_layerBranches and new_layerBranches != []: for lb in new_layerBranches: logger.info('Resolving dependency %s from %s to %s from %s' % (name, lindex['CFG']['DESCRIPTION'], name, pindex['CFG']['DESCRIPTION'])) Queue.append( (pindex, lb) ) lindex = None layerBranch = None break return (lindex, layerBranch) while requiredQueue: (lindex, layerBranch) = requiredQueue.popleft() (lindex, layerBranch) = resolveIndexOrder(lindex, layerBranch, requiredQueue) if not lindex or not layerBranch: continue if not checkCache(lindex, layerBranch, True): self.requiredlayers.append( (lindex, layerBranch) ) (required, recommended) = self.index.getDependencies(lindex, layerBranch) for dep in required: requiredQueue.append( (lindex, dep) ) for dep in recommended: recommendedQueue.append( (lindex, dep) ) while recommendedQueue: (lindex, layerBranch) = recommendedQueue.popleft() (lindex, layerBranch) = resolveIndexOrder(lindex, layerBranch, recommendedQueue) if not lindex or not layerBranch: continue if not checkCache(lindex, layerBranch, True): if self.dl_layers != True: layers = self.index.find_layer(lindex, id=layerBranch['layer']) if layers and ('-dl-' in layers[0]['name'] or layers[0]['name'].endswith('-dl')): # Skip the download layer continue self.recommendedlayers.append( (lindex, layerBranch) ) (required, recommended) = self.index.getDependencies(lindex, layerBranch) for dep in required + recommended: recommendedQueue.append( (lindex, dep) ) unexpected_groups = [] for (lindex, layerBranch) in self.requiredlayers + self.recommendedlayers: for layer in self.index.find_layer(lindex, id=layerBranch['layer']): l_name = layer['name'] if self.is_group_layer(l_name) and not self.is_enabled_group(l_name): if not l_name in unexpected_groups: unexpected_groups.append(l_name) unexpected_groups.sort() if unexpected_groups: logger.error('The following layer(s) need --use-layer-group= or --layers=:\n%s' % '\n'.join(unexpected_groups)) logger.error('Try to rerun with one of the options?') self.exit(1) # Also compute the various remotes self.remotes['base'] = self.base_url def process_remote(lindex, layerBranch): for layer in self.index.find_layer(lindex, id=layerBranch['layer']): add_remote_entry(layer['vcs_url']) for remote_layer in self.remote_layers: add_remote_entry(remote_layer.get('url')) def add_remote_entry(vcs_url): found = False for remote in self.remotes: if vcs_url.startswith(self.remotes[remote]): found = True break if not found: url = urlparse(vcs_url) if not url.scheme: self.remotes['local'] = '/' found = True if not found: for (remoteurl, remotename) in settings.REMOTES: if vcs_url.startswith(remoteurl): self.remotes[remotename] = remoteurl found = True break if not found: self.remotes[url.scheme + '_' + url.netloc.translate(str.maketrans('/:', '__'))] = url.scheme + '://' + url.netloc for (lindex, layerBranch) in self.requiredlayers + self.recommendedlayers: process_remote(lindex, layerBranch) def display_layer(lindex, layerBranch): branchid = self.index.getBranchId(lindex, self.get_branch(lindex=lindex)) for layer in self.index.find_layer(lindex, id=layerBranch['layer']): vcs_url = layer['vcs_url'] path = 'layers/' + "".join(vcs_url.split('/')[-1:]) if (layer['name'] == 'openembedded-core'): bitbakeBranch = self.index.getBranch(lindex, layerBranch['branch'])['bitbake_branch'] logger.debug('bitbake: %s %s %s' % ( settings.BITBAKE, path + '/bitbake', bitbakeBranch )) actual_branch = layerBranch['actual_branch'] or self.index.getBranch(lindex, branchid)['name'] logger.debug('%s: %s %s %s' % (layer['name'], vcs_url, path, actual_branch )) logger.debug('Computed required layers:') for (lindex, layerBranch) in self.requiredlayers: display_layer(lindex, layerBranch) logger.debug('Computed recommended layers:%s' % (["", ' (skipping)'][self.no_recommend == True])) for (lindex, layerBranch) in self.recommendedlayers: display_layer(lindex, layerBranch) # Recommends are disabled, filter it... if self.no_recommend == True: if self.dl_layers == True: newRecommendedlayers = [] for (lindex, layerBranch) in self.recommendedlayers: layers = self.index.find_layer(lindex, id=layerBranch['layer']) if layers and ('-dl-' in layers[0]['name'] or layers[0]['name'].endswith('-dl')): newRecommendedlayers.append( (lindex, layerBranch) ) self.recommendedlayers = newRecommendedlayers else: self.recommendedlayers = [] # if any of the remote layers have the same path as one of the # layerindex layers override the vcs_url and # actual_branch. Keep a list of remote layers that were not # overrides so the final list of remote layers will have them # removed. Cannot modify a list while iterating over it additional_remote_layers = [] for remote_layer in self.remote_layers: found = False for (lindex, layerBranch) in self.requiredlayers + self.recommendedlayers: for layer in self.index.find_layer(lindex, id=layerBranch['layer']): vcs_url = layer['vcs_url'] path = 'layers/' + "".join(vcs_url.split('/')[-1:]) if remote_layer.get('path') == path: layer['vcs_url'] = remote_layer.get('url') layerBranch['actual_branch'] = remote_layer.get('branch') found = True break if found: break if not found and remote_layer not in additional_remote_layers: additional_remote_layers.append(remote_layer) # remote layers with the overrides removed self.remote_layers = additional_remote_layers logger.debug('Done') def project_setup(self): logger.debug('Starting') self.__setup_local_layer() if self.mirror != True: # We need to make sure the environment-setup link is always current for (dirpath, dirnames, filenames) in os.walk(os.path.join(self.project_dir, 'bin/buildtools')): for filename in filenames: if filename.startswith('environment-setup-'): src = os.path.relpath(os.path.join(dirpath, filename), self.project_dir) dst = os.path.join(self.project_dir, filename) if os.path.islink(dst): os.unlink(dst) os.symlink(src, dst) logger.debug('Done') def update_project(self): logger.debug('Starting') if not os.path.exists(self.project_dir + '/.templateconf'): tmplconf = open(self.project_dir + '/.templateconf', 'w') tmplconf.write('# Project template settings\n') tmplconf.write('TEMPLATECONF=${TEMPLATECONF:-$OEROOT/config}\n') tmplconf.close() self.copySample(self.install_dir + '/data/samples/README.sample', self.project_dir + '/README') self.copySample(self.install_dir + '/data/samples/bblayers.conf.sample', self.project_dir + '/config/bblayers.conf.sample') self.copySample(self.install_dir + '/data/samples/conf-notes.sample', self.project_dir + '/config/conf-notes.txt') self.copySample(self.install_dir + '/data/samples/local.conf.sample', self.project_dir + '/config/local.conf.sample') with open(self.project_dir + '/config/local.conf.sample', 'a') as f: inc = '\n# Add known layers to whitelist' inc += '\nrequire conf/wrlinux-whitelist.inc\n' f.write(inc) if os.path.exists(self.install_dir + '/data/samples/site.conf.sample'): self.copySample(self.install_dir + '/data/samples/site.conf.sample', self.project_dir + '/config/site.conf.sample') with open(self.saved_base_branch, 'w') as f: f.write('%s\n' % self.base_branch) def update_mirror(self): self.copySample(self.install_dir + '/data/samples/README-MIRROR.sample', self.project_dir + '/README') def __prep_replacements(self): self.replacement['layers'] = [] self.replacement['machines'] = {} self.replacement['distros'] = {} def addLayer(lindex, layerBranch): branchid = self.index.getBranchId(lindex, self.get_branch(lindex=lindex)) paths = [] for layer in self.index.find_layer(lindex, id=layerBranch['layer']): vcs_url = layer['vcs_url'] path = 'layers/' + "".join(vcs_url.split('/')[-1:]) if layerBranch['vcs_subdir']: path += '/' + layerBranch['vcs_subdir'] paths.append(path) return paths # Add layers to 'LAYERS' for (lindex, layerBranch) in self.requiredlayers + self.recommendedlayers: self.replacement['layers'] = self.replacement['layers'] + addLayer(lindex, layerBranch) # Add machines to 'MACHINES' for (lindex, layerBranch) in self.requiredlayers + self.recommendedlayers: for machine in lindex['machines']: if machine['layerbranch'] == layerBranch['id']: desc = machine['description'] or machine['name'] self.replacement['machines'][machine['name']] = desc # Add distro to 'DISTROS' for (lindex, layerBranch) in self.requiredlayers + self.recommendedlayers: for distro in lindex['distros']: if distro['layerbranch'] == layerBranch['id']: desc = distro['description'] or distro['name'] self.replacement['distros'][distro['name']] = desc def copySample(self, src, dst): src = open(src, 'r') dst = open(dst, 'w') for line in src: if '####LAYERS####' in line: for l in self.replacement['layers']: dst.write(line.replace('####LAYERS####', '##OEROOT##/%s' % (l))) for rl in self.remote_layers: dst.write(line.replace('####LAYERS####', '##OEROOT##/%s' % (rl.get('path')))) for ll in self.local_layers: dst.write(line.replace('####LAYERS####', '%s' % (ll))) continue if '####SETUP_ARGS####' in line: dst.write(line.replace('####SETUP_ARGS####', self.setup_args)) continue if '####MACHINES####' in line: for (name, desc) in sorted(self.replacement['machines'].items(), key=lambda t: t[0]): dst.write('# %s\n' % desc.strip()) dst.write(line.replace('####MACHINES####', name)) continue if '####DEFAULTMACHINE####' in line: name = self.machines[0] if ':' in name: name = ':'.join(name.split(':')[1:]) dst.write(line.replace('####DEFAULTMACHINE####', name)) continue if '####DISTROS####' in line: for (name, desc) in sorted(self.replacement['distros'].items(), key=lambda t: t[0]): dst.write('# %s\n' % desc.strip()) dst.write(line.replace('####DISTROS####', name)) continue if '####DEFAULTDISTRO####' in line: name = self.distros[0] if ':' in name: name = ':'.join(name.split(':')[1:]) dst.write(line.replace('####DEFAULTDISTRO####', name)) continue if '####DEFAULTWRTEMPLATE####' in line: dst.write(line.replace('####DEFAULTWRTEMPLATE####', ' '.join(self.wrtemplates))) continue dst.write(line) src.close() dst.close() def update_mirror_index(self): logger.debug('Starting') path = os.path.join(self.project_dir, 'mirror-index') logger.plain('Exporting mirror-index %s...' % (path)) if not os.path.exists(path): cmd = [self.tools['git'], 'init', path] utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=self.project_dir) try: cmd = [self.tools['git'], 'checkout', '-b', self.base_branch] utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=path) except: # if we failed, then simply try to switch branches cmd = [self.tools['git'], 'checkout', self.base_branch] utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=path) # Make sure the directory is empty, use -f to ignore failures for (dirpath, dirnames, filenames) in os.walk(path): if dirpath.endswith('/.git') or path + '/.git' in dirpath: continue for filename in filenames: os.remove(os.path.join(dirpath, filename)) # Construct a list of all layers we've downloaded, by url, including sublayers not activated url_cache = {} for (lindex, layerBranch) in self.requiredlayers + self.recommendedlayers: for layer in self.index.find_layer(lindex, id=layerBranch['layer']): vcs_url = layer['vcs_url'] if not vcs_url in url_cache: url_cache[vcs_url] = [] url_cache[vcs_url].append((lindex, layerBranch['branch'])) # Serialize the information for each of the layers (and their sublayers) for vcs_url in url_cache: for (lindex, branchid) in url_cache[vcs_url]: for layer in lindex['layerItems']: if layer['vcs_url'] in url_cache: for lb in self.index.getLayerBranch(lindex, branchid=branchid, layerItem=layer): self.index.serialize_index(lindex, os.path.join(path, lindex['CFG']['DESCRIPTION']), split=True, layerBranches=[lb], IncludeCFG=True, mirror=True, base_url=self.base_url) name = layer['name'] destdir = os.path.join(path, 'xml') srcfile = os.path.join(self.xml_dir, '%s.inc' % (name)) if os.path.exists(srcfile): os.makedirs(destdir, exist_ok=True) shutil.copy(srcfile, destdir) srcfile = os.path.join(self.xml_dir, '%s.xml' % (name)) if os.path.exists(srcfile): os.makedirs(destdir, exist_ok=True) shutil.copy(srcfile, destdir) # Special processing for the openembedded-core layer if name == 'openembedded-core': srcfile = os.path.join(self.xml_dir, 'bitbake.inc') if os.path.exists(srcfile): os.makedirs(destdir, exist_ok=True) shutil.copy(srcfile, destdir) srcfile = os.path.join(self.xml_dir, 'bitbake.xml') if os.path.exists(srcfile): os.makedirs(destdir, exist_ok=True) shutil.copy(srcfile, destdir) # git add file. cmd = [self.tools['git'], 'add', '-A', '.'] utils_setup.run_cmd(cmd, environment=self.env, cwd=path) try: cmd = [self.tools['git'], 'diff-index', '--quiet', 'HEAD', '--'] utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=path, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) except: logger.debug('Updating mirror-index') cmd = [self.tools['git'], 'commit', '-m', 'Updated index - %s' % (self.setup_args)] utils_setup.run_cmd(cmd, environment=self.env, cwd=path) logger.debug('Done') def update_manifest(self): logger.debug('Starting') fxml = open(os.path.join(self.project_dir, self.default_xml), 'w') fxml.write('\n') remote = 'base' fxml.write(' \n' % (remote, self.remotes[remote])) fxml.write(' \n' % (self.base_branch, remote, self.jobs)) for remote in sorted(self.remotes): if remote == 'base': continue fxml.write(' \n' % (remote, self.remotes[remote])) def open_xml_tag(name, url, remote, path, revision): fxml.write(' \n' % (url, remote, path, revision)) def inc_xml(name, url, remote, path, revision): # incfile is included inline and has to work as elements of the 'project' incfile = os.path.join(self.xml_dir, '%s.inc' % (name)) logger.debug('Looking for %s' % (incfile)) if os.path.exists(incfile): fbase = open(incfile, 'r') for line in fbase: fxml.write(line) fbase.close() def close_xml_tag(name, url, remote, path, revision): fxml.write(' \n') def add_xml(name, url, remote, path, revision): # xmlfile is included after the entry and is completely standalone xmlfile = os.path.join(self.xml_dir, '%s.xml' % (name)) logger.debug('Looking for %s' % (xmlfile)) if os.path.exists(xmlfile): fbase = open(xmlfile, 'r') for line in fbase: fxml.write(line) fbase.close() def write_xml(name, url, remote, path, revision): open_xml_tag(name, url, remote, path, revision) inc_xml(name, url, remote, path, revision) close_xml_tag(name, url, remote, path, revision) add_xml(name, url, remote, path, revision) if self.mirror == True and self.buildtools_branch: if self.buildtools_remote: write_xml('buildtools', self.buildtools_remote, 'base', self.buildtools_remote, self.buildtools_branch) if self.another_buildtools_remote: write_xml('buildtools', self.another_buildtools_remote, 'base', self.another_buildtools_remote, self.buildtools_branch) def process_xml_layers(allLayers): def process_xml_layer(lindex, layerBranch): branchid = self.index.getBranchId(lindex, self.get_branch(lindex=lindex)) for layer in self.index.find_layer(lindex, id=layerBranch['layer']): revision = layerBranch['actual_branch'] or self.index.getBranch(lindex, branchid)['name'] vcs_url = layer['vcs_url'] for remote in self.remotes: if vcs_url.startswith(self.remotes[remote]): break url = vcs_url[len(self.remotes[remote]):] url = url.strip('/') path = 'layers/' + "".join(url.split('/')[-1:]) entry = { 'name' : layer['name'], 'remote' : remote, 'path' : path, 'revision' : revision, } if url not in cache: cache[url] = [] if entry['name'] == 'openembedded-core': bitbakeurl = '/'.join(url.split('/')[:-1] + [ settings.BITBAKE ]) bitbakeBranch = self.index.getBranch(lindex, layerBranch['branch'])['bitbake_branch'] bitbake_entry = { 'name' : 'bitbake', 'remote' : remote, 'path' : path + '/bitbake', 'revision' : bitbakeBranch, } if bitbakeurl not in cache: cache[bitbakeurl] = [] cache[bitbakeurl].append(bitbake_entry) cache[url].append(entry) # We need to construct a list of layers with same urls... cache = {} for (lindex, layerBranch) in allLayers: process_xml_layer(lindex, layerBranch) from collections import OrderedDict for url in OrderedDict(sorted(cache.items(), key=lambda t: t[0])): name = cache[url][0]['name'] remote = cache[url][0]['remote'] path = cache[url][0]['path'] revision = cache[url][0]['revision'] open_xml_tag(name, url, remote, path, revision) for entry in cache[url]: inc_xml(entry['name'], url, remote, path, revision) close_xml_tag(name, url, remote, path, revision) for entry in cache[url]: add_xml(entry['name'], url, remote, path, revision) process_xml_layers(self.requiredlayers + self.recommendedlayers) # process the remote layers for remote_layer in self.remote_layers: vcs_url = remote_layer.get('url') for remote in self.remotes: if vcs_url.startswith(self.remotes[remote]): break url = vcs_url[len(self.remotes[remote]):] url = url.strip('/') name = "".join(url.split('/')[-1:]) path = remote_layer.get('path') revision = remote_layer.get('branch') open_xml_tag(name, url, remote, path, revision) close_xml_tag(name, url, remote, path, revision) fxml.write('\n') fxml.close() logger.debug('Done') def check_default_xml(self): """ * Check for duplicated basename in default.xml, e.g.: - path="/foo1/bar" - path="/foo2/bar" This doesn't work for flatten mirrors. * Check whether 'name' is endded with '.git' """ logger.debug('Starting checking duplicated path in xml') default_xml_dict = {} tree = ET.parse(self.default_xml) root = tree.getroot() for project in root.iter('project'): path = project.attrib['path'] if path: basename = os.path.basename(path) if basename in default_xml_dict: default_xml_dict[basename].append(path) else: default_xml_dict[basename] = [path] # The 'name' cannot be ended with '.git' name = project.attrib['name'] if name and name.endswith('.git'): name_no_git = name[:-4] logger.warning("%s cannot be ended with '.git', suggest %s" %(name, name_no_git)) for basename, paths in default_xml_dict.items(): if len(paths) > 1: logger.warning("Found duplicated basename in %s's attribute 'path':" % self.default_xml) for path in paths: logger.warning(path) logger.debug('Done') def get_git_premirrors_from_mirror_index(self, url, protocol): """ Generate PREMIRRORS conf for git from mirror-index. """ premirrors_conf_git = [] for xml in os.listdir(self.xml_dir): xml_file = os.path.join(self.xml_dir, xml) if not (xml.endswith('.xml') and '-dl' in xml): logger.debug('Skipping %s' % xml_file) continue # Parse xml file and save the name, these xml files are just xml # sections, not integrated xml, so can't use ET to parse them. logger.debug('Parsing %s' % xml_file) with open(xml_file) as x: for line in x.readlines(): if not 'name=' in line: continue # Part of the parsing errors may not matter since we only # need their parent dirs. try: name = line.split('name=')[1].split()[0].strip('"') except Exception as esc: logger.warning('Failed to get name from %s:%s' % (xml_file, line)) if name: conf = " git://.*/.* git://%s%s/%s/MIRRORNAME;protocol=%s \\n \\" % \ (url.netloc, url.path, os.path.dirname(name), protocol) if not conf in premirrors_conf_git: premirrors_conf_git.append(conf) premirrors_conf_git.append(conf.replace('git://', 'gitsm://')) # Make the subdirs in front to have a higher priority in PREMIRRORS, e.g.: # external/yocto # external/qt5 # [snip] # external if not premirrors_conf_git: logger.warning('Failed get figure out dirs from %s' % self.xml_dir) else: premirrors_conf_git.reverse() return premirrors_conf_git def make_mirror_as_premirrors(self): """ * dl*/downlaods: Use "git clone --local --branch " to clone dl layers into premirrors-dl, and copy (link) dl*/downloads/* into premirrors-dl/downloads, the premirrors-dl/downloads will be used as PREMIRRORS by client. * git repo sources: No action is needed since they are already bare repos. """ logger.info('Making project mirror as PREMIRRORS...') premirrors_dict = {} tree = ET.parse(self.default_xml) root = tree.getroot() for project in root.iter('project'): # Only need the dl layers if 'bare' in project.attrib and project.attrib['bare'] == "True": continue name = project.attrib['name'] if name and (name.endswith('-dl') or '-dl-' in name): try: path = project.attrib['path'] if path and path.startswith('layers/'): revision = project.attrib['revision'].replace('refs/tags/', '') premirrors_dict[name] = revision except Exception as esc: logger.warning('%s: Failed to find revision or path: %s' % (name, esc)) if premirrors_dict: if not os.path.exists(self.premirrors_dl): os.mkdir(self.premirrors_dl) else: logger.warning("mirror-as-premirrors: No dl layers found!") return for name, revision in premirrors_dict.items(): src = os.path.join(self.project_dir, name) dst = os.path.join(self.premirrors_dl, os.path.basename(name)) # Run the git reset and pull in the existed repo dst_git = os.path.join(dst, '.git') need_clone = True if os.path.exists(dst_git): logger.debug('Making %s as a PREMIRROR' % src) try: for cmd in ([self.tools['git'], 'fetch', '-q', 'origin', revision], \ [self.tools['git'], 'checkout', '-q', 'FETCH_HEAD']): utils_setup.run_cmd(cmd, environment=self.env, cwd=dst) need_clone = False except Exception as esc: logger.warning('%s: Failed to update it: %s' % (dst, esc)) logger.warning('%s: Removing it...' % dst) shutil.rmtree(dst) if need_clone: # There are a lot of messages when run "git clone --branch " # which rush the screen, so use "git clone -nq" to make it # quiet, and "git checkout " to checkout the files. cmd = [self.tools['git'], 'clone', '--local', '-nq', src, dst] utils_setup.run_cmd(cmd, environment=self.env) cmd = [self.tools['git'], 'checkout', '-q', revision] utils_setup.run_cmd(cmd, environment=self.env, cwd=dst) # Create a clean premirrors-dl/downloads as PREMIRRORS if os.path.exists(self.premirrors_dl_downloads): shutil.rmtree(self.premirrors_dl_downloads) os.mkdir(self.premirrors_dl_downloads) cmd = 'cp -alf %s/*-dl*/downloads/* %s' % (self.premirrors_dl, self.premirrors_dl_downloads) subprocess.run(cmd, shell=True) logger.info('The PREMIRROR files are prepared in %s' % self.premirrors_dl_downloads) def use_mirror_as_premirrors(self): """ Use mirror as PREMIRRORS if the project is cloned from a mirror """ logger.info('Using mirror as PREMIRRORS...') if not self.mirror_index_path: logger.warning("This project isn't setup from a project mirror, skipping mirror-as-premirrors") return premirrors_output = os.path.join(self.conf_dir, 'mirror-as-premirrors.conf') enable_network = False url = urlparse(self.base_url) if url.scheme and url.scheme != 'file': protocol = url.scheme enable_network = True else: protocol = 'file' premirrors_conf_git = self.get_git_premirrors_from_mirror_index(url, protocol) premirrors_conf_downloads = [] downloads_via_git = "" if protocol == 'git': downloads_via_git = "The protocol git:// is invalid for premirrors-dl/downloads" logger.warning('mirror-as-mirrorrs: %s' % downloads_via_git) else: premirrors_conf_downloads.append(" .*://.*/.* %s://%s%s/premirrors-dl/downloads/ \\n \\" % \ (protocol, url.netloc, url.path)) with open(premirrors_output, 'w') as f: f.write('# Use project mirror as PREMIRRORS for the build\n') if premirrors_conf_git: f.write('PREMIRRORS:append = " \\\n') f.write('%s\n"\n' % '\n'.join(premirrors_conf_git)) if premirrors_conf_downloads: f.write('PREMIRRORS:append = " \\\n') f.write('%s\n"\n' % '\n'.join(premirrors_conf_downloads)) if enable_network: # The network is for PREMIRRORS only f.write('\n# The network is for PREMIRRORS only\n') f.write('BB_NO_NETWORK = "0"\n') f.write('BB_FETCH_PREMIRRORONLY = "1"\n') local_conf_sample = os.path.join(self.project_dir, 'config/local.conf.sample') with open(local_conf_sample, 'a+') as f: f.seek(0) line = 'require ##OEROOT##/config/mirror-as-premirrors.conf\n' if not line in f.readlines(): f.write('\n# Use project mirror as PREMIRRORS for the build\n') f.write(line) def update_gitignore(self): logger.debug('Starting') ign_list = [ '.repo*', '*.pyc', '*.pyo', '*.swp', '*.orig', '*.rej', '*~', '/bin', '/environment-setup-*', '/layers/*', '!layers/local', os.path.basename(self.install_dir), ] tree = ET.parse(os.path.join(self.project_dir, 'default.xml')) root = tree.getroot() for linkfile in root.iter('linkfile'): dest = linkfile.attrib['dest'] if not '/' in dest: ign_list.append(dest) with open(os.path.join(self.project_dir, '.gitignore'), 'a+') as f: f.seek(0) existed = f.readlines() for l in ign_list: item = '%s\n' % l if item not in existed: f.write(item) logger.debug('Done') def commit_files(self): logger.debug('Starting') # List of all files that may change due to config filelist = [ 'README', 'default.xml', '.gitignore', '.gitconfig', 'config/index-cache', ] # If we are mirroring, skip all of these... if self.mirror != True: filelist.append('layers/local') filelist.append('.templateconf') filelist.append('config/bblayers.conf.sample') filelist.append('config/conf-notes.txt') filelist.append('config/local.conf.sample') filelist.append('config/saved_base_branch') if os.path.exists('config/site.conf.sample'): filelist.append('config/site.conf.sample') # Add log dir if it contains files if os.listdir('config/log'): filelist.append('config/log') # git init if not os.path.exists(self.project_dir + '/.git'): cmd = [self.tools['git'], 'init', self.project_dir] if self.quiet == self.default_repo_quiet: cmd.append(self.quiet) utils_setup.run_cmd(cmd, environment=self.env, cwd=self.conf_dir) # git add manifest. (Since these files are new, always try to add them) cmd = [self.tools['git'], 'add', '--'] + filelist utils_setup.run_cmd(cmd, environment=self.env, cwd=self.project_dir) try: cmd = [self.tools['git'], 'diff-index', '--quiet', 'HEAD', '--'] + filelist utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=self.project_dir, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) except: logger.plain('Updated project configuration') # Command failed -- so self.default_xml changed... cmd = [self.tools['git'], 'commit', '-q', '-m', 'Configuration change - %s' % (self.setup_args), '--'] + filelist utils_setup.run_cmd(cmd, environment=self.env, cwd=self.project_dir) logger.debug('Done') def repo_sync(self): if self.repo_no_fetch: logger.info('Skipping repo sync') return logger.debug('Starting') def checkout_to_repo_branch(): # Checkout to specific branch if self.repo_rev: cmd = [self.tools['git'], 'checkout', '-B', self.repo_rev, 'origin/%s' % self.repo_rev] utils_setup.run_cmd(cmd, environment=self.env, cwd=os.path.dirname(self.repo_dir)) self.repo_dir = os.path.join(self.project_dir, self.check_repo_install_dir) if os.path.exists(self.repo_dir): checkout_to_repo_branch() cmd = ['-j', self.jobs] self.call_repo_sync(cmd) else: # repo init cmd = ['-m', self.default_xml, '-u', self.project_dir] if self.mirror == True: cmd.append('--mirror') cmd.append('--no-repo-verify') self.call_repo_init(cmd) checkout_to_repo_branch() # repo sync cmd = ['-j', self.jobs] self.call_initial_repo_sync(cmd) logger.debug('Done') def __check_and_update_layerseries_compat(self, project_local_layer_path, data_local_layer_path): project_layer_conf = os.path.join(project_local_layer_path, 'conf/layer.conf') data_layer_conf = os.path.join(data_local_layer_path, 'conf/layer.conf') project_layerseries_compat_line = "" data_layerseries_compat_line = "" update_local_conf = False with open(data_layer_conf, 'r') as f: for line in f.readlines(): if 'LAYERSERIES_COMPAT_local' in line: data_layerseries_compat_line = line break with open(project_layer_conf, 'r') as f: lines = f.readlines() newlines = [] for line in lines: if 'LAYERSERIES_COMPAT_local' in line: project_layerseries_compat_line = line if project_layerseries_compat_line != data_layerseries_compat_line: newlines.append(data_layerseries_compat_line) update_local_conf = True else: newlines.append(project_layerseries_compat_line) else: newlines.append(line) if update_local_conf: logger.info("Update LAYERSERIES_COMPAT_local for local layer") with open(project_layer_conf, 'w') as f: for line in newlines: f.write(line) def __set_pnwhitelist_layers(self): """Set PNWHITELIST_LAYERS in wrlinux-whitelist.inc of layer local""" logger.debug('Checking PNWHITELIST_LAYERS in local layer') local_layer_conf = os.path.join(self.project_dir, 'layers/local', 'conf/wrlinux-whitelist.inc') index_layers = self.index.get_index_layers(self.base_branch) collections = [] for _, layers in index_layers.items(): for layer in layers: col = layer['collection'] if col: collections.append(layer['collection']) else: logger.warning('%s: BBFILE_COLLECTIONS is None' % layer['name']) existed = False lines = [] if os.path.isfile(local_layer_conf): with open(local_layer_conf, 'r') as f: for line in f: if re.match(r'PNWHITELIST_LAYERS\s*=', line): lines.append("PNWHITELIST_LAYERS = '%s'" % ' '.join(sorted(collections))) existed = True else: lines.append(line) else: lines.append('# DO NOT EDIT THIS FILE!') lines.append('\n# This file is automatically generated by setup.py\n') if not existed: lines.append('\n# Set PNWHITELIST_LAYERS with all known layers') lines.append("\nPNWHITELIST_LAYERS = '%s'" % ' '.join(sorted(collections))) with open(local_layer_conf, 'w') as f: f.writelines(lines) def __setup_local_layer(self): """Setup the local layer in /layers/local - if required.""" logger.debug('Checking local layer') if self.mirror is True: return project_local_layer_path = os.path.join(self.project_dir,'layers/local') data_local_layer_path = os.path.join(self.install_dir, 'data/local_layer') if os.path.exists(project_local_layer_path): # update LAYERSERIES_COMPAT_local if necessary self.__check_and_update_layerseries_compat(project_local_layer_path, data_local_layer_path) self.__set_pnwhitelist_layers() return logger.debug('Starting local layer') if not os.path.exists(os.path.join(self.project_dir, 'layers')): os.makedirs(os.path.join(self.project_dir, 'layers')) if not os.path.exists(os.path.join(self.project_dir, 'layers/local')): shutil.copytree(os.path.join(self.install_dir, 'data/local_layer'), os.path.join(self.project_dir, 'layers/local')) # make sure the local layer is writeable try: cmd = ['chmod', '-R', '+w', '%s' % (project_local_layer_path)] utils_setup.run_cmd(cmd, environment=self.env) except Exception as e: raise self.__set_pnwhitelist_layers() logger.debug('Done') def setup_env(self): self.set_ssl_cert() self.set_repo_git_env() self.add_bin_path() def add_bin_path(self): self.env["PATH"] = self.install_dir + "/bin:" + self.env["PATH"] def set_repo_git_env(self): # Set HOME to install_dir to use install_dir/.gitconfig settings. Otherwise the user will # be prompted for information. self.env["HOME"] = self.project_dir def set_ssl_cert(self): fn = self.project_dir + self.BINTOOLS_SSL_CERT dn = self.project_dir + self.BINTOOLS_SSL_DIR if os.path.exists(fn) and os.path.exists(dn): self.env["GIT_SSL_CAINFO"] = fn self.env["CURL_CA_BUNDLE"] = fn self.env["SSL_CERT_FILE"] = fn self.env["SSL_CERT_DIR"] = dn os.environ["SSL_CERT_FILE"] = fn os.environ["SSL_CERT_DIR"] = dn def call_repo_init(self, args): logger.debug('Starting') repo = self.tools['repo'] directory = os.path.join(self.project_dir, self.check_repo_install_dir) if os.path.exists(directory): logger.info('Done: detected repo init already run since %s exists' % directory) return cmd = [repo, 'init', '--no-clone-bundle'] if self.depth: cmd.append(self.depth) if self.repo_url: cmd.extend(['--repo-url', self.repo_url]) if self.repo_rev: cmd.extend(['--repo-branch', self.repo_rev]) log_it = 1 if self.repo_verbose is not True and self.quiet == self.default_repo_quiet: cmd.append(self.quiet) log_it = 0 cmd.extend(args) logger.debug("cmd: %s" % cmd) try: utils_setup.run_cmd(cmd, environment=self.env, log=log_it) except Exception as e: raise logger.debug('Done') # This only exists to check if we have fully sync'ed the project # Updating should use call_repo_sync def call_initial_repo_sync(self, args): logger.debug('Starting') sync_file= os.path.join(self.project_dir, self.check_repo_sync_file) local_only = 0 orig_args = list(args) if os.path.exists(sync_file): logger.info('Detected repo sync already run since %s exists' % sync_file) logger.info('Only running local update.') args.append('--local-only') local_only = 1 try: self.call_repo_sync(args) except Exception as e: if not local_only: raise else: logger.info('Using --local-only failed. Trying full sync.') try: self.call_repo_sync(orig_args) except Exception as e2: raise logger.debug('Done') def call_repo_sync(self, args): logger.debug('Starting') repo = self.tools['repo'] cmd = [repo, 'sync', '--prune'] # disable use of /clone.bundle on HTTP/HTTPS cmd.append('--no-clone-bundle') if self.force_sync: cmd.append(self.force_sync) log_it = 1 if self.repo_verbose is not True and self.quiet == self.default_repo_quiet: cmd.append(self.quiet) log_it = 0 cmd.extend(args) utils_setup.run_cmd(cmd, environment=self.env, log=log_it) logger.debug('Done') def get_branch(self, lindex=None): if lindex: return self.index.getIndexBranch(default=self.base_branch, lindex=lindex) return self.base_branch def get_path(self, tool): cmd = self.which(tool) if (not cmd): logger.critical('Cannot find %s in path!' % tool) logger.critical('Path was: %s' % os.environ['PATH']) return cmd # Helpers: Set_*, which.. def set_repo_verbose(self, verbose): self.repo_verbose = verbose def set_jobs(self, jobs): logger.debug('Setting jobs to %s' % jobs) self.jobs = jobs def set_depth(self, depth): if int(depth) <= 1: logger.info('repo depth %s is invalid, setting to 2' % depth) depth = '2' logger.debug('Setting depth to %s' % depth) self.depth = '--depth=%s' % depth def set_force_sync(self, sync): logger.debug('Setting force-sync to %s' % sync) if sync is True: self.force_sync = '--force-sync' def set_repo_url(self, url): logger.debug('Setting repo-url to %s' % url) self.repo_url = url def set_repo_rev(self, rev): logger.debug('Setting repo-rev to %s' % rev) self.repo_rev = rev def set_debug(self): self.debug_lvl += 1 self.set_debug_env() self.quiet = None logger.setLevel(logging.DEBUG) logger.debug('logging level set to DEBUG') def set_base_url(self, url): logger.debug('Setting base-url to %s' % url) self.base_url = url def set_base_branch(self, branch): logger.debug('Setting base-branch to %s' % branch) self.base_branch = branch def set_debug_env(self): self.env["REPO_CURL_VERBOSE"] = '1' def touch(self, fn): logger.debug("Creating %s" % fn) open(fn, 'a').close() ''' When this is python3.3, use built in version''' def which(self, program): path=self.env["PATH"] for path in path.split(os.path.pathsep): fullpath=os.path.join(path, program) if os.path.exists(fullpath) and os.access(fullpath,os.X_OK): return fullpath return None if __name__ == '__main__': try: x = Setup() x.main(sys.argv) except KeyboardInterrupt: logger.warning("Aborted by user, will terminate this setup.") sys.exit(1) ================================================ FILE: bin/test-network.py ================================================ #!/usr/bin/env python3 # Copyright (C) 2016-2017 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Please keep these sorted. import logging import os import sys import argparse import logger_setup logger = logger_setup.setup_logging() logger.setLevel(logging.DEBUG) # Redirect stdout and stderr to the custom logger. This allows us to use # python modules that may output only via stdout/stderr. sys.stdout = logger_setup.LoggerOut(logger.info) sys.stderr = logger_setup.LoggerOut(logger.error) def parse_args(): parser = argparse.ArgumentParser(description='test-network.py: Application to check network connection ability.') # Define arguments parser.add_argument("repoURL", help="The repoURL provided by Windshare.") parser.parse_args() return parser.parse_args() def dump_proxies(): logger.info("Checking if proxies are enabled in the environment...") if 'http_proxy' in os.environ: logger.debug("http_proxy = %s" % (os.environ['http_proxy'])) else: logger.debug("No http_proxy defined.") if 'https_proxy' in os.environ: logger.debug("https_proxy = %s" % (os.environ['https_proxy'])) else: logger.debug("No https_proxy defined.") def test_windshare(repoUrl): logger.info("Running Windshare test based on repoURL: %s" % repoUrl) from windshare import Windshare ws = Windshare(debug=1) ws.interactive = 1 from urllib.parse import urlsplit, urlunsplit (uscheme, uloc, upath, uquery, ufragid) = urlsplit(repoUrl) base_folder = os.path.basename(upath) if not base_folder or base_folder == "": logger.error("Invalid URL, unable to determine base_folder") return # Folder root is one directory higher upath = os.path.dirname(upath) base_url = urlunsplit((uscheme, uloc, upath, uquery, ufragid)) logger.debug("BASE URL = %s" % base_url) # Determine if this is a windshare install (ws_base_url, _, ws_entitlement_url) = ws.get_windshare_urls(base_url) if ws_base_url and ws_base_url != "" and ws.load_folders(ws_entitlement_url): logger.info('Detected Windshare configuration. Processing entitlements and indexes.') else: logger.info('Unable to detect Windshare configuration! No entitlements available.') return ws if __name__ == "__main__": args = parse_args() logger.info("------------- Environment Information --------------") logger.info(sys.version) logger.plain("") logger.info("---------------- Proxy Information -----------------") dump_proxies() logger.plain("") if args.repoURL: logger.info("-------------- Windshare Information ---------------") windshare = test_windshare(args.repoURL) logger.plain("") logger.info("------------- Available Entitlements ---------------") for folder in windshare.folders or [ 'none (Unable to load Windshare data)' ]: logger.info(' %s' % folder) logger.plain("") ================================================ FILE: bin/texttable.py ================================================ # texttable - module for creating simple ASCII tables # Copyright (C) 2003-2018 Gerome Fournier """module for creating simple ASCII tables Example: table = Texttable() table.set_cols_align(["l", "r", "c"]) table.set_cols_valign(["t", "m", "b"]) table.add_rows([["Name", "Age", "Nickname"], ["Mr\\nXavier\\nHuon", 32, "Xav'"], ["Mr\\nBaptiste\\nClement", 1, "Baby"], ["Mme\\nLouise\\nBourgeau", 28, "Lou\\n\\nLoue"]]) print table.draw() + "\\n" table = Texttable() table.set_deco(Texttable.HEADER) table.set_cols_dtype(['t', # text 'f', # float (decimal) 'e', # float (exponent) 'i', # integer 'a']) # automatic table.set_cols_align(["l", "r", "r", "r", "l"]) table.add_rows([["text", "float", "exp", "int", "auto"], ["abcd", "67", 654, 89, 128.001], ["efghijk", 67.5434, .654, 89.6, 12800000000000000000000.00023], ["lmn", 5e-78, 5e-78, 89.4, .000000000000128], ["opqrstu", .023, 5e+78, 92., 12800000000000000000000]]) print table.draw() Result: +----------+-----+----------+ | Name | Age | Nickname | +==========+=====+==========+ | Mr | | | | Xavier | 32 | | | Huon | | Xav' | +----------+-----+----------+ | Mr | | | | Baptiste | 1 | | | Clement | | Baby | +----------+-----+----------+ | Mme | | Lou | | Louise | 28 | | | Bourgeau | | Loue | +----------+-----+----------+ text float exp int auto =========================================== abcd 67.000 6.540e+02 89 128.001 efgh 67.543 6.540e-01 90 1.280e+22 ijkl 0.000 5.000e-78 89 0.000 mnop 0.023 5.000e+78 92 1.280e+22 """ from __future__ import division __all__ = ["Texttable", "ArraySizeError"] __author__ = 'Gerome Fournier ' __license__ = 'MIT' __version__ = '1.6.1' __credits__ = """\ Jeff Kowalczyk: - textwrap improved import - comment concerning header output Anonymous: - add_rows method, for adding rows in one go Sergey Simonenko: - redefined len() function to deal with non-ASCII characters Roger Lew: - columns datatype specifications Brian Peterson: - better handling of unicode errors Frank Sachsenheim: - add Python 2/3-compatibility Maximilian Hils: - fix minor bug for Python 3 compatibility frinkelpi: - preserve empty lines """ import sys import unicodedata # define a text wrapping function to wrap some text # to a specific width: # - use cjkwrap if available (better CJK support) # - fallback to textwrap otherwise try: import cjkwrap def textwrapper(txt, width): return cjkwrap.wrap(txt, width) except ImportError: try: import textwrap def textwrapper(txt, width): return textwrap.wrap(txt, width) except ImportError: sys.stderr.write("Can't import textwrap module!\n") raise # define a function to calculate the rendering width of a unicode character # - use wcwidth if available # - fallback to unicodedata information otherwise try: import wcwidth def uchar_width(c): """Return the rendering width of a unicode character """ return max(0, wcwidth.wcwidth(c)) except ImportError: def uchar_width(c): """Return the rendering width of a unicode character """ if unicodedata.east_asian_width(c) in 'WF': return 2 elif unicodedata.combining(c): return 0 else: return 1 from functools import reduce if sys.version_info >= (3, 0): unicode_type = str bytes_type = bytes else: unicode_type = unicode bytes_type = str def obj2unicode(obj): """Return a unicode representation of a python object """ if isinstance(obj, unicode_type): return obj elif isinstance(obj, bytes_type): try: return unicode_type(obj, 'utf-8') except UnicodeDecodeError as strerror: sys.stderr.write("UnicodeDecodeError exception for string '%s': %s\n" % (obj, strerror)) return unicode_type(obj, 'utf-8', 'replace') else: return unicode_type(obj) def len(iterable): """Redefining len here so it will be able to work with non-ASCII characters """ if isinstance(iterable, bytes_type) or isinstance(iterable, unicode_type): return sum([uchar_width(c) for c in obj2unicode(iterable)]) else: return iterable.__len__() class ArraySizeError(Exception): """Exception raised when specified rows don't fit the required size """ def __init__(self, msg): self.msg = msg Exception.__init__(self, msg, '') def __str__(self): return self.msg class FallbackToText(Exception): """Used for failed conversion to float""" pass class Texttable: BORDER = 1 HEADER = 1 << 1 HLINES = 1 << 2 VLINES = 1 << 3 def __init__(self, max_width=80): """Constructor - max_width is an integer, specifying the maximum width of the table - if set to 0, size is unlimited, therefore cells won't be wrapped """ self.set_max_width(max_width) self._precision = 3 self._deco = Texttable.VLINES | Texttable.HLINES | Texttable.BORDER | \ Texttable.HEADER self.set_chars(['-', '|', '+', '=']) self.reset() def reset(self): """Reset the instance - reset rows and header """ self._hline_string = None self._row_size = None self._header = [] self._rows = [] return self def set_max_width(self, max_width): """Set the maximum width of the table - max_width is an integer, specifying the maximum width of the table - if set to 0, size is unlimited, therefore cells won't be wrapped """ self._max_width = max_width if max_width > 0 else False return self def set_chars(self, array): """Set the characters used to draw lines between rows and columns - the array should contain 4 fields: [horizontal, vertical, corner, header] - default is set to: ['-', '|', '+', '='] """ if len(array) != 4: raise ArraySizeError("array should contain 4 characters") array = [ x[:1] for x in [ str(s) for s in array ] ] (self._char_horiz, self._char_vert, self._char_corner, self._char_header) = array return self def set_deco(self, deco): """Set the table decoration - 'deco' can be a combinaison of: Texttable.BORDER: Border around the table Texttable.HEADER: Horizontal line below the header Texttable.HLINES: Horizontal lines between rows Texttable.VLINES: Vertical lines between columns All of them are enabled by default - example: Texttable.BORDER | Texttable.HEADER """ self._deco = deco return self def set_header_align(self, array): """Set the desired header alignment - the elements of the array should be either "l", "c" or "r": * "l": column flushed left * "c": column centered * "r": column flushed right """ self._check_row_size(array) self._header_align = array return self def set_cols_align(self, array): """Set the desired columns alignment - the elements of the array should be either "l", "c" or "r": * "l": column flushed left * "c": column centered * "r": column flushed right """ self._check_row_size(array) self._align = array return self def set_cols_valign(self, array): """Set the desired columns vertical alignment - the elements of the array should be either "t", "m" or "b": * "t": column aligned on the top of the cell * "m": column aligned on the middle of the cell * "b": column aligned on the bottom of the cell """ self._check_row_size(array) self._valign = array return self def set_cols_dtype(self, array): """Set the desired columns datatype for the cols. - the elements of the array should be either a callable or any of "a", "t", "f", "e" or "i": * "a": automatic (try to use the most appropriate datatype) * "t": treat as text * "f": treat as float in decimal format * "e": treat as float in exponential format * "i": treat as int * a callable: should return formatted string for any value given - by default, automatic datatyping is used for each column """ self._check_row_size(array) self._dtype = array return self def set_cols_width(self, array): """Set the desired columns width - the elements of the array should be integers, specifying the width of each column. For example: [10, 20, 5] """ self._check_row_size(array) try: array = list(map(int, array)) if reduce(min, array) <= 0: raise ValueError except ValueError: sys.stderr.write("Wrong argument in column width specification\n") raise self._width = array return self def set_precision(self, width): """Set the desired precision for float/exponential formats - width must be an integer >= 0 - default value is set to 3 """ if not type(width) is int or width < 0: raise ValueError('width must be an integer greater then 0') self._precision = width return self def header(self, array): """Specify the header of the table """ self._check_row_size(array) self._header = list(map(obj2unicode, array)) return self def add_row(self, array): """Add a row in the rows stack - cells can contain newlines and tabs """ self._check_row_size(array) if not hasattr(self, "_dtype"): self._dtype = ["a"] * self._row_size cells = [] for i, x in enumerate(array): cells.append(self._str(i, x)) self._rows.append(cells) return self def add_rows(self, rows, header=True): """Add several rows in the rows stack - The 'rows' argument can be either an iterator returning arrays, or a by-dimensional array - 'header' specifies if the first row should be used as the header of the table """ # nb: don't use 'iter' on by-dimensional arrays, to get a # usable code for python 2.1 if header: if hasattr(rows, '__iter__') and hasattr(rows, 'next'): self.header(rows.next()) else: self.header(rows[0]) rows = rows[1:] for row in rows: self.add_row(row) return self def draw(self): """Draw the table - the table is returned as a whole string """ if not self._header and not self._rows: return self._compute_cols_width() self._check_align() out = "" if self._has_border(): out += self._hline() if self._header: out += self._draw_line(self._header, isheader=True) if self._has_header(): out += self._hline_header() length = 0 for row in self._rows: length += 1 out += self._draw_line(row) if self._has_hlines() and length < len(self._rows): out += self._hline() if self._has_border(): out += self._hline() return out[:-1] @classmethod def _to_float(cls, x): if x is None: raise FallbackToText() try: return float(x) except (TypeError, ValueError): raise FallbackToText() @classmethod def _fmt_int(cls, x, **kw): """Integer formatting class-method. - x will be float-converted and then used. """ return str(int(round(cls._to_float(x)))) @classmethod def _fmt_float(cls, x, **kw): """Float formatting class-method. - x parameter is ignored. Instead kw-argument f being x float-converted will be used. - precision will be taken from `n` kw-argument. """ n = kw.get('n') return '%.*f' % (n, cls._to_float(x)) @classmethod def _fmt_exp(cls, x, **kw): """Exponential formatting class-method. - x parameter is ignored. Instead kw-argument f being x float-converted will be used. - precision will be taken from `n` kw-argument. """ n = kw.get('n') return '%.*e' % (n, cls._to_float(x)) @classmethod def _fmt_text(cls, x, **kw): """String formatting class-method.""" return obj2unicode(x) @classmethod def _fmt_auto(cls, x, **kw): """auto formatting class-method.""" f = cls._to_float(x) if abs(f) > 1e8: fn = cls._fmt_exp else: if f - round(f) == 0: fn = cls._fmt_int else: fn = cls._fmt_float return fn(x, **kw) def _str(self, i, x): """Handles string formatting of cell data i - index of the cell datatype in self._dtype x - cell data to format """ FMT = { 'a':self._fmt_auto, 'i':self._fmt_int, 'f':self._fmt_float, 'e':self._fmt_exp, 't':self._fmt_text, } n = self._precision dtype = self._dtype[i] try: if callable(dtype): return dtype(x) else: return FMT[dtype](x, n=n) except FallbackToText: return self._fmt_text(x) def _check_row_size(self, array): """Check that the specified array fits the previous rows size """ if not self._row_size: self._row_size = len(array) elif self._row_size != len(array): raise ArraySizeError("array should contain %d elements" \ % self._row_size) def _has_vlines(self): """Return a boolean, if vlines are required or not """ return self._deco & Texttable.VLINES > 0 def _has_hlines(self): """Return a boolean, if hlines are required or not """ return self._deco & Texttable.HLINES > 0 def _has_border(self): """Return a boolean, if border is required or not """ return self._deco & Texttable.BORDER > 0 def _has_header(self): """Return a boolean, if header line is required or not """ return self._deco & Texttable.HEADER > 0 def _hline_header(self): """Print header's horizontal line """ return self._build_hline(True) def _hline(self): """Print an horizontal line """ if not self._hline_string: self._hline_string = self._build_hline() return self._hline_string def _build_hline(self, is_header=False): """Return a string used to separated rows or separate header from rows """ horiz = self._char_horiz if (is_header): horiz = self._char_header # compute cell separator s = "%s%s%s" % (horiz, [horiz, self._char_corner][self._has_vlines()], horiz) # build the line l = s.join([horiz * n for n in self._width]) # add border if needed if self._has_border(): l = "%s%s%s%s%s\n" % (self._char_corner, horiz, l, horiz, self._char_corner) else: l += "\n" return l def _len_cell(self, cell): """Return the width of the cell Special characters are taken into account to return the width of the cell, such like newlines and tabs """ cell_lines = cell.split('\n') maxi = 0 for line in cell_lines: length = 0 parts = line.split('\t') for part, i in zip(parts, list(range(1, len(parts) + 1))): length = length + len(part) if i < len(parts): length = (length//8 + 1) * 8 maxi = max(maxi, length) return maxi def _compute_cols_width(self): """Return an array with the width of each column If a specific width has been specified, exit. If the total of the columns width exceed the table desired width, another width will be computed to fit, and cells will be wrapped. """ if hasattr(self, "_width"): return maxi = [] if self._header: maxi = [ self._len_cell(x) for x in self._header ] for row in self._rows: for cell,i in zip(row, list(range(len(row)))): try: maxi[i] = max(maxi[i], self._len_cell(cell)) except (TypeError, IndexError): maxi.append(self._len_cell(cell)) ncols = len(maxi) content_width = sum(maxi) deco_width = 3*(ncols-1) + [0,4][self._has_border()] if self._max_width and (content_width + deco_width) > self._max_width: """ content too wide to fit the expected max_width let's recompute maximum cell width for each cell """ if self._max_width < (ncols + deco_width): raise ValueError('max_width too low to render data') available_width = self._max_width - deco_width newmaxi = [0] * ncols i = 0 while available_width > 0: if newmaxi[i] < maxi[i]: newmaxi[i] += 1 available_width -= 1 i = (i + 1) % ncols maxi = newmaxi self._width = maxi def _check_align(self): """Check if alignment has been specified, set default one if not """ if not hasattr(self, "_header_align"): self._header_align = ["c"] * self._row_size if not hasattr(self, "_align"): self._align = ["l"] * self._row_size if not hasattr(self, "_valign"): self._valign = ["t"] * self._row_size def _draw_line(self, line, isheader=False): """Draw a line Loop over a single cell length, over all the cells """ line = self._splitit(line, isheader) space = " " out = "" for i in range(len(line[0])): if self._has_border(): out += "%s " % self._char_vert length = 0 for cell, width, align in zip(line, self._width, self._align): length += 1 cell_line = cell[i] fill = width - len(cell_line) if isheader: align = self._header_align[length - 1] if align == "r": out += fill * space + cell_line elif align == "c": out += (int(fill/2) * space + cell_line \ + int(fill/2 + fill%2) * space) else: out += cell_line + fill * space if length < len(line): out += " %s " % [space, self._char_vert][self._has_vlines()] out += "%s\n" % ['', space + self._char_vert][self._has_border()] return out def _splitit(self, line, isheader): """Split each element of line to fit the column width Each element is turned into a list, result of the wrapping of the string to the desired width """ line_wrapped = [] for cell, width in zip(line, self._width): array = [] for c in cell.split('\n'): if c.strip() == "": array.append("") else: array.extend(textwrapper(c, width)) line_wrapped.append(array) max_cell_lines = reduce(max, list(map(len, line_wrapped))) for cell, valign in zip(line_wrapped, self._valign): if isheader: valign = "t" if valign == "m": missing = max_cell_lines - len(cell) cell[:0] = [""] * int(missing / 2) cell.extend([""] * int(missing / 2 + missing % 2)) elif valign == "b": cell[:0] = [""] * (max_cell_lines - len(cell)) else: cell.extend([""] * (max_cell_lines - len(cell))) return line_wrapped if __name__ == '__main__': table = Texttable() table.set_cols_align(["l", "r", "c"]) table.set_cols_valign(["t", "m", "b"]) table.add_rows([["Name", "Age", "Nickname"], ["Mr\nXavier\nHuon", 32, "Xav'"], ["Mr\nBaptiste\nClement", 1, "Baby"], ["Mme\nLouise\nBourgeau", 28, "Lou\n \nLoue"]]) print(table.draw() + "\n") table = Texttable() table.set_deco(Texttable.HEADER) table.set_cols_dtype(['t', # text 'f', # float (decimal) 'e', # float (exponent) 'i', # integer 'a']) # automatic table.set_cols_align(["l", "r", "r", "r", "l"]) table.add_rows([["text", "float", "exp", "int", "auto"], ["abcd", "67", 654, 89, 128.001], ["efghijk", 67.5434, .654, 89.6, 12800000000000000000000.00023], ["lmn", 5e-78, 5e-78, 89.4, .000000000000128], ["opqrstu", .023, 5e+78, 92., 12800000000000000000000]]) print(table.draw()) ================================================ FILE: bin/toaster_fixture.py ================================================ #!/usr/bin/env python3 # Copyright (C) 2017 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import argparse import json import os import sys import xml.etree.ElementTree as ET from xml.dom import minidom import re # # TODO # - Multiple Indexes # - Multiple bitbakes # - Multiple remotes # # Get the 'setup.py' default settings (BASE_LAYERS, DEFAULT_DISTRO, ...) import settings # Global variables top_layers=[] # list of top layers list_layers=[] # list of top and dependent layers json_dct={} # layer index json cache xmltree=None # default.xml object # Add 'layer index' type layer_branch records TYPE_LAYERINDEX = 1 # Insert custom settings starting at 100 CUSTOM_SETTINGS_BASE=100 # Toaster fixture file's relative path from install directory FIXTURE_FILE='layers/oe-core/bitbake/lib/toaster/orm/fixtures/custom.xml' # Do not include optional layers in default layer list INCLUDE_DEFAULT_LAYERS=False ############################################ ### formatted output def add_field(obj,attr_list,value): field = ET.SubElement(obj, "field") for attr in attr_list: field.set(attr[0], attr[1]) field.text = value def write_prolog(): global root root = ET.Element("django-objects") root.set('version', '1.0') def append_setting(name,value,pk): global root obj = ET.SubElement(root, "object") obj.set('model', 'orm.toastersetting') obj.set('pk', str(pk)) add_field(obj,[('type','CharField'),('name', 'name')],name) add_field(obj,[('type','CharField'),('name', 'value')],value) return pk+1 def append_bitbake(name,giturl,branch,pk): global root obj = ET.SubElement(root, "object") obj.set('model', 'orm.bitbakeversion') obj.set('pk', str(pk)) add_field(obj,[('type','CharField'),('name', 'name')],name) add_field(obj,[('type','CharField'),('name', 'giturl')],giturl) add_field(obj,[('type','CharField'),('name', 'branch')],branch) add_field(obj,[('type','CharField'),('name', 'dirpath')],'') return pk+1 def append_releases(name,desc,bitbake_version,branch,help): global root obj = ET.SubElement(root, "object") obj.set('model', 'orm.release') obj.set('pk', '1') add_field(obj,[('type','CharField'),('name', 'name')],name) add_field(obj,[('type','CharField'),('name', 'description')],desc) add_field(obj,[('rel','ManyToOneRel'),('to','orm.bitbakeversion'),('name', 'bitbake_version')],str(bitbake_version)) add_field(obj,[('type','CharField'),('name', 'branch_name')],branch) add_field(obj,[('type','TextField'),('name', 'helptext')],help) def write_default_layer_release(release,pk): global root for layer in list_layers: obj = ET.SubElement(root, "object") obj.set('model', 'orm.releasedefaultlayer') obj.set('pk', str(pk)) add_field(obj,[('rel','ManyToOneRel'),('to','orm.release'),('name', 'release')],str(release)) add_field(obj,[('type','CharField'),('name', 'layer_name')],layer) pk += 1 return pk def write_layer_release(layer_pk,layer_version_pk,layer_source): global root layers = json_dct["layerItems"] for layer_name in list_layers: for layer in layers: if layer_name == layer["name"]: break else: print("ERROR: Layer Name '%s' in not found" % layer_name) return obj = ET.SubElement(root, "object") obj.set('model', 'orm.layer') obj.set('pk', str(layer_pk)) add_field(obj,[('type','CharField'),('name', 'name')],layer['name']) add_field(obj,[('type','CharField'),('name', 'layer_index_url')],'') add_field(obj,[('type','CharField'),('name', 'vcs_url')],layer['vcs_url']) # for release in releases: layer_id=layer["id"] for release in range(1, 2): layerBranches = json_dct["layerBranches"] for layer_branch in layerBranches: if layer_id == layer_branch["layer"]: break else: print("ERROR: LayerId '%d' in layerBranches not found" % layer_id) return obj = ET.SubElement(root, "object") obj.set('model', 'orm.layer_version') obj.set('pk', str(layer_version_pk)) add_field(obj,[('rel','ManyToOneRel'),('to','orm.layer'),('name', 'layer')],str(layer_pk)) add_field(obj,[('type','IntegerField'),('name', 'layer_source')],str(layer_source)) add_field(obj,[('rel','ManyToOneRel'),('to','orm.release'),('name', 'release')],str(release)) add_field(obj,[('type','CharField'),('name', 'branch')],layer_branch['actual_branch']) add_field(obj,[('type','CharField'),('name', 'dirpath')],layer_branch['vcs_subdir']) layer_version_pk+=1 layer_pk+=1 return layer_pk,layer_version_pk def write_epilog(): parsed = minidom.parseString(ET.tostring(root, 'utf-8')) print(parsed.toprettyxml(indent=" "),file=output_fd) ############################################ ### worker functions def read_default_xml(xml_file): xmltree = ET.parse(xml_file) xmlroot = xmltree.getroot() remote_base=None remote_base_revision=None bitbake_branch=None bitbake_path=None for child in xmlroot: if 'remote' == child.tag and 'base' == child.attrib['name']: remote_base_fetch = child.attrib['fetch'] if 'default' == child.tag and 'base' == child.attrib['remote']: remote_base_revision = child.attrib['revision'] if 'default' == child.tag and 'base' == child.attrib['remote']: remote_base_revision = child.attrib['revision'] if 'project' == child.tag and child.attrib['name'].endswith('bitbake'): bitbake_branch = child.attrib['revision'] bitbake_path=child.attrib['name'] return remote_base_fetch,remote_base_revision,bitbake_branch,bitbake_path def read_layer_index_cache(json_cache): global json_dct with open(json_cache,"r") as json_data: json_dct = json.load(json_data) def find_layer2id(layer_name): layers = json_dct["layerItems"] for layer in layers: if layer_name == layer["name"]: return layer["id"] return None def find_id2layer(layer_id): layers = json_dct["layerItems"] for layer in layers: if layer_id == layer["id"]: return layer["name"] return None def find_layerBranch2layer(layerBranch_id): layers = json_dct["layerItems"] layerBranches = json_dct["layerBranches"] for layerBranch in layerBranches: if layerBranch_id == layerBranch["id"]: layer_id=layerBranch["layer"] for layer in layers: if layer_id == layer["id"]: return layer["name"] return None def find_layer2layerBranch(layer): layerBranches = json_dct["layerBranches"] layer_id=find_layer2id(layer) if None == layer_id: print("ERROR: Index for layer '%s' not found" % add_layer) return None for layerBranch in layerBranches: if layer_id == layerBranch["layer"]: layer_branch_id=layerBranch["id"] break else: print("ERROR: layerbranch layer '%d' not found" % layer_id) return None return layer_id,layer_branch_id def add_machine_layers(add_machine): global top_layers machines = json_dct["machines"] for machine in machines: if add_machine == machine["name"]: layerBranch_id = machine["layerbranch"] layer = find_layerBranch2layer(layerBranch_id) if None == layer: print("ERROR: Layer '%s' for machine '%s' not found" % (layer,add_machine)) else: top_layers.append(layer) def add_distro_layers(add_distro): global top_layers distros = json_dct["distros"] for distro in distros: if add_distro == distro["name"]: layerBranch_id = distro["layerbranch"] layer = find_layerBranch2layer(layerBranch_id) if None == layer: print("ERROR: Layer '%s' for distro '%s' not found" % (layer,add_distro)) else: top_layers.append(layer) def add_dependent_layers(add_layer,include_optional): global list_layers layers = json_dct["layerItems"] layerBranches = json_dct["layerBranches"] layerDependencies = json_dct["layerDependencies"] # do we already have this layer? if add_layer in list_layers: return # find layer ID layer_id,layer_branch_id = find_layer2layerBranch(add_layer) # find dependent layers for dep in layerDependencies: if not include_optional and not dep['required']: continue if layer_branch_id == dep['layerbranch']: dep_layer=find_id2layer(dep["dependency"]) if None == dep_layer: print("ERROR: Index to dep layer '%d' not found" % dep_id) else: add_dependent_layers(dep_layer,include_optional) # add layers depth first, top last list_layers.append(add_layer) ############################################ ### main() def main(argv): global top_layers global list_layers global output_fd global root parser = argparse.ArgumentParser(description='toaster_fixture.py: create Toaster fixture file from setup output') parser.add_argument('--project-dir', dest='project_dir',help='Project Directory') parser.add_argument('--verbose', '-v', action='store_true', dest='verbose',help='Verbose mode') args = parser.parse_args() # Core paths script_dir=os.path.dirname(os.path.abspath(argv[0])) wrlinux_dir=os.path.dirname(script_dir) if args.project_dir: install_dir = args.project_dir else: install_dir = os.getcwd() # Read setup default.xml data default_xml_file=os.path.join(install_dir,'default.xml') if not os.path.exists(default_xml_file): print("ERROR: 'default.xml' does not exist. You need to run the 'setup' program.") exit(-1) else: remote_base_fetch,remote_base_revision,bitbake_branch,bitbake_path=read_default_xml(default_xml_file) bitbake_url=os.path.join(remote_base_fetch,bitbake_path) # Load layer index cache json_cache=os.path.join(install_dir,settings.INDEXES[0]['CACHE']+'.json') read_layer_index_cache(json_cache) # Discover the XML directory xml_dir=os.path.join(wrlinux_dir,'data/xml') if os.path.exists(os.path.join(install_dir,'config','mirror-index','xml')): xml_dir=os.path.join(install_dir,'config','mirror-index','xml') # Prepare the output file output_fd=open(os.path.join(install_dir,FIXTURE_FILE), 'w') write_prolog() # Write Toaster environment hints # 1. Point Toaster to the wrlinux-x directory root.append(ET.Comment(' HINT:WRLINUX_DIR="%s" ' % wrlinux_dir)) # Write default setting overrides root.append(ET.Comment(' Set the project default values ')) append_setting('DEFCONF_DISTRO',settings.DEFAULT_DISTRO,1) append_setting('DEFAULT_RELEASE',remote_base_revision,2) append_setting('DEFCONF_MACHINE',settings.DEFAULT_MACHINE,4) # append custom settings setting_pk=CUSTOM_SETTINGS_BASE setting_pk=append_setting('DEFCONF_LINUX_KERNEL_TYPE','standard',setting_pk) setting_pk=append_setting('DEFAULT_KTYPE_LIST','standard preempt-rt tiny',setting_pk) setting_pk=append_setting('CUSTOM_LAYERINDEX_SERVER','file://'+json_cache,setting_pk) setting_pk=append_setting('SETUP_XMLDIR',xml_dir,setting_pk) setting_pk=append_setting('SETUP_GITURL',remote_base_fetch,setting_pk) setting_pk=append_setting('SETUP_PATH_FILTER','s|layers/[a-zA-Z0-9_\\-.]*||',setting_pk) # Write bitbake version root.append(ET.Comment(' Bitbake versions which correspond to the metadata release ')) bitbake_pk=1 bitbake_pk=append_bitbake(remote_base_revision,bitbake_url,bitbake_branch,bitbake_pk) # Write releases root.append(ET.Comment(' Releases available ')) append_releases(remote_base_revision,"Wind River Linux " + remote_base_revision,1,remote_base_revision, "Toaster will run your builds using the tip of the Wind River Linux '%s' branch." % remote_base_revision) # Write base default layers for layer in settings.BASE_LAYERS.split(): top_layers.append(layer) # Write DEFAULT_MACHINE layer add_machine_layers(settings.DEFAULT_MACHINE) # Write DEFAULT_DISTRO layer add_distro_layers(settings.DEFAULT_DISTRO) # Resolve dependent layers, exclude optional layers for layer in top_layers: add_dependent_layers(layer,INCLUDE_DEFAULT_LAYERS) # Write default layer list per release root.append(ET.Comment(' Default project layers for each release ')) default_layers_pk=1 default_layers_pk=write_default_layer_release(1,default_layers_pk) # Write layer list root.append(ET.Comment(' Default layers from wrlinux defaults ')) layer_pk,layer_version_pk = write_layer_release(1,1,TYPE_LAYERINDEX) write_epilog() output_fd.close() if args.verbose: print("Done:") print(" Layers=%d, LayerRelease=%d, LayerVersions=%d, Custom Settings=%d" % (len(list_layers),layer_pk,layer_version_pk,setting_pk-CUSTOM_SETTINGS_BASE)) print(" Default Layers=%s" % list_layers) if __name__ == '__main__': main(sys.argv) ================================================ FILE: bin/transform_index.py ================================================ #!/usr/bin/env python3 # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # This program allows you to transform layer index data from one source # to a specific output format. The output format can be in either restapi # or django format. It can be a single file, or split by layerbranch. # # You will need to adjust the items below to control the input/output # # OUTPUT - output directory to write file # # OUTPUT_FMT - restapi or django -- use Django for dataloads # # INDEXES - where to pull the data from # # REPLACES - what replacements to make on the -url- parts # # SPLIT - True or False, if True split the output # The following will let us load a layer index from one source, and make # it so we can load it into another for a stand-a-lone layerindex-Web # session. # # Set OUTPUT_FMT to 'django' # Set SPLIT to False # # Once configured, run the program then follow the steps below... # # To setup a new layerindex-Web session: # # git clone git://git.wrs.com/tools/layerindex-web # cd layerindex-web # virtualenv -p python3 venv # . ./venv/bin/activate # pip3 install -r requirements.txt # # (configure settings.py -- see README) # I recommend (adjust paths as necessary): # # USE_TZ = True # # DEBUG = True # # DATABASES = { # 'default': { # 'ENGINE': 'django.db.backends.sqlite3', # 'NAME': '/home/mhatle/git/layerIndex/wr9-db', # 'USER': '', # 'PASSWORD': '', # 'HOST': '', # 'PORT': '', # } # } # # SECRET_KEY = "" # # LAYER_FETCH_DIR = "/home/mhatle/git/layerIndex/wr9-layers" # # BITBAKE_REPO_URL = "git://git.wrs.com/bitbake" # # # python3 manage.py syncdb # # Answer yes to creating an admin user # python3 manage.py migrate # # cp .json layerindex/fixtures/. # python3 manage.py loaddata .json # # To start webserver # python3 manage.py runserver # This can also be used to take a database dump, either from the django DB # or from the RestAPI and split the pieces for individual entitlement # indexes. # # To get the input file, in your layerIndex run: # python3 manage.py dumpdata > /tmp/input.json # # By default the output will be organized by layerbranch in /tmp/output # # Split the results by base/bsp/addon # # example: # (cd /tmp/transform ; cp `grep layer_type * | grep -v \"B\" | cut -f 1 -d :` /home/mhatle/git/lpd/wrlinux-x/data/index/base/.) # (cd /tmp/transform ; cp `grep layer_type * | grep \"B\" | cut -f 1 -d :` /home/mhatle/git/lpd/wrlinux-x/data/index/bsps/.) # # We load from git.wrs.com and transform to msp-git.wrs.com # Adjusting both the git URL and the webgit URL REPLACE = [ ( 'git://git.wrs.com/', '#BASE_URL#' ), ( 'http://git.wrs.com/cgit/', '#BASE_WEB#' ), ( '#BASE_URL#', 'git://msp-git.wrs.com/' ), ( '#BASE_WEB#', 'http://msp-git.wrs.com/cgi-bin/cgit.cgi/' ), ] # Note the branch can be hard coded. This is required only when you want # to limit the branch from a restapi-web import. (This does not do anything # on other input formats.) INDEXES = [ { 'DESCRIPTION' : 'Wind River Developer Layer Index', 'TYPE' : 'restapi-web', 'URL' : 'http://layers.wrs.com/layerindex/api/', 'CACHE' : None, 'BRANCH' : 'master-wr', }, ] OUTPUT = '/tmp/transform' OUTPUT_FMT = 'django' #OUTPUT_FMT = 'restapi' SPLIT = False import os import sys from layer_index import Layer_Index index = Layer_Index(INDEXES, base_branch=None, replace=REPLACE) for lindex in index.index: print('Dump %s as %s (split=%s)...' % (lindex['CFG']['DESCRIPTION'], OUTPUT_FMT, SPLIT)) os.makedirs(OUTPUT, exist_ok=True) if OUTPUT_FMT == 'django': index.serialize_django_export(lindex, OUTPUT + '/' + lindex['CFG']['DESCRIPTION'], split=SPLIT) elif OUTPUT_FMT == 'restapi': index.serialize_index(lindex, OUTPUT + '/' + lindex['CFG']['DESCRIPTION'], split=SPLIT) else: print('Unknown output format!') ================================================ FILE: bin/utils_setup.py ================================================ #!/usr/bin/env python3 # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import os import sys import subprocess # Setup-specific modules import logger_setup logger = logger_setup.setup_logging() def run_cmd(cmd, environment=None, cwd=None, log=1, expected_ret=0, err=b'GitError', err2=b'error', err3=b'fatal', stderr=None, stdout=None): err_msg = [] logger.debug('Running cmd: "%s"' % repr(cmd)) if cwd: logger.debug('From %s' % cwd) # log 0 - output goes to stdout/stderr, not logged # log 1 - send output to plain # log 2 - send output to debug if log == 1 or log == 2: if stderr is None: stderr = subprocess.STDOUT ret = subprocess.Popen(cmd, env=environment, cwd=cwd, stderr=stderr, stdout=subprocess.PIPE) while True: output = ret.stdout.readline() if not output and ret.poll() is not None: break if output: output = output.strip() if len(err_msg) > 0 or output.startswith(err) or output.startswith(err2) or output.startswith(err3): err_msg.append("%s" % output.decode('utf-8')) if log == 1: logger.plain("%s" % output.decode('utf-8')) elif log == 2: logger.debug("%s" % output.decode('utf-8')) else: logger.debug('output not logged for this command (%s) without verbose flag (-v).' % (cmd)) ret = subprocess.Popen(cmd, env=environment, cwd=cwd, close_fds=True, stderr=stderr, stdout=stdout) ret.wait() if ret.returncode != expected_ret: if stderr != subprocess.DEVNULL: if environment: for key in environment.keys(): logger.to_file('%20s = %s' % (key, repr(environment[key]))) if log != 2: logger.critical('cmd "%s" returned %d' % (cmd, ret.returncode)) else: logger.debug('cmd "%s" returned %d' % (cmd, ret.returncode)) msg = '' if log: if cwd: msg += cwd + ': ' msg += " ".join(cmd) + '\n' msg += '\n'.join(err_msg) msg += '\n' raise Exception(msg) logger.debug('Finished running cmd: "%s"' % repr(cmd)) def query_input(question, interactive): client = os.environ.get('GIT_ASKPASS', None) if not client: client = os.environ.get('SSH_ASKPASS', None) if not client: if interactive: client = "[internal]" else: raise Exception('Unable to get authentication via ASKPASS.') cmd = [ client, question ] logger.debug("cmd: %s " % (cmd)) if cmd[0] == "[internal]": import getpass retval = getpass.getpass(cmd[1]) else: environ = os.environ.copy() # We do NOT want to inherit python home from the environment # See Issue: LIN1018-2934 # python3 wrapper from the buildtools sets this, which causes host # python tools to fail if 'PYTHONHOME' in environ: del environ['PYTHONHOME'] ret = subprocess.Popen(cmd, env=environ, close_fds=True, stdout=subprocess.PIPE) retval = "" while True: lin = ret.stdout.readline() if not lin and ret.poll() is not None: break retval += lin.decode('utf-8') ret.wait() if ret.returncode != 0: raise Exception('return code != 0 from %s.' % cmd) retval = retval.rstrip('\n') return retval def fetch_url(url=None, auth=False, debuglevel=0, interactive=0): assert url is not None import urllib from urllib.request import urlopen, Request from urllib.parse import urlparse if auth: logger.debug("Configuring authentication for %s..." % url) up = urlparse(url) uname = query_input("Username for '%s://%s': " % (up.scheme, up.netloc), interactive) passwd = query_input("Password for '%s://%s@%s': " % (up.scheme, uname, up.netloc), interactive) # This is a security leak, as the username/password could be logged. # Only enable this during development. #logger.debug("%s: u:'%s' p:'%s'" % ( url, uname, passwd )) password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() password_mgr.add_password(None, "%s://%s" % (up.scheme, up.netloc), uname, passwd) handler = urllib.request.HTTPBasicAuthHandler(password_mgr) opener = urllib.request.build_opener(handler, urllib.request.HTTPSHandler(debuglevel=debuglevel)) else: opener = urllib.request.build_opener(urllib.request.HTTPSHandler(debuglevel=debuglevel)) urllib.request.install_opener(opener) logger.debug("Fetching %s (%s)..." % (url, ["without authentication", "with authentication"][auth])) try: res = urlopen(Request(url, headers={'User-Agent': 'Mozilla/5.0 (Wind River Linux/setup.sh)'}, unverifiable=True)) except urllib.error.HTTPError as e: logger.debug("HTTP Error: %s: %s" % (e.code, e.reason)) logger.debug(" Requested: %s" % (url)) logger.debug(" Actual: %s" % (e.geturl())) if auth: logger.debug(" Authentication enabled. Using username '%s'." % uname) if not auth and e.code == 401: logger.debug("Retrying with authentication...") res = fetch_url(url, auth=True, debuglevel=debuglevel, interactive=interactive) logger.debug("...retrying with authentication successful, continuing.") elif e.code == 404: logger.debug("Request not found.") raise e else: logger.debug("Headers:\n%s" % (e.headers)) raise e except OSError as e: error = 0 reason = "" # Process base OSError first... if hasattr(e, 'errno'): error = e.errno reason = e.strerror # Process gaierror (socket error) subclass if available. if hasattr(e, 'reason') and hasattr(e.reason, 'errno') and hasattr(e.reason, 'strerror'): error = e.reason.errno reason = e.reason.strerror if error == -2: raise e if error and error != 0: logger.critical("Unable to fetch %s due to exception: [Error %s] %s" % (url, error, reason)) else: logger.critical("Unable to fetch %s due to OSError exception: %s" % (url, e)) sys.exit(1) except Exception as e: logger.critical('Unable to fetch entitlement: %s (%s)' % (type(e), e)) sys.exit(1) finally: logger.debug("...fetching %s (%s), done." % (url, ["without authentication", "with authentication"][auth])) return res ================================================ FILE: bin/windshare.py ================================================ # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # The windshare distribution mechanism has unique requirements for dividing # the components into specific entitled sections. We need to take these # 'folders' and reconstruct the related items (mirror-index, xml files, etc) # into a flat view that works like a regular mirror would. import json import xml.etree.ElementTree as ET import os import sys import logger_setup import utils_setup logger = logger_setup.setup_logging() class Windshare(): def __init__(self, debug=0): self.folders = None self.indexes = {} self.xmls = {} self.debug = debug # This is only used if we want to instruct the system to ask the user # for credentials, if a better credential manager is not available. self.interactive = 0 def get_windshare_urls(self, base_url): from urllib.parse import urlsplit, urlunsplit (uscheme, uloc, upath, uquery, ufragid) = urlsplit(base_url) # What folder are we in? ws_base_folder = os.path.basename(upath) logger.debug('Product base folder = %s' % ws_base_folder) if not ws_base_folder or ws_base_folder == "": # Invalid URL logger.debug('Invalid base folder, not Windshare.') return (None, None, None) # Folder root is one directory higher then the base_url upath = os.path.dirname(upath) ws_base_url = urlunsplit((uscheme, uloc, upath, uquery, ufragid)) if uscheme and (uscheme != "http" and uscheme != "https"): logger.debug('Scheme (%s) not valid for Windshare.' % uscheme) return (None, None, None) # Magic URL to the entitlement file ws_entitlement_url = ws_base_url + '/wrlinux-lts.21.json' logger.debug('Entitlement url %s' % ws_entitlement_url) # If no uscheme, this is file access, check here if an entitlement # file exists. If not, we know we're not windshare. if not uscheme and not os.path.exists(ws_entitlement_url): logger.debug('Local file path, file does not exist. Not a Windshare install.') return (None, None, None) return (ws_base_url, ws_base_folder, ws_entitlement_url) def load_folders(self, url=None): assert url is not None def _get_json_response(wsurl=None, retry=True): assert wsurl is not None from urllib.parse import urlparse up = urlparse(wsurl) if not up.scheme: # Check for it on the disk... if os.path.exists(wsurl): parsed = json.load(open(wsurl, 'rt', encoding='utf-8')) else: return None else: # Go out to the network... from urllib.request import URLError try: res = utils_setup.fetch_url(wsurl, debuglevel=self.debug, interactive=self.interactive) except URLError as e: if 'windshare' in up.netloc: # Authentication failure, we need to stop now. if hasattr(e, 'code') and e.code == 401: logger.critical('Unable to authenticate: %s' % wsurl) else: logger.critical('Unable to contact Wind Share: %s: %s' % (wsurl, e.reason)) logger.critical("Check your credentials, network and proxy settings.") sys.exit(1) raise e try: result = res.read().decode('utf-8') except ConnectionResetError: if retry: logger.debug("%s: Connection reset by peer. Retrying..." % wsurl) result = _get_json_response(wsurl=wsurl, retry=False) logger.debug("%s: retry successful." % wsurl) else: logger.critical("%s: Connection reset by peer." % wsurl) logger.critical("Is there a firewall blocking your connection?") sys.exit(1) logger.debug('Result:\n%s' % result) parsed = json.loads(result) return parsed try: entitlement = _get_json_response(url) if entitlement and 'dataFolderTrueFolders' in entitlement: self.folders = entitlement['dataFolderTrueFolders'] else: return False except Exception as e: logger.debug('Unable to fetch entitlement: %s (%s)' % (type(e), e)) return False return True # Note base_url is _NOT_ setup.base_url, it is the root of the folders dir def load_mirror_index(self, setup, base_url, folder): mirror_index_path = setup.load_mirror_index(base_url + '/' + folder + '/mirror-index', folder=folder + "_") if not mirror_index_path: raise Exception("Unable to load mirror index %s." % (base_url + '/' + folder + '/mirror-index')) # Mirror index returns with the fetched item checked out... #cmd = [setup.tools['git'], 'checkout', folder + "_" + setup.base_branch ] #utils_setup.run_cmd(cmd, environment=setup.env, cwd=mirror_index_path) for (dirpath, _, filenames) in os.walk(mirror_index_path): if dirpath.endswith('/.git') or '/.git/' in dirpath: continue for filename in filenames: if filename == 'README': continue if filename.endswith('.json'): try: (_, _, jlayer) = filename[:-5].split('__') except: raise Exception('Unable to parse windshare json file %s (%s).' % (filename, folder + "_" + setup.base_branch)) path = os.path.join(dirpath, filename) pindex = json.load(open(path, 'rt', encoding='utf-8')) if 'layerItems' in pindex: newItems = [] for entry in pindex['layerItems']: # Verify this is the jlayer, otherwise remove it as it won't be in this folder! if entry['name'] != jlayer: continue entry['vcs_url'] = entry['vcs_url'].replace('#BASE_URL#', '#BASE_URL#' + '/' + folder) newItems.append(entry) pindex['layerItems'] = newItems self.indexes[filename] = pindex elif filename.endswith('.xml') or filename.endswith('.inc'): self.xmls[filename] = [] path = os.path.join(dirpath, filename) # Prefix the &2 echo "git-repo, which is used by the setup program, does not permit a nested" >&2 echo "repository structure. You must run setup in a different location or" >&2 echo "remove the directory $path/.repo" >&2 exit 1 fi done } ================================================ FILE: data/environment.d/00_wrl_eula.sh ================================================ # Copyright (C) 2016 - 2020 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Verify EULA acceptance # We don't 'keep' this command & argument as acceptance is stored setup_add_arg --accept-eula ACCEPT_EULA setup_add_func eula_setup eula_askuser() { accept=${ACCEPT_EULA} # Check whether it is set or not. if [ -n "${ACCEPT_EULA+x}" ]; then if [ "${accept}" != "yes" -a "${accept}" != "no" ]; then echo "error: argument --accept-eula: invalid choice: '${accept}' \ (choose from 'yes', 'no')" >&2 exit 1 fi if [ "${accept}" = "no" ]; then echo "You must agree to the EULA to continue." >&2 exit 1 fi fi while [ "${accept}" != "yes" ] ; do echo echo "The End User License Agreement is available at:" echo " ${BASEDIR}/EULA" echo read -p "I have read the EULA and accept it - yes/no/read " accept case ${accept} in [yY][eE][sS]) accept="yes" ;; [nN][oO]) echo "You must agree to the EULA to continue." >&2 exit 1 ;; [rR] | [rR][eE][aA][dD]) # Prefer 'less' if we have it, otherwise fall back to more if which less >/dev/null 2>&1 ; then cat ${BASEDIR}/EULA | less -P"Type 'q' when done." else cat ${BASEDIR}/EULA | more fi ;; *) echo "Only yes, no and read are accepted." >&2 ;; esac done echo mkdir -p config # Log the EULA acceptance if there is any question in the future... { echo "#End User License Agreement Accepted" echo "EULA_DATE=\"$(date)\"" echo "EULA_USER=\"$(whoami)@$(hostname)\"" echo "EULA_VERSION=\"$(tail -n 1 ${BASEDIR}/EULA)\"" echo "EULA_SHA=\"$(shasum ${BASEDIR}/EULA | cut -d ' ' -f 1)\"" echo } >> config/eula_accepted } eula_setup() { if [ -e config/eula_accepted ]; then . ./config/eula_accepted fi NEW_SHA=$(shasum ${BASEDIR}/EULA | cut -d ' ' -f 1) if [ -n "${EULA_SHA}" -a "${NEW_SHA}" != "${EULA_SHA}" ]; then echo "The End User User License has changed since you last agreed to it." EULA_SHA="" fi if [ -z "${EULA_SHA}" -o -z "${EULA_VERSION}" -o -z "${EULA_DATE}" -o -n "${ACCEPT_EULA+x}" ]; then eula_askuser else echo "End User License Agreement '${EULA_VERSION}' agreed to by ${EULA_USER} on ${EULA_DATE}." fi } ================================================ FILE: data/environment.d/01_wrl_buildtools.sh ================================================ # Copyright (C) 2016, 2020 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Check if the buildtools are available, if so load it early... BUILDTOOLS="${BUILDTOOLS:-bin/buildtools}" setup_add_func buildtools_pre_setup unset -v BUILDTOOLS_LOADED buildtools_pre_setup() { ENVIRON=$(find -L ${BUILDTOOLS} -name "environment-setup-${SDKARCH}-*-linux" 2>/dev/null | head -n1) if [ -z "${ENVIRON}" ]; then # Nothing there yet... return 0 fi . "${ENVIRON}" if [ $? -ne 0 ]; then # Something went wrong.. ignore it return 0 fi BUILDTOOLS_LOADED=1 return 0 } ================================================ FILE: data/environment.d/02_wrl_anspass.sh ================================================ # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Check if we have access to anspassd, if so try to setup anspass. # Quick walk through args to look for --no-anspass # We do it here because we need to decide right now if the anspass # needs to be started/stopped args="$@" # Skip anspass if --no-anspass OR the BASEURL starts with ssh:// # for ssh:// we don't want to store these values in anspass, as it can not # understand the format of the question if [ "${args/--no-anspass//}" != "${args}" -o "${BASEURL##ssh://}" != "${BASEURL}" ] ; then export NO_ANSPASS=1 fi if [ "$NO_ANSPASS" = "" ] ; then setup_add_func anspass_setup setup_shutdown_func anspass_early_shutdown . ${BASEDIR}/data/environment.d/setup_anspass fi # anspass_setup defined in setup_anspass # This isn't really a shutdown, but a transfer. Before we stop 'askpass', we # need to transfer any credential into anspass. By this point anspass # should be ready for a transfer. # # If anspass isn't running yet, we have to start it, so we can transfer the # credentials. But it should be shutdown right after. # # anspass_start defined in setup_anspass anspass_early_shutdown() { # Before shutting down, try to transfer askpass items to anspass if [ -n "${WRL_ASKPASS_SOCKET}" ]; then if [ -z "$(${BASEDIR}/data/environment.d/setup_askpass --dump)" ]; then return 0 fi # If anspass is not running, we start it so we can transfer # credentials for storage if [ -z "${ANSPASS_PATH}" -o -z "${ANSPASS_TOKEN}" ]; then anspass_start fi echo "Storing credentials into anspass." ${BASEDIR}/data/environment.d/setup_askpass --dump | anspass_transfer anspass_stop fi return 0 } ================================================ FILE: data/environment.d/03_wrl_askpass.sh ================================================ # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # This needs to be run BEFORE we run any password checks... setup_add_arg --user WINDSHARE_USER setup_add_arg --password WINDSHARE_PASS setup_add_func askpass_setup setup_shutdown_func askpass_shutdown askpass_setup() { # These values may also be used by anspass WINDSHARE_SCHEME=$(echo ${BASEURL} | sed 's,\([^:/]*\).*,\1,') if [ ${WINDSHARE_SCHEME} ]; then WINDSHARE_HOST=$(echo ${BASEURL} | sed 's,\([^:/]*\)://\([^/]*\).*,\2,') fi # If anspass is already enabled, use it instead! if [ -n "${ANSPASS_TOKEN}" ]; then return 0 fi if [ "$WINDSHARE_SCHEME" = "http" ] || [ "$WINDSHARE_SCHEME" = "https" ]; then if [ -n "$GIT_ASKPASS" ]; then echo "INFO: Detected GIT_ASKPASS configuration. Disabling built-in askpass functionality." return 0 fi fi export WRL_ASKPASS_SOCKET=${PWD}/bin/.setup_askpass # Cleanup any old instances ${BASEDIR}/data/environment.d/setup_askpass --quit >/dev/null 2>&1 rm -f ${WRL_ASKPASS_SOCKET} mkdir -p $(dirname ${WRL_ASKPASS_SOCKET}) ${BASEDIR}/data/environment.d/setup_askpass --server & askpass_pid=$! askpass_jid=%% while [ ! -e ${WRL_ASKPASS_SOCKET} ]; do if ! jobs $askpass_jid >/dev/null 2>&1 ; then echo "Unable to start the askpass server." >&2 return 1 fi # We have to give it time to start... sleep 1 done # We need to tell it what tty to use for questions... echo $(tty) | ${BASEDIR}/data/environment.d/setup_askpass --set "tty" > /dev/null 2> /dev/null # If user/pass passed in (can't do this for a file path) if [ -n "${WINDSHARE_HOST}" ]; then if [ -n "${WINDSHARE_USER}" ]; then echo "${WINDSHARE_USER}" | ${BASEDIR}/data/environment.d/setup_askpass --set "Username for '${WINDSHARE_SCHEME}://${WINDSHARE_HOST}': " > /dev/null echo "${WINDSHARE_PASS}" | ${BASEDIR}/data/environment.d/setup_askpass --set "Password for '${WINDSHARE_SCHEME}://${WINDSHARE_USER}@${WINDSHARE_HOST}': " "${WINDSHARE_PASS}" > /dev/null fi if [ ${WINDSHARE_SCHEME} = "ssh" -a -n "${WINDSHARE_PASS}" ]; then echo "${WINDSHARE_PASS}" | ${BASEDIR}/data/environment.d/setup_askpass --set "${WINDSHARE_HOST}'s password: " > /dev/null fi fi export GIT_SSH=${BASEDIR}/data/environment.d/setup_ssh export GIT_ASKPASS=${BASEDIR}/data/environment.d/setup_askpass export SSH_ASKPASS=${BASEDIR}/data/environment.d/setup_askpass return 0 } askpass_shutdown() { if [ -n "${WRL_ASKPASS_SOCKET}" ]; then ${BASEDIR}/data/environment.d/setup_askpass --quit rm -f ${WRL_ASKPASS_SOCKET} unset WRL_ASKPASS_SOCKET fi } ================================================ FILE: data/environment.d/04_wrl_buildtools.sh ================================================ # Copyright (C) 2016-2021 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Download, install and load the buildtools tarball (as needed) BUILDTOOLS_VERSION=${BUILDTOOLS_VERSION:-10.21.20.4} BUILDTOOLS_EXT_VERSION=${BUILDTOOLS_EXT_VERSION:-10.21.01b.0} # Special windshare folders to search BUILDTOOLS_FOLDERS="WRLinux-lts-21-Core WRLinux-lts-21-Base" # Where to install the build tools BUILDTOOLS="${BUILDTOOLS:-bin/buildtools}" # Arch of the SDK to load SDKARCH=${SDKARCH:-$(uname -m)} setup_add_arg --buildtools-branch BUILDTOOLSBRANCH keep setup_add_arg --buildtools-type BUILDTOOLS_TYPE keep setup_add_func buildtools_setup setup_export_func buildtools_export . ${BASEDIR}/data/environment.d/setup_utils buildtools_setup() { if [ -z "${BUILDTOOLSBRANCH}" ]; then BUILDTOOLSBRANCH="${BASEBRANCH}" fi # According to https://gcc.gnu.org/projects/cxx-status.html#cxx14 # gcc 6.0.0 is the minimal version fully support c++14 gcc_cur_ver=$(gcc -dumpfullversion -dumpversion 2>/dev/null) required_ver=6.0.0 # check whether host gcc version less than $required_ver if [ ! "$(printf '%s\n' "$required_ver" "$gcc_cur_ver" | sort -V | head -n1)" = "$required_ver" ]; then if [ basic = "${BUILDTOOLS_TYPE}" ]; then echo -e "\nWarning: The version of host gcc is too low to support c++14 standard. It may cause some packages such as doxygen-native fail to build with basic buildtools.\n" fi if [ -z "${BUILDTOOLS_TYPE}" ]; then BUILDTOOLS_TYPE=extended fi fi if [ -z "${BUILDTOOLS_TYPE}" ]; then BUILDTOOLS_TYPE=basic fi if [ basic != "${BUILDTOOLS_TYPE}" ] && [ extended != "${BUILDTOOLS_TYPE}" ]; then echo "Wrong argument \"${BUILDTOOLS_TYPE}\" for option --buildtools-type. Supported arguments: basic, extended." >&2 return 1 fi if [ basic = "${BUILDTOOLS_TYPE}" ]; then buildtools=buildtools buildtools_version=$BUILDTOOLS_VERSION else buildtools=buildtools-extended buildtools_version=$BUILDTOOLS_EXT_VERSION fi # Buildtools location can change -- this is the path on top of the BASEURL BUILDTOOLS_REMOTE="${BUILDTOOLS_REMOTE:-${buildtools}-standalone-${buildtools_version}}" # Where to cache the git fetch BUILDTOOLS_GIT="${BUILDTOOLS_GIT:-bin/${buildtools}.git}" FETCH_BUILDTOOLS=0 # Install them into the project directory EXTRACT_BUILDTOOLS=0 BUILDTOOLS_REF=$(echo ${BUILDTOOLS_REMOTE} | sed -e 's,.*/buildtools-standalone-,,') if [ ! -d "${BUILDTOOLS_GIT}" ]; then FETCH_BUILDTOOLS=1 (mkdir -p ${BUILDTOOLS_GIT} && git init ${BUILDTOOLS_GIT}) if [ $? -ne 0 ]; then echo "Unable to create ${BUILDTOOLS_GIT} directory." >&2 return 1 fi else # Did the buildtools URL change? LASTREF=$(git config -f ${BUILDTOOLS_GIT}/.git/config local.last.ref) if [ "${LASTREF}" != "${BUILDTOOLS_REF}" ]; then FETCH_BUILDTOOLS=1 fi fi if [ ${FETCH_BUILDTOOLS} -ne 1 ]; then # We need this in order to have the right path for subsequent mirror operations BUILDTOOLS_REMOTE=$(git config -f ${BUILDTOOLS_GIT}/.git/config local.${BUILDTOOLS_REF}.path) else echo "Searching for ${BUILDTOOLS_REMOTE}..." retries=0 duration=5 for i in {1..5} ; do if ! setup_check_url "${BASEURL}/${BUILDTOOLS_REMOTE}" ; then ORIG_BT_REMOTE=${BUILDTOOLS_REMOTE} # Additional places to search... for folder in ${BUILDTOOLS_FOLDERS} layers/buildtools; do NEW_REMOTE=${folder}/${BUILDTOOLS_REMOTE} if setup_check_url "${BASEURL}/${NEW_REMOTE}" ; then BUILDTOOLS_REMOTE=${NEW_REMOTE} fi done if [ "${BUILDTOOLS_REMOTE}" = "${ORIG_BT_REMOTE}" ]; then retries=$(($retries+1)) echo "Retrying $1 after $duration seconds -- $retries time(s) (max: 5)" sleep $duration duration=$(($duration+$(random 1 5))) fi else break fi done if [ $retries -eq 5 ]; then echo "Unable to find ${BUILDTOOLS_REMOTE}. Search path:">&2 for folder in ${BUILDTOOLS_FOLDERS} layers/buildtools; do echo " ${BASEURL}/${folder}/${BUILDTOOLS_REMOTE}" >&2 done return 1 fi echo "Fetching buildtools.." # Check if it's a tag if [ "$BASEBRANCH" != "${BASEBRANCH##refs/tags/}" ]; then local_name="${BUILDTOOLSBRANCH}:tags/${BUILDTOOLS_REF}" else local_name="${BUILDTOOLSBRANCH}:${BUILDTOOLS_REF}" fi trap : INT retries=0 duration=5 ret=0 for i in {1..5} ; do echo "${BASEURL}/${BUILDTOOLS_REMOTE}" (cd ${BUILDTOOLS_GIT} && git fetch -f -n -u "${BASEURL}/${BUILDTOOLS_REMOTE}" $local_name) ret=$? if [ $ret -eq 0 ] || [ $ret -eq 130 ]; then break else retries=$(($retries+1)) echo "Retrying $1 after $duration seconds -- $retries time(s) (max: 5)" sleep $duration duration=$(($duration+$(random 1 5))) fi done if [ $retries -eq 5 ] || [ $ret -eq 130 ]; then echo "Error fetching buildtools repository ${BASEURL}/${BUILDTOOLS_REMOTE}" >&2 return 1 fi trap - INT # Set a flag so we know where the fetch was from... ( cd ${BUILDTOOLS_GIT} git config "local.${BUILDTOOLS_REF}.url" "${BASEURL}/${BUILDTOOLS_REMOTE}" git config "local.${BUILDTOOLS_REF}.path" "${BUILDTOOLS_REMOTE}" git config local.last.ref "${BUILDTOOLS_REF}" git checkout "${BUILDTOOLS_REF}" ) if [ $? -ne 0 ]; then echo "Unable to checkout branch ${BUILDTOOLS_REF}." >&2 return 1 fi echo "Done" EXTRACT_BUILDTOOLS=1 fi if [ ! -d "${BUILDTOOLS}.${BUILDTOOLS_REF}" ]; then EXTRACT_BUILDTOOLS=1 fi if [ ${EXTRACT_BUILDTOOLS} -ne 1 ]; then ENVIRON=$(find -L ${BUILDTOOLS} -name "environment-setup-${SDKARCH}-*-linux" | head -n1) if [ -z "${ENVIRON}" ]; then # Something is wrong, try to fix it! EXTRACT_BUILDTOOLS=1 fi fi if [ ${EXTRACT_BUILDTOOLS} -eq 1 ]; then # Needs python. buildtoolssdk=$(find "${BUILDTOOLS_GIT}" -name "${SDKARCH}-${buildtools}-nativesdk-standalone-*.sh" 2>/dev/null | sort | head -n1) buildtoolssdk_list=$(find "${BUILDTOOLS_GIT}" -name "${SDKARCH}-${buildtools}-nativesdk-standalone-*.list" 2>/dev/null | sort | head -n1) if [ -z "${buildtoolssdk_list}" ]; then if [ -z "${buildtoolssdk}" ]; then echo "Unable to find buildtools-nativesdk-standalone archive for ${SDKARCH}." >&2 echo >&2 echo "SDKARCH values found:" >&2 echo $(find "${BUILDTOOLS_GIT}" -name "*-${buildtools}-nativesdk-standalone-*.sh" | xargs -n 1 basename | cut -d '-' -f 1) >&2 echo >&2 echo "If one of these is compatible, set SDKARCH in your environment." >&2 echo >&2 return 1 fi else buildtoolssdk=${buildtoolssdk_list/%.list/.sh} rm -f ${buildtoolssdk} for part in $(cat ${buildtoolssdk_list}); do cat ${BUILDTOOLS_GIT}/${part} >>${buildtoolssdk} done chmod +x ${buildtoolssdk} fi echo "Installing buildtools.." if [ -d "${BUILDTOOLS}.${BUILDTOOLS_REF}" ]; then rm -rf "${BUILDTOOLS}.${BUILDTOOLS_REF}" fi trap : INT ${buildtoolssdk} -d "${BUILDTOOLS}.${BUILDTOOLS_REF}" -y if [ $? -ne 0 ]; then echo >&2 echo "Error installing the buildtools-nativesdk-standalone archive: ${buildtoolssdk}" >&2 # We try to cleanup, but an over zealous (sigint) user can stop the rm as well. rm -rf ${BUILDTOOLS}.${BUILDTOOLS_REF} return 1 fi trap - INT echo "Done" fi # force to re-create the link that buildtools type may change rm -f ${BUILDTOOLS} ln -s $(basename ${BUILDTOOLS}).${BUILDTOOLS_REF} ${BUILDTOOLS} unset FETCH_BUILDTOOLS EXTRACT_BUILDTOOLS ENVIRON=$(find -L ${BUILDTOOLS} -name "environment-setup-${SDKARCH}-*-linux" | head -n1) if [ -z "${ENVIRON}" ]; then echo "Error unable to load buildtools environment-setup file." >&2 return 1 fi . "${ENVIRON}" if [ $? -ne 0 ]; then echo "Unable to load the buildtools environment setup file." >&2 return 1 fi BUILDTOOLS_LOADED=1 return 0 } buildtools_export() { if [ -z "${BUILDTOOLSBRANCH}" ]; then BUILDTOOLSBRANCH="${BASEBRANCH}" fi export OE_BUILDTOOLS_BRANCH=${BUILDTOOLSBRANCH} export OE_BUILDTOOLS_REMOTE=${BUILDTOOLS_REMOTE} if [ basic = "${BUILDTOOLS_TYPE}" ]; then export OE_ANOTHER_BUILDTOOLS_REMOTE=`echo ${BUILDTOOLS_REMOTE} | sed -e "s,buildtools-standalone-,buildtools-extended-standalone-," \ | sed -e "s,${BUILDTOOLS_VERSION},${BUILDTOOLS_EXT_VERSION},"` else export OE_ANOTHER_BUILDTOOLS_REMOTE=`echo ${BUILDTOOLS_REMOTE} | sed -e "s,buildtools-extended-standalone-,buildtools-standalone-," \ | sed -e "s,${BUILDTOOLS_EXT_VERSION},${BUILDTOOLS_VERSION},"` fi return 0 } ================================================ FILE: data/environment.d/05_wrl_anspass.sh ================================================ # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Check if we have access to anspassd, if so try to setup anspass. if [ "$NO_ANSPASS" = "" ] ; then setup_add_func anspass_late_setup setup_shutdown_func anspass_shutdown . ${BASEDIR}/data/environment.d/setup_anspass fi # anspass_setup defined in setup_anspass anspass_late_setup() { # If we've already started anspass, skip this if [ -n "${ANSPASS_PATH}" -a -n "${ANSPASS_TOKEN}" ]; then return 0 fi # We want to use askpass (if enabled), no reason to invoke # anspass yet.... unless the user passed in a user/pass # meaning they want an offline install, no Q's asked. if [ -n "${WINDSHARE_USER}" ]; then anspass_setup return $? fi return 0 } # anspass_stop defined in setup_anspass anspass_shutdown() { anspass_stop } ================================================ FILE: data/environment.d/06_wrl_repo.sh ================================================ # Copyright (C) 2016 - 2021 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Identify the right path for the Wind River version of git-repo # setup the REPO_URL to point there... setup_add_arg --repo-url REPO_URL setup_add_arg --repo-branch REPO_REV setup_add_func wr_repo_setup setup_add_func wr_repo_clone # Special windshare folders to search REPO_FOLDERS="WRLinux-lts-21-Core WRLinux-lts-21-Base" . ${BASEDIR}/data/environment.d/setup_utils wr_repo_find() { echo "Searching for git-repo..." REPO_URL=${BASEURL}/git-repo retries=0 duration=5 for i in {1..5} ; do if ! setup_check_url "${REPO_URL}" ; then # Clear in case there are no REPO_FOLDERS REPO_URL="" for folder in ${REPO_FOLDERS} ; do REPO_URL=${BASEURL}/${folder}/git-repo if ! setup_check_url "${REPO_URL}" ; then REPO_URL="" continue fi break done if [ "${REPO_URL}" = "" ]; then retries=$(($retries+1)) echo "Retrying $1 after $duration seconds -- $retries time(s) (max: 5)" sleep $duration duration=$(($duration+$(random 1 5))) fi else break fi done if [ $retries -eq 5 ]; then echo "Unable to find git-repo repository. Search path:" >&2 echo " ${BASEURL}/git-repo" >&2 for folder in ${REPO_FOLDERS} ; do echo " ${BASEURL}/${folder}/git-repo" >&2 done return 1 fi return 0 } repo_branch_fallback="WRLINUX_10_21_BASE" wr_repo_setup() { local update_url update_url=true # If the user passed it in, we use it after we verify it! if [ -n "$REPO_URL" ]; then retries=0 duration=5 for i in {1..5}; do if ! setup_check_url "${REPO_URL}" ; then retries=$(($retries+1)) echo "Retrying $1 after $duration seconds -- $retries time(s) (max: 5)" sleep $duration duration=$(($duration+$(random 1 5))) fi done if [ $retries -eq 5 ]; then echo "Unable to find git-repo repository. ${REPO_URL}" >&2 return 1 fi fi # If we still don't know it, check the file... if [ -z "$REPO_URL" ]; then if [ -e bin/.git-repo ]; then REPO_URL=$(cat bin/.git-repo) update_url=false # If we still don't know it, go find it... elif ! wr_repo_find ; then return 1 fi fi # Repo rev time... # User passed it in, verify it if [ -n "$REPO_REV" ]; then retries=0 duration=5 for i in {1..5} ; do output=$(setup_check_url_branch "${REPO_URL}" "${REPO_REV}") if [ "$output" != "$REPO_REV" -o $? -ne 0 ] ; then retries=$(($retries+1)) echo "Retrying $1 after $duration seconds -- $retries time(s) (max: 5)" sleep $duration duration=$(($duration+$(random 1 5))) fi done if [ $retries -eq 5 ]; then echo "Unable to find branch ${REPO_REV} in git-repo repository (${REPO_URL})" >&2 return 1 fi fi if [ -z "$REPO_REV" ]; then BASEBRANCHES[0]=${BASEBRANCH} # Skip master-wr, we don't use this branch any longer in git-repo... if [ "${BASEBRANCHES[0]}" == "master-wr" ]; then BASEBRANCHES[0]="$repo_branch_fallback" fi if [ "${BASEBRANCHES[0]}" != "$repo_branch_fallback" ]; then BASEBRANCHES[1]="$repo_branch_fallback" fi retries=0 duration=5 for i in {1..5} ; do REPO_REV=$(setup_check_url_branch "${REPO_URL}" "${BASEBRANCHES[@]}") if [ -z "${REPO_REV}" ]; then retries=$(($retries+1)) echo "Retrying $1 after $duration seconds -- $retries time(s) (max: 5)" sleep $duration duration=$(($duration+$(random 1 5))) fi done if [ $retries -eq 5 ];then echo "Unable to find a usable branch (${BASEBRANCHES[@]}) in git-repo repository (${REPO_URL})" >&2 return 1 fi fi # Ensure subsequent 'repo' calls use the correct URL if $update_url ; then echo ${REPO_URL} > bin/.git-repo fi export REPO_URL export REPO_REV return 0 } wr_repo_clone() { # git-repo is limited to working on it's own branches only. # In otherwords, we can't checkout a tag in git-repo and work with it, # otherwise we get numerous errors that things fail due to them not being # based on branches. # # Due to the design of git-repo, it is safe to use the latest version # on a branch associated with the tag. For instance, if the tag # vWRLINUX_CI_10.19.29.0 is a tag based on WRLINUX_CI branch, we can just # use the branch for cloning. # # Since we can't check what branch the tag is from w/o a clone, we clone... if [ ! -d bin/git-repo ]; then trap : INT git clone "${REPO_URL}" bin/git-repo if [ $? -ne 0 ]; then echo "Unable to clone git-repo from ${REPO_URL}." >&2 return 1 fi trap - INT else if [ "${REPO_URL}" != "$(git config -f bin/git-repo/.git/config remote.origin.url)" ]; then echo "Updating git-repo remote url" git config -f bin/git-repo/.git/config remote.origin.url "${REPO_URL}" fi # We always clear local changes to make sure we're synced up! (cd bin/git-repo && git remote update origin && git reset --hard @{upstream}) if [ $? -ne 0 ]; then echo "WARNING: Unable to reset the git-repo respository." >&2 fi fi # Translate REPO_REV, if a tag to a branch if [ "$REPO_REV" != "${REPO_REV##refs/tags/}" ]; then # Find the first branch containing that commit.. REPO_BRANCH=$(cd bin/git-repo && git branch -r --contains ${REPO_REV}^{commit} 2>/dev/null | grep -v '\->' | head -n 1) if [ -z ${REPO_BRANCH} ]; then echo "ERROR: Unable to find branch containing $REPO_REV in git-repo repository." exit 1 fi # Strip spaces REPO_BRANCH=$(echo ${REPO_BRANCH}) # Turn into a local branch REPO_BRANCH=${REPO_BRANCH##origin/} echo "Translated tag ${REPO_REV} to branch ${REPO_BRANCH}" REPO_REV=$REPO_BRANCH fi if [ "* ${REPO_REV}" != "$(cd bin/git-repo && git branch | grep '\*')" ]; then (cd bin/git-repo && git checkout ${REPO_REV}) if [ $? -ne 0 ]; then echo "ERROR: Unable to checkout branch ${REPO_REV}" >&2 return 1 fi fi if [ -d .repo/repo/.git ]; then echo "Syncing git-repo to configured .repo/repo" repo_resync=false bin_repo_url=$(git config -f bin/git-repo/.git/config remote.origin.url) git_repo_url=$(git config -f .repo/repo/.git/config remote.origin.url) if [ "${bin_repo_url}" != "${git_repo_url}" ]; then git config -f .repo/repo/.git/config remote.origin.url "${bin_repo_url}" repo_resync=true fi bin_repo_branch=$(git config -f bin/git-repo/.git/config branch.${REPO_REV}.merge) git_repo_branch=$(git config -f .repo/repo/.git/config branch.default.merge) if [ "${bin_repo_branch}" != "${git_repo_branch}" ]; then git config -f .repo/repo/.git/config branch.default.merge "${bin_repo_branch}" repo_resync=true git_repo_branch=$(git config -f .repo/repo/.git/config branch.default.merge) fi if $repo_resync ; then (cd .repo/repo && git remote update origin) || exit 1 (cd .repo/repo && git reset --hard @{upstream}) || exit 1 fi fi export PATH=$(cd bin/git-repo && pwd):$PATH } ================================================ FILE: data/environment.d/07_toaster_update.sh ================================================ # Copyright (C) 2017 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Regenerate the wrlinux-specific Toaster fixture file based on the # current 'layers_wrs_com.json' and 'default.xml' setup_shutdown_func update_toaster_fixture_stop update_toaster_fixture_stop() { if [ -f default.xml ] && [ -d bitbake ] ; then # generate the wrlinux-specific Toaster fixture file $BASEDIR/bin/toaster_fixture.py --project-dir $PWD fi } ================================================ FILE: data/environment.d/check_update.sh ================================================ # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Check if this is an existing product, if the default.xml has beeen changed # warning the user they may need to update any build directories setup_add_func check_update_start setup_shutdown_func check_update_stop check_update_start() { if [ -e config/bblayers.conf.sample ]; then SHASUM_BBLAYERS=$(shasum config/bblayers.conf.sample | cut -d ' ' -f 1) fi if [ -e config/local.conf.sample ]; then SHASUM_LOCALCONF=$(shasum config/local.conf.sample | cut -d ' ' -f 1) fi } check_update_stop() { if [ -e config/bblayers.conf.sample -a -n "${SHASUM_BBLAYERS}" ]; then NEW_SHASUM=$(shasum config/bblayers.conf.sample | cut -d ' ' -f 1) if [ "${NEW_SHASUM}" != "${SHASUM_BBLAYERS}" ]; then cat << EOF Note: The project layers have been updated. You should inspect the conf/bblayers.conf file in all build directories and syncronize them to match the updated conf/bblayers.conf.sample file, as necessary. EOF fi fi if [ -e config/local.conf.sample -a -n "${SHASUM_LOCALCONF}" ]; then NEW_SHASUM=$(shasum config/local.conf.sample | cut -d ' ' -f 1) if [ "${NEW_SHASUM}" != "${SHASUM_LOCALCONF}" ]; then cat << EOF Note: The project local.conf.sample has been updated. You should inspect the conf/local.conf file in all build directories and syncronize them to match, as necessary. EOF fi fi } ================================================ FILE: data/environment.d/kernel_type.sh ================================================ # Copyright (C) 2017 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # For compatibility with the commercial version, we need to allow for the # accept-eula argument, even though it's a no-op # We don't 'keep' this command & argument as the option is not longer used. setup_add_arg --kernel KTYPE setup_add_func ktype_check ktype_check() { if [ -n "${KTYPE+x}" ]; then echo echo "WARNING: argument --kernel is no longer supported and will be ignored." >&2 echo " To adjust the LINUX_KERNEL_TYPE, edit the build conf/local.conf file." >&2 echo fi } ================================================ FILE: data/environment.d/setup_anspass ================================================ # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # This needs to run after anspass is installed by the buildtools-tarball # Transfer askpass to anspass anspass_transfer() { urls="" type=unknown while read line; do case "$line" in Username*) url=$(echo $line | sed "s,Username for '\(.*\)': *,\1,") type=username ;; Password*) url=$(echo $line | sed "s,Password for '\(.*\)': *,\1,") url=$(echo $url | sed "s,\([^:]*://\)\(.*\)@\([^@]*\),\1\3,") type=password ;; *) if [ $type == "unknown" -o -z "$line" ] ; then continue fi murl=$(echo $url | sed 's,[-:/.@],_,g') if [ "${urls}" == "${urls/$murl/}" ]; then urls="$urls $murl" fi eval url_${murl}="${url}" case $type in username) eval username_${murl}=\${line} ;; password) eval password_${murl}=\${line} ;; esac ;; esac done for murl in $urls ; do anspass_transfer_dump | eval anspass-ctrl --add \${url_${murl}} > /dev/null 2>&1 done } anspass_transfer_dump() { eval echo \${username_${murl}} eval echo \${password_${murl}} eval echo \${password_${murl}} } anspass_check_credential() { err=$(LANG=C git ls-remote $REMOTEURL 2>&1 >/dev/null) ret=$? if [ $ret -ne 0 ] && [ "${err}" != "${err/Authentication/}" ]; then echo echo "${err}" while true ; do echo echo "The saved user or password seems invalid, remove it and try again?" read -p "Remove the anspass db bin/.anspass? [Yes/No]" accept case ${accept} in [yY][eE][sS]) anspass_stop rm -rf ${PWD}/bin/.anspass break ;; [nN][oO]) break ;; *) echo "Only yes and no are accepted." >&2 ;; esac done fi } anspass_start() { # Do we have anspassd available yet? if ! which anspassd >/dev/null 2>&1 ; then return 0 fi # Have we already configured anspass? if [ -n "${ANSPASS_PATH}" -a -n "${ANSPASS_TOKEN}" ]; then return 0 fi echo "Starting anspass..." export ANSPASS_PATH=${PWD}/bin/.anspass result=$(anspassd < /dev/null 2>&1) if [ $? -ne 0 ]; then echo "anspassd failed to start:" echo "${result}" return 1 fi export ANSPASS_TOKEN=$(echo "$result" | grep "Token:" | head -n 1 | sed 's,Token: ,,') #echo "Token: ${ANSPASS_TOKEN}" wrl_anspass_started=1 return 0 } anspass_setup() { # Do we have anspassd available yet? if ! which anspassd >/dev/null 2>&1 ; then return 0 fi # Have we already configured anspass? if [ -n "${ANSPASS_PATH}" -a -n "${ANSPASS_TOKEN}" ]; then return 0 fi WINDSHARE_SCHEME=$(echo ${BASEURL} | sed 's,\([^:/]*\).*,\1,') if [ -n "${WINDSHARE_SCHEME}" ]; then WINDSHARE_HOST=$(echo ${BASEURL} | sed 's,\([^:/]*\)://\([^/]*\).*,\2,') fi # Has anspass been started before? if [ -d ${PWD}/bin/.anspass ]; then anspass_start rc=$? if [ $rc != 0 ]; then return $rc fi else return 0 fi # If user/pass passed in, default these first (can't do this for a file path) if [ -n "${WINDSHARE_USER}" -a -n "${WINDSHARE_HOST}" ]; then # If this has already been set, delete it so we can add the new one echo "${WINDSHARE_USER}" | anspass-ctrl --delete ${WINDSHARE_SCHEME}://${WINDSHARE_HOST} > /dev/null 2>&1 cat << EOF | anspass-ctrl --add ${WINDSHARE_SCHEME}://${WINDSHARE_HOST} > /dev/null 2>&1 ${WINDSHARE_USER} ${WINDSHARE_PASS} ${WINDSHARE_PASS} EOF if [ $? -ne 0 ]; then echo "Unable to set the credentials for ${WINDSHARE_SCHEME}://${WINDSHARE_HOST}" >&2 anspass-ctrl --quit > /dev/null 2>&1 return 1 fi fi export GIT_ASKPASS=$(which anspass) export SSH_ASKPASS=$(which anspass) # check the old credentials works or not anspass_check_credential } anspass_stop() { if [ -n "${ANSPASS_PATH}" -a -n "${ANSPASS_TOKEN}" ]; then echo "Stopping anspass..." anspass-ctrl --quit >/dev/null 2>&1 unset ANSPASS_PATH unset ANSPASS_TOKEN fi } ================================================ FILE: data/environment.d/setup_askpass ================================================ #! /usr/bin/env python3 # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import os import sys import socket import subprocess import termios if len(sys.argv) <= 1: sys.exit(0) named_socket = os.getenv("WRL_ASKPASS_SOCKET") if not named_socket or named_socket == "": print('You must specify a named pipe in WRL_ASKPASS_SOCKET.') sys.exit(1) sock_dir = os.path.dirname(named_socket) sock_name = os.path.basename(named_socket) # Make sure the socket directory exists if not os.path.exists(sock_dir): os.makedirs(sock_dir) os.chdir(sock_dir) if sys.argv[1] == '--server': q_cache = {} try: os.unlink(sock_name) except: pass s = socket.socket(socket.AF_UNIX) s.bind(sock_name) s.listen(1) quit = False while not quit: conn, addr = s.accept() while 1: data = conn.recv(1024).decode() if not data: break if data[0] == 'q': q = data[1:] response = '' if q in q_cache: response = q_cache[q] #print('Q: "%s"' % q) #print('R: "%s"' % response) conn.send(response.encode()) break if data[0] == 's': res = data.split('\n') q = res[0][1:] a = res[1] q_cache[q] = a #print('S: "%s" = "%s"' % (q, q_cache[q])) break if data[0] == 'd': for q in q_cache: if q == 'tty': continue #print('D: "%s" = "%s"' % (q, q_cache[q])) conn.send((q + '\n' + q_cache[q][1:] + '\n').encode()) break if data[0] == 'x': #print('quit') quit = True break #print("skip: " + data) conn.close() sys.exit(0) if sys.argv[1] == '--quit': s = socket.socket(socket.AF_UNIX) s.connect(sock_name) s.send('x'.encode()) s.close() sys.exit(0) if sys.argv[1] == '--set': result = sys.stdin.readline() s = socket.socket(socket.AF_UNIX) s.connect(sock_name) s.send(('s' + sys.argv[2] + '\n' + 'A' + result).encode()) s.close() sys.exit(0) if sys.argv[1] == '--dump': s = socket.socket(socket.AF_UNIX) s.connect(sock_name) s.send('d'.encode()) while 1: result = s.recv(1024).decode() if not result: break print(result) s.close() sys.exit(0) s = socket.socket(socket.AF_UNIX) s.connect(sock_name) s.send(('q' + sys.argv[1]).encode()) result = s.recv(1024).decode() s.close() if result: print(result[1:]) sys.exit(0) s = socket.socket(socket.AF_UNIX) s.connect(sock_name) s.send('qtty'.encode()) tty_file = s.recv(1024)[1:].decode() s.close() if not tty_file: tty_file = "/dev/tty" with open(tty_file, 'wb+', buffering=0) as tty_fd: if not tty_fd.isatty(): # Only operate on a tty sys.exit(1) question = sys.argv[1] if question.startswith('Password') or question.endswith('password: '): fd = tty_fd.fileno() old = termios.tcgetattr(fd) new = termios.tcgetattr(fd) try: # Disable echoing and signals # If we don't disable signals, readline can get confused and # 'break' the terminal, this way the user MUST enter 'something' new[3] &= ~termios.ECHO new[3] &= ~termios.ISIG termios.tcsetattr(fd, termios.TCSADRAIN, new) if termios.tcgetattr(fd)[3] != new[3]: tty_fd.write("WARNING: Password may echo to the screen!\n".encode()) tty_fd.write(question.encode()) result = tty_fd.readline().decode() tty_fd.write('\n'.encode()) finally: termios.tcsetattr(fd, termios.TCSADRAIN, old) else: tty_fd.write(question.encode()) result = tty_fd.readline().decode() s = socket.socket(socket.AF_UNIX) s.connect(sock_name) s.send(('s' + question + '\n' + 'A'+ result).encode()) s.close() s = socket.socket(socket.AF_UNIX) s.connect(sock_name) s.send(('q' + sys.argv[1]).encode()) result = s.recv(1024).decode() s.close() if result: print(result[1:]) ================================================ FILE: data/environment.d/setup_ssh ================================================ #! /bin/sh # # Intercept the SSH calls during the installer to ensure that we have control # over the display and terminal. # # ssh will always prompt the user for their passphrase (password) if the tty # is setup. This avoids using SSH_ASKPASS, which is needed for the setup # functionality. # # We use setsid to break the tty connection, and also ensure that DISPLAY is # cleared. This combination forces SSH to use 'SSH_ASKPASS'. # # Note: storying SSH credentials can be a bit sketchy, so it is recommended # to use an authorized_key file instead. But askpass/anspass do their best # to obfuscate the data. # # Use this file by setting GIT_SSH=setup_ssh DISPLAY= setsid ssh $@ ================================================ FILE: data/environment.d/setup_utils ================================================ # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # These are common utility functions multiple environment scripts # This unit may be loaded more then once # $1 is the url (or file path) to check if it's a git repository # Function returns: # 0 if git repository is available # 1 if it's a bad path # 2 if authentication failed (and prints a message) # 3 an unknown error occurs (and prints a message) setup_check_url() { err=$(LANG=C git ls-remote "$1" 2>&1 >/dev/null) ret=$? # Found! if [ $ret -eq 0 ]; then return 0 fi # Check for authentication error... if [ "${err}" != "${err/Authentication/}" ]; then echo "Authentication failed for $1" >&2 return 2 fi # Check if it's just a bad path if [ "${err}" != "${err/not found/}" ]; then # This is a normal, git repo not found error return 1 fi if [ "${err}" != "${err/Could not read/}" ]; then # This is just a bad path, git repo not found error return 1 fi # This error is unique to git servers if [ "${err}" != "${err/access denied or repository not exported/}" ]; then # This is just a bad path, git repo not found error return 1 fi # Unknown error echo "An unknown error occured looking for $1" >&2 echo "$err" >&2 return 3 } # $1 is the url (or file path) to check if it's a git repository # $2...$n is the branches to check if they exist # Function returns: # 0 if git repository is available # 1 if it's a bad path # 2 if authentication failed (and prints a message) # 3 an unknown error occurs (and prints a message) # # stdout will be the first branch that exists (rc 0) setup_check_url_branch() { local repo repo=$1 shift setup_check_url "$repo" ret=$? if [ $? -ne 0 ]; then return $ret fi local branch for branch in "$@"; do output=$(LANG=C git ls-remote "$repo" "$branch") if [ -n "$output" ]; then echo $branch return 0 fi done } # function to generate a random number between min and max random(){ min=$1 max=$(($2-$min+1)) num=$(($RANDOM+1000000000)) echo $(($num%$max+$min)) } ================================================ FILE: data/local_layer/README ================================================ local layer =========== This layer is intended to work as a scratch space in the default projects. Dependencies ------------ None. Maintenance ----------- This layer is maintained by the project owner. License ------- Copyright (C) 2013-2016 Wind River Systems, Inc. Source code included in the tree for individual recipes is under the LICENSE stated in the associated recipe (.bb file) unless otherwise stated. The metadata is under the following license unless otherwise stated. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: data/local_layer/classes/.keepme ================================================ ================================================ FILE: data/local_layer/conf/layer.conf ================================================ # We have a conf and classes directory, add to BBPATH BBPATH =. "${LAYERDIR}:" # We have a recipe-* directory, add to BBFILES BBFILES += "${LAYERDIR}/recipes-*/*/*.bb \ ${LAYERDIR}/recipes-*/*/*.bbappend" BBFILE_COLLECTIONS += "local" BBFILE_PATTERN_local = "^${LAYERDIR}/" BBFILE_PRIORITY_local = "1000" # Add scripts to PATH PATH .= ":${LAYERDIR}/scripts" # Allow the developer to put pre-downloaded src into the downloads directory # Also allow the user to have bare clones of repositories in the git directory PREMIRRORS_append = "\ .*://.*/.* file://${LAYERDIR}/downloads/ \n \ git://.*/.* git://${LAYERDIR}/git/BASENAME;protocol=file \n \ " # Enable the local layer unconditionally if the whitelist in enabled PNWHITELIST_LAYERS_remove = "local" # We are compatiable with all the possibilities. # LAYERSERIES_COMPAT_local = "${LAYERSERIES_CORENAMES}" ================================================ FILE: data/local_layer/downloads/.keepme ================================================ ================================================ FILE: data/local_layer/git/.keepme ================================================ ================================================ FILE: data/local_layer/recipes-sample/hello/hello/hello.c ================================================ /* * Copyright 2012 Wind River Systems, Inc. */ #include main() { printf("Hello World\n"); } ================================================ FILE: data/local_layer/recipes-sample/hello/hello_1.0.bb ================================================ DESCRIPTION = "This package contains the simple Hello World program." LICENSE = "windriver" LICENSE_FLAGS = "commercial_windriver" LIC_FILES_CHKSUM = "file://hello.c;beginline=1;endline=3;md5=3e8f741b049bec8146c81a2667ab4b45" SECTION = "sample" PR = "r1" SRC_URI = "file://hello.c" S = "${WORKDIR}" do_compile() { ${CC} ${CFLAGS} ${LDFLAGS} -o hello hello.c } do_install() { install -d ${D}${bindir} install -m 0755 hello ${D}${bindir} } ================================================ FILE: data/local_layer/scripts/.keepme ================================================ ================================================ FILE: data/samples/README-MIRROR.sample ================================================ This is a Wind River Linux Repository Mirror. It was constructed by the setup program using the following arguments: ####SETUP_ARGS#### To update your mirror ---------------------- The preferred way of updating your repository mirror is to re-run the setup program. First you should update the wrlinux-x, for example: $ cd wrlinux-x $ git pull $ cd .. Then re-run the setup program with the original arguments. $ wrlinux-x/setup.sh ####SETUP_ARGS#### You can change the arguments to the setup program. This will change the what is downloaded as part of your repository mirror. If repositories have been removed, it may break users of this mirror. ================================================ FILE: data/samples/README.sample ================================================ This is a Wind River Linux build project. It was constructed by the setup.sh tool using the following arguments: ####SETUP_ARGS#### Buildtools ---------- Wind River provides special host build-tools in order to supplement the build systems tools. While this is not required, it can solve many problems with missing or incompatible host tools. These tools are installed into bin/buildtools. We recomend you always source this into your environment each time you open a new shell session, and before issueing any build system commands. To do this: $ . ./environment-setup--wrlinuxsdk-linux Quickstart Instructions ----------------------- Once you have sourced the environment file, as noted above... You must first create a build directory: $ . ./oe-init-build-env The is the directory to create, if not specified the system will default to the directory 'build'. Once the build directory has been created, the system will change your working directory to be inside of the build directory. Each time you open a new shell session you will need to source the oe-init-build-env script as indicated above. Once in the build directory edit the 'conf/local.conf' file. Please refer to the comments in the file for specific configuration instructions. After configuring your build directory, you may now run bitbake commands. Note that bitbake, actually python3, requires the LANG environment variable to be a utf-8 variant, e.g. en_US.UTF-8, or it will terminate with an error. Bitbake commands are of the format: bitbake For example: $ export LANG=en_US.UTF-8 # if needed $ bitbake core-image-minimal or $ LANG=en_US.UTF-8 bitbake core-image-minimal Once your have built an image, you can boot this using QEMU (for compatible MACHINE settings) using the 'runqemu' command, such as: $ runqemu qemux86-64 To update your project ---------------------- To update your project, you must update the platform project git repository and re-run the setup.sh tool. First update wrlinux-x, for example: $ cd wrlinux-x $ git pull $ cd .. Then re-run the setup.sh tool with the original arguments. $ wrlinux-x/setup.sh ####SETUP_ARGS#### Note: existing local.conf and bblayers.conf files are not modified by setup.sh tool -- you will have to update them manually, or start a new build directory. Local layer ----------- A special 'local' layer has been added to your project automatically. This layer is located at the path 'layers/local' in your project. The purpose of the local layer is a staging location for your project work, including any new recipes, classes or scripts. ================================================ FILE: data/samples/bblayers.conf.sample ================================================ # LAYER_CONF_VERSION is increased each time build/conf/bblayers.conf # changes incompatibly LCONF_VERSION = "7" BBPATH = "${TOPDIR}" BBFILES ?= "" BBLAYERS ?= " \ ####LAYERS#### \ ##OEROOT##/layers/local \ " ================================================ FILE: data/samples/conf-notes.sample ================================================ This project was configured with the following options: ####SETUP_ARGS#### Common Wind River images are: wrlinux-image-small (suggests distro: wrlinux and feature/busybox) wrlinux-image-core (suggests distro: wrlinux) wrlinux-image-std (suggests distro: wrlinux) wrlinux-image-std-sato (requires distro: wrlinux-graphics) Common Yocto Project images, typically built with distro poky, are: core-image-minimal core-image-base core-image-sato You can also run generated qemu images with a command like 'runqemu qemux86-64' Other commonly useful commands are: - 'devtool' and 'recipetool' handle common recipe tasks - 'bitbake-layers' handles common layer tasks - 'oe-pkgdata-util' handles common target package tasks ================================================ FILE: data/samples/local.conf.sample ================================================ # # This file is your local configuration file and is where all local user settings # are placed. The comments in this file give some guide to the options a new user # to the system might want to change but pretty much any configuration option can # be set in this file. More adventurous users can look at local.conf.extended # which contains other examples of configuration which can be placed in this file # but new users likely won't need any of them initially. # # Lines starting with the '#' character are commented out and in some cases the # default values are provided as comments to show people example syntax. Enabling # the option is a question of removing the # character and making any change to the # variable as required. # # Network Control # # Disable bitbake from being able to access the network BB_NO_NETWORK ?= '1' # Only allow downloads from PREMIRRORs, not search the main SRC_URI or MIRRORS. # If you have enabled download (dl-layers), use this to restrict fetch to those # layers. This also has a side effect of telling the system to ensure the # fetched repositories from the local download layer (pre)mirrors are # up-to-date. BB_FETCH_PREMIRRORONLY ?= '${@d.getVar('BB_NO_NETWORK') or 0}' # # Wind River Linux Templates # # The following variables control the template inclusion mechanism. # # Specify templates you wish to use in a space separated list. WRTEMPLATE ?= "####DEFAULTWRTEMPLATE####" # If you want to disable one or more templates # Note: this matches the 'end' of any templates that were found.. so you can do # things like: # # default (skip all default templates) # feature/foobar (skip all layers feature/foobar) # wr-base/feature/foobar (skip only feature/foobar in wr-base) #WRTEMPLATE_SKIP = "feature/item3 feature/item4 ..." # Selectively enable or disable the template 'image.inc' additions to a given # image recipe. # (the default value is 1) #WRTEMPLATE_IMAGE_pn- = "0" # Disable or enable processing of the special 'bsp-pkgs.*' template pieces. # (the default value is 1) #WRTEMPLATE_BSP_PKGS = "0" # # Machine Selection # # You need to select a specific machine to target the build with. # Some machines are emulated, usually the name starts with qemu, these can boot # and run in the QEMU emulator: # #MACHINE ?= "####MACHINES####" # # This sets the default machine if no other machine has been selected: MACHINE ??= "####DEFAULTMACHINE####" # # Linux Kernel Type # # The kernel type is selected by the recipe that provides the kernel. # By default, this recipe is selected by the DISTRO. # # If you want to override the default kernel, you must select the desired # recipe via PREFERRED_PROVIDER_virtual/kernel. # # To select a specific kernel recipe, uncomment the line below # and set the value. # # Typical values include: linux-yocto, linux-yocto-rt, and linux-yocto-tiny # #PREFERRED_PROVIDER_virtual/kernel = "linux-yocto" # # Where to place downloads # # During a first build the system will download many different source code tarballs # from various upstream projects. This can take a while, particularly if your network # connection is slow. These are all stored in DL_DIR. When wiping and rebuilding you # can preserve this directory to speed up this part of subsequent builds. This directory # is safe to share between multiple builds on the same machine too. # # The default is a downloads directory under TOPDIR which is the build directory. # #DL_DIR ?= "${TOPDIR}/downloads" # # Where to place shared-state files # # BitBake has the capability to accelerate builds based on previously built output. # This is done using "shared state" files which can be thought of as cache objects # and this option determines where those files are placed. # # You can wipe out TMPDIR leaving this directory intact and the build would regenerate # from these files if no changes were made to the configuration. If changes were made # to the configuration, only shared state files where the state was still valid would # be used (done using checksums). # # The default is a sstate-cache directory under TOPDIR. # #SSTATE_DIR ?= "${TOPDIR}/sstate-cache" # # Where to place the build output # # This option specifies where the bulk of the building work should be done and # where BitBake should place its temporary files and output. Keep in mind that # this includes the extraction and compilation of many applications and the toolchain # which can use Gigabytes of hard disk space. # # The default is a tmp directory under TOPDIR. # #TMPDIR = "${TOPDIR}/tmp" # # Default policy config # # The distribution setting controls which policy settings are used as defaults. # The default value is fine for general Yocto project use, at least initially. # Ultimately when creating custom policy, people will likely end up subclassing # these defaults. # #DISTRO ?= "####DISTROS####" # # This sets the default distro if no other distro has been selected: DISTRO ??= "####DEFAULTDISTRO####" # # Package Management configuration # # This variable lists which packaging formats to enable. Multiple package backends # can be enabled at once and the first item listed in the variable will be used # to generate the root filesystems. # Options are: # - 'package_deb' for debian style deb files # - 'package_ipk' for ipk files are used by opkg (a debian style embedded package manager) # - 'package_rpm' for rpm style packages # E.g.: PACKAGE_CLASSES ?= "package_rpm package_deb package_ipk" # We default to rpm: PACKAGE_CLASSES ?= "package_rpm" # # SDK target architecture # # This variable specifies the architecture to build SDK items for and means # you can build the SDK packages for architectures other than the machine you are # running the build on (i.e. building i686 packages on an x86_64 host). # Supported values are i686 and x86_64 #SDKMACHINE ?= "i686" # # Extra image configuration defaults # # The EXTRA_IMAGE_FEATURES variable allows extra packages to be added to the generated # images. Some of these options are added to certain image types automatically. The # variable can contain the following options: # "dbg-pkgs" - add -dbg packages for all installed packages # (adds symbol information for debugging/profiling) # "src-pkgs" - add -src packages for all installed packages # (adds source code for debugging) # "dev-pkgs" - add -dev packages for all installed packages # (useful if you want to develop against libs in the image) # "ptest-pkgs" - add -ptest packages for all ptest-enabled packages # (useful if you want to run the package test suites) # "tools-sdk" - add development tools (gcc, make, pkgconfig etc.) # "tools-debug" - add debugging tools (gdb, strace) # "eclipse-debug" - add Eclipse remote debugging support # "tools-profile" - add profiling tools (oprofile, lttng, valgrind) # "tools-testapps" - add useful testing tools (ts_print, aplay, arecord etc.) # "debug-tweaks" - make an image suitable for development # e.g. ssh root access has a blank password # There are other application targets that can be used here too, see # meta/classes/image.bbclass and meta/classes/core-image.bbclass for more details. # We default to enabling the debugging tweaks. EXTRA_IMAGE_FEATURES ?= "debug-tweaks" # # Additional image features # # The following is a list of additional classes to use when building images which # enable extra features. Some available options which can be included in this variable # are: # - 'buildstats' collect build statistics # - 'image-prelink' in order to prelink the filesystem image USER_CLASSES ?= "buildstats image-prelink" # # Additional rootfs image types # # The following is a partial list of additional image filesystem types that # may be supported by the configured machine; # - 'tar.gz' to create a gzip compressed tarball of the image # - 'tar.bz2' to create a bz2 compressed tarball of the image # - 'ext4' to create an ext4 image # # NOTE: # Due to the way the OpenEmbedded build system processes this variable, # you cannot update its contents by using _append or _prepend. You must use # the += operator to add one or more options to the IMAGE_FSTYPES variable. IMAGE_FSTYPES += "tar.bz2" # The following options will build a companion 'debug filesystem' in addition # to the normal deployable filesystem. This companion system allows a # debugger to know the symbols and related sources. It can be used to # debug a remote 'production' system without having to add the debug symbols # and sources to remote system. If IMAGE_FSTYPES_DEBUGFS is not defined, it # defaults to IMAGE_FSTYPES. #IMAGE_GEN_DEBUGFS = "1" #IMAGE_FSTYPES_DEBUGFS = "tar.bz2" # The network based PR service host and port # Uncomment the following lines to enable PRservice. # Set PRSERV_HOST to 'localhost:0' to automatically # start local PRService. # Set to other values to use remote PRService. #PRSERV_HOST = "localhost:0" # # Runtime testing of images # # The build system can test booting virtual machine images under qemu (an emulator) # after any root filesystems are created and run tests against those images. It can also # run tests against any SDK that are built. To enable this uncomment these lines. # See classes/test{image,sdk}.bbclass for further details. #IMAGE_CLASSES += "testimage testsdk" #TESTIMAGE_AUTO_qemuall = "1" # Note: test image requires sshd and scp support on the target, this can be added by # adding one of the following two lines. #IMAGE_INSTALL_append = " openssh-sshd openssh-scp" #IMAGE_INSTALL_append = " dropbear" # # Interactive shell configuration # # Under certain circumstances the system may need input from you and to do this it # can launch an interactive shell. It needs to do this since the build is # multithreaded and needs to be able to handle the case where more than one parallel # process may require the user's attention. The default is iterate over the available # terminal types to find one that works. # # Examples of the occasions this may happen are when resolving patches which cannot # be applied, to use the devshell or the kernel menuconfig # # Supported values are auto, gnome, xfce, rxvt, screen, konsole (KDE 3.x only), none # Note: currently, Konsole support only works for KDE 3.x due to the way # newer Konsole versions behave #OE_TERMINAL = "auto" # By default disable interactive patch resolution (tasks will just fail instead): PATCHRESOLVE = "noop" # # Disk Space Monitoring during the build # # Monitor the disk space during the build. If there is less that 1GB of space or less # than 100K inodes in any key build location (TMPDIR, DL_DIR, SSTATE_DIR), gracefully # shutdown the build. If there is less that 100MB or 1K inodes, perform a hard abort # of the build. The reason for this is that running completely out of space can corrupt # files and damages the build in ways which may not be easily recoverable. # It's necesary to monitor /tmp, if there is no space left the build will fail # with very exotic errors. BB_DISKMON_DIRS ??= "\ STOPTASKS,${TMPDIR},1G,100K \ STOPTASKS,${DL_DIR},1G,100K \ STOPTASKS,${SSTATE_DIR},1G,100K \ STOPTASKS,/tmp,100M,100K \ ABORT,${TMPDIR},100M,1K \ ABORT,${DL_DIR},100M,1K \ ABORT,${SSTATE_DIR},100M,1K \ ABORT,/tmp,10M,1K" # # Shared-state files from other locations # # As mentioned above, shared state files are prebuilt cache data objects which can # used to accelerate build time. This variable can be used to configure the system # to search other mirror locations for these objects before it builds the data itself. # # This can be a filesystem directory, or a remote url such as http or ftp. These # would contain the sstate-cache results from previous builds (possibly from other # machines). This variable works like fetcher MIRRORS/PREMIRRORS and points to the # cache locations to check for the shared objects. # NOTE: if the mirror uses the same structure as SSTATE_DIR, you need to add PATH # at the end as shown in the examples below. This will be substituted with the # correct path within the directory structure. #SSTATE_MIRRORS ?= "\ #file://.* http://someserver.tld/share/sstate/PATH;downloadfilename=PATH \n \ #file://.* file:///some/local/dir/sstate/PATH" # # Make sstate can be downloaded from network when BB_NO_NETWORK is set to 1 #SSTATE_MIRROR_ALLOW_NETWORK ?= "1" # Select the default init manager # use systemd as the default init manager DISTRO_FEATURES_append = " systemd" DISTRO_FEATURES_BACKFILL_CONSIDERED += "sysvinit" VIRTUAL-RUNTIME_init_manager = "systemd" VIRTUAL-RUNTIME_initscripts = "systemd-compat-units" KERNEL_FEATURES_append = " cfg/systemd.scc" # use sysvinit as the default init manager #DISTRO_FEATURES_append = " sysvinit" #DISTRO_FEATURES_BACKFILL_CONSIDERED += "systemd" #VIRTUAL-RUNTIME_init_manager = "sysvinit" #VIRTUAL-RUNTIME_initscripts = "initscripts" # # Qemu configuration # # By default qemu will build with a builtin VNC server where graphical output can be # seen. The two lines below enable the SDL backend too. By default libsdl2-native will # be built, if you want to use your host's libSDL instead of the minimal libsdl built # by libsdl2-native then uncomment the ASSUME_PROVIDED line below. PACKAGECONFIG_append_pn-qemu-system-native = " sdl" PACKAGECONFIG_append_pn-nativesdk-qemu = " sdl" #ASSUME_PROVIDED += "libsdl2-native" # CONF_VERSION is increased each time build/conf/ changes incompatibly and is used to # track the version of this file when it was generated. This can safely be ignored if # this doesn't mean anything to you. CONF_VERSION = "1" # The overrides osv-wrlinux will enable wrlinux specific bbappend OVERRIDES_append = ":osv-wrlinux" # Add WRLinux uninative support require wrlinux-distro/conf/distro/include/wrlinux-uninative.inc # Exclude buildtools dir from pseudo database to fix do_package failure PSEUDO_IGNORE_PATHS .= ",##OEROOT##/bin" ================================================ FILE: data/samples/site.conf.sample ================================================ # # local.conf covers user settings, site.conf covers site specific information # such as proxy server addresses and optionally any shared download location # # SITE_CONF_VERSION is increased each time build/conf/site.conf # changes incompatibly SCONF_VERSION = "1" # Uncomment to cause CVS to use the proxy host specified #CVS_PROXY_HOST = "proxy.example.com" #CVS_PROXY_PORT = "81" # For svn, you need to create ~/.subversion/servers containing: #[global] #http-proxy-host = proxy.example.com #http-proxy-port = 81 # # To use git with a proxy, you must use an external git proxy command, such as # the one provided by scripts/oe-git-proxy. To use this script, copy it to # your PATH and uncomment the following: #GIT_PROXY_COMMAND ?= "oe-git-proxy" #ALL_PROXY ?= "socks://socks.example.com:1080" #or #ALL_PROXY ?= "https://proxy.example.com:8080" # If you wish to use certain hosts without the proxy, specify them in NO_PROXY. # See the script for details on syntax. The script oe-git-proxy uses some tools # that may not be included on HOSTTOOLS, thus add them manually through # HOSTTOOLS += "getent" # Uncomment this to use a shared download directory #DL_DIR = "/some/shared/download/directory/" ================================================ FILE: data/xml/axxiaarm-dl.xml ================================================ ================================================ FILE: data/xml/bitbake.inc ================================================ ================================================ FILE: data/xml/intel-socfpga-dl.xml ================================================ ================================================ FILE: data/xml/meta-anaconda-dl.xml ================================================ ================================================ FILE: data/xml/meta-cgl-common-dl.xml ================================================ ================================================ FILE: data/xml/meta-clang-dl.xml ================================================ ================================================ FILE: data/xml/meta-cloud-services-dl-3-3.xml ================================================ ================================================ FILE: data/xml/meta-dpdk-dl.xml ================================================ ================================================ FILE: data/xml/meta-efi-secure-boot-dl.xml ================================================ ================================================ FILE: data/xml/meta-encrypted-storage-dl.xml ================================================ ================================================ FILE: data/xml/meta-freescale-dl.xml ================================================ ================================================ FILE: data/xml/meta-ids-dl.xml ================================================ ================================================ FILE: data/xml/meta-integrity-dl.xml ================================================ ================================================ FILE: data/xml/meta-intel-dl.xml ================================================ ================================================ FILE: data/xml/meta-iot-cloud-dl.xml ================================================ ================================================ FILE: data/xml/meta-openembedded-dl-3-3.xml ================================================ ================================================ FILE: data/xml/meta-qt5-dl.xml ================================================ ================================================ FILE: data/xml/meta-raspberrypi-dl.xml ================================================ ================================================ FILE: data/xml/meta-realtime-dl-3-3.xml ================================================ ================================================ FILE: data/xml/meta-security-compliance-dl.xml ================================================ ================================================ FILE: data/xml/meta-selinux-dl.xml ================================================ ================================================ FILE: data/xml/meta-signing-key-dl.xml ================================================ ================================================ FILE: data/xml/meta-tensorflow-dl-3-3.xml ================================================ ================================================ FILE: data/xml/meta-tpm-dl.xml ================================================ ================================================ FILE: data/xml/meta-tpm2-dl.xml ================================================ ================================================ FILE: data/xml/meta-virtualization-dl-3-3.xml ================================================ ================================================ FILE: data/xml/nxp-imx7.xml ================================================ ================================================ FILE: data/xml/nxp-s32g2xx-dl.xml ================================================ ================================================ FILE: data/xml/oe-core-dl-3-3.xml ================================================ ================================================ FILE: data/xml/openembedded-core.inc ================================================ ================================================ FILE: data/xml/ti-j72xx-dl.xml ================================================ ================================================ FILE: data/xml/wr-ostree-dl.xml ================================================ ================================================ FILE: data/xml/wrlinux-dl.xml ================================================ ================================================ FILE: data/xml/wrlinux.xml ================================================ ================================================ FILE: data/xml/xilinx-zynqmp-dl.xml ================================================ ================================================ FILE: setup.sh ================================================ #!/bin/bash # # Copyright (C) 2016 Wind River Systems, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA if [ -z "${BASH_VERSION}" ]; then echo "This script must be run with bash." >&2 exit 1 fi which python3 > /dev/null if [ $? -ne 0 ]; then echo >&2 "WRLinux setup requires 'python3'." echo >&2 "Please install python3." exit 1 fi if [ `id -u` = 0 ]; then echo >&2 "Do not run setup as root." exit 1 fi : "${GIT_USERNAME=customer}" : "${GIT_USEREMAIL=customer@company.com}" # Requires python3 CMD="bin/setup.py" # Adds arguments to the arg processing # 1 - argument # 2 - variable to define # 3 - keep or discard (if defined, keep) # there may be arguments you don't want passed to the .py script setup_add_arg() { found=0 for parse in ${ARGPARSE[@]}; do comp=${parse%%:*} if [ "${comp}" = "$1" ]; then found=1 fi done if [ ${found} -eq 0 ]; then ARGPARSE[${#ARGPARSE[@]}]="$1:$2:$3" fi } # Functions that add functionality during early processing setup_add_func() { ADDFUNCS[${#ADDFUNCS[@]}]="$1" } # Functions that export variables (or need to run very late) setup_export_func() { EXPORTFUNCS[${#EXPORTFUNCS[@]}]="$1" } # Functions that run on shutdown setup_shutdown_func() { SHUTDOWNFUNCS[${#SHUTDOWNFUNCS[@]}]="$1" } # Takes value_name default_value # value_name is set to the first value found in the list: # git config, git config --global, and finally default_value add_gitconfig() { VAR=$(git config "$1" || git config --global "$1" || echo "$2") git config -f .gitconfig "${1}" "${VAR}" } shutdown() { for func in "${SHUTDOWNFUNCS[@]}"; do # During shutdown, we don't care about return codes $func done } shutdown_handler() { echo -e "\nAborted by user, will terminate this setup." shutdown exit 1 } # Input: argument list # Output: 'help=1' or unset # PASSARGS set to the arguments to pass on parse_arguments() { local found keep comp next val while [ $# -ge 1 ] ; do found=0 if [ "$1" = "--help" -o "$1" = "-h" ]; then # Default into a --help module which is part of setup.py help=1 PASSARGS[${#PASSARGS[@]}]="$1" shift continue fi for parse in ${ARGPARSE[@]}; do comp=${parse%%:*} next=${parse#${comp}:} val=${next%%:*} next=${next#${val}:} if [ "${next}" != "${val}" ]; then keep=${next} else keep="" fi case "$1" in ${comp}=*) eval ${val}=\${1#*=} if [ -n "${keep}" ]; then PASSARGS[${#PASSARGS[@]}]="$1" fi shift found=1 break ;; ${comp}) eval ${val}=\${2} if [ -n "${keep}" ]; then PASSARGS[${#PASSARGS[@]}]="$1" # Only check whether $2 is set or not, set to "" or '--foo' # should work because: # - set to "": keep align with argparse since it works in this way. # - set to "--foo": argparse knows it's not the arg of $1, # but another option, and can handle it correctly. if [ -n "${2+x}" ]; then PASSARGS[${#PASSARGS[@]}]="$2" fi fi if [ -z "${2+x}" ]; then shift 1 else shift 2 fi found=1 break ;; esac done if [ $found -ne 1 ]; then PASSARGS[${#PASSARGS[@]}]="$1" shift fi done } trap shutdown_handler INT # Setup the minimal defaults first.. # BASEDIR, BASEURL and BASEBRANCH BASEDIR=$(readlink -f "$(dirname "$0")") # Argument parsing, define a limited set of args setup_add_arg --base-url BASEURL keep setup_add_arg --base-branch BASEBRANCH keep help=0 parse_arguments "$@" unset PASSARGS # setup git url REMOTEURL=$(cd "$BASEDIR" ; git config remote.origin.url 2>/dev/null) # BASEURL is one directory above the git checkout BASEREPO="" if [ -z "${BASEURL}" ]; then BASEURL=$(echo "$REMOTEURL" | sed -e 's,/$,,' -e 's,/[^/]*$,,') BASEREPO=${REMOTEURL##$BASEURL\/} fi # First check if this is an absolute path (starts w/ '/') # If it's not, we then check if it's a valid URL (contains ://) if [ "${BASEURL:0:1}" != '/' ]; then if [ "${BASEURL#*://}" == "${BASEURL}" -a "${BASEURL#*:}" == "${BASEURL}" ]; then echo >&2 echo "ERROR: The BASEURL ($BASEURL) is not in a supported format." >&2 if [ -n "${BASEREPO}" ]; then echo "The BASEURL was derived from the URL of $BASEREPO ($REMOTEURL)." >&2 echo "Either update the repository URL or use the --base-url argument to override." >&2 fi echo >&2 echo "BASEURL must use an absolute file path, or a properly formatted remote URL" >&2 echo "such as:" >&2 echo " /home/user/path or file:///home/user/path" >&2 echo " http://hostname/path" >&2 echo " https://hostname/path" >&2 echo " git://hostname/path" >&2 echo " ssh://user@hostname/path" >&2 echo " [user@]hostname:path" >&2 echo >&2 exit 1 fi fi git_cmd="git --git-dir=$BASEDIR/.git" if [ -z "${BASEBRANCH}" ]; then BASEBRANCH=$($git_cmd rev-parse --abbrev-ref HEAD) if [ "$BASEBRANCH" = "HEAD" ]; then # Maybe this is a tag instead? BASEBRANCH=$($git_cmd describe HEAD 2>/dev/null) if [ $? -ne 0 ]; then # No reasonable branch/tag name found... BASEBRANCH="" else echo "$BASEBRANCH" | grep -q "vWRLINUX_CI_" if [ $? -ne 0 ]; then echo "ERROR: Tag $BASEBRANCH is not supported by setup.sh, please use a branch." >&2 echo "" exit 1 fi latest_tag=$($git_cmd tag --sort=taggerdate |tail -1) if [ "$latest_tag" != "$BASEBRANCH" ]; then echo "ERROR: Only latest tag is supported" >&2 echo "ERROR: Current tag: $BASEBRANCH" >&2 echo "ERROR: Latest tag: $latest_tag" >&2 echo "" exit 1 fi BASEBRANCH="refs/tags/$BASEBRANCH" fi fi fi # Load custom setup additions if [ -d "${BASEDIR}/data/environment.d" ]; then for envfile in ${BASEDIR}/data/environment.d/*.sh ; do . $envfile done fi # We need to reparse the arguments as we've now loaded the environment.d # extensions help=0 parse_arguments "$@" if [ $help -ne 1 ]; then # Before doing anything else, error out if the project directory # is unsafe. case "$PWD" in $BASEDIR | $BASEDIR/*) echo >&2 "$0: The current working directory is used as your project directory" echo >&2 " Your project directory must not be in or under" echo >&2 " '${BASEDIR}'" echo >&2 "" echo >&2 " Typically a project is setup by doing:" echo >&2 " $ mkdir my_project" echo >&2 " $ cd my_project" echo >&2 " $ git clone --branch $BASEBRANCH $REMOTEURL" echo >&2 " $ .$(echo $REMOTEURL | sed "s,$BASEURL,,")/setup.sh $@" exit 1 ;; esac if [ -z "$BASEBRANCH" ]; then echo "May be on a detached HEAD, HEAD must be on a branch or tag. ($BASEDIR)" >&2 echo "You can avoid this by passing the branch using --base-branch=" >&2 exit 1 fi # Is this a tag? If so, don't allow tags w/ '-' if [ "$BASEBRANCH" != "${BASEBRANCH##refs/tags/}" ]; then if [ "$BASEBRANCH" != "${BASEBRANCH//-*}" ]; then echo "Checkout may be on a detached HEAD, this HEAD does not appear to" >&2 echo "correspond to a specific, tag. (It appears you may be working with" >&2 echo "tag ${BASEBRANCH//-*}. If this is correct, use" >&2 echo "--base-branch=${BASEBRANCH//-*} in the arguments to" >&2 echo "$0." exit 1 fi fi for func in "${ADDFUNCS[@]}"; do $func rc=$? if [ $rc -ne 0 ]; then echo "Stopping: an error occurred in $func." >&2 shutdown exit $rc fi done # Configure the current directory so repo works seemlessly add_gitconfig "user.name" "${GIT_USERNAME}" add_gitconfig "user.email" "${GIT_USEREMAIL}" add_gitconfig "color.ui" "false" add_gitconfig "color.diff" "false" add_gitconfig "color.status" "false" fi # if help -ne 1 # We potentially have code that doesn't parse correctly with older versions # of Python, and rather than fixing that and being eternally vigilant for # any other new feature use, just check the version here. py_v35_check=$(python3 -c 'import sys; print(sys.version_info >= (3,5,0))') if [ "$py_v35_check" != "True" ]; then echo >&2 "BitBake requires Python 3.5.0 or later as 'python3 (scripts/install-buildtools can be used if needed)'" return 1 fi unset py_v35_check # This can happen if python3/urllib was not built with SSL support. python3 -c 'import urllib.request ; dir(urllib.request.HTTPSHandler)' >/dev/null 2>&1 if [ $? -ne 0 ]; then echo >&2 "The setup tool requires Python 3.4.0 or later with support for 'urllib.request.HTTPSHandler'" exit 1 fi # Python 3 required utf-8 support to work properly, adjust the LANG to en_US.UTF-8. export LANG='en_US.UTF-8' # Pass the computed url and branch to ${cmd} export OE_BASEURL=${BASEURL} export OE_BASEBRANCH=${BASEBRANCH} for func in "${EXPORTFUNCS[@]}"; do $func rc=$? if [ $rc -ne 0 ]; then echo "Stopping: an error occurred in $func." >&2 shutdown exit $rc fi done trap - INT # Switch to the python script ${BASEDIR}/${CMD} "${PASSARGS[@]}" rc=$? shutdown # Preserve the return code from the python script exit $rc