Repository: H-M-H/Weylus Branch: master Commit: 38a01a8f8e42 Files: 69 Total size: 414.1 KB Directory structure: gitextract_cqulqu0f/ ├── .clang-format ├── .github/ │ ├── FUNDING.yml │ └── workflows/ │ └── build.yml ├── .gitignore ├── CONTRIBUTORS ├── Cargo.toml ├── LICENSE ├── Readme.md ├── build.rs ├── build_in_local_container.sh ├── compile_flags.txt ├── deps/ │ ├── awk.patch │ ├── build.sh │ ├── clean.sh │ ├── clean_all.sh │ ├── command_limit.patch │ ├── download.sh │ ├── ffmpeg.sh │ ├── hashes.sh │ ├── libva.sh │ ├── nv-codec-headers.sh │ ├── refs.sh │ ├── update_hashes.sh │ └── x264.sh ├── docker/ │ ├── Dockerfile │ └── Dockerfile_alpine ├── docker_build.sh ├── lib/ │ ├── encode_video.c │ ├── error.c │ ├── error.h │ ├── linux/ │ │ ├── uinput.c │ │ ├── uinput_info.md │ │ ├── xcapture.c │ │ ├── xhelper.c │ │ └── xhelper.h │ ├── log.c │ └── log.h ├── src/ │ ├── capturable/ │ │ ├── captrs_capture.rs │ │ ├── core_graphics.rs │ │ ├── mod.rs │ │ ├── pipewire.rs │ │ ├── remote_desktop_dbus.rs │ │ ├── testsrc.rs │ │ ├── win_ctx.rs │ │ └── x11.rs │ ├── cerror.rs │ ├── config.rs │ ├── gui.rs │ ├── input/ │ │ ├── autopilot_device.rs │ │ ├── autopilot_device_win.rs │ │ ├── device.rs │ │ ├── mod.rs │ │ ├── uinput_device.rs │ │ └── uinput_keys.rs │ ├── log.rs │ ├── main.rs │ ├── protocol.rs │ ├── strings/ │ │ └── uinput_error.txt │ ├── video.rs │ ├── web.rs │ ├── websocket.rs │ └── weylus.rs ├── ts/ │ └── lib.ts ├── tsconfig.json ├── weylus.desktop ├── weylus_tls.sh └── www/ ├── static/ │ ├── access_code.html │ └── style.css └── templates/ └── index.html ================================================ FILE CONTENTS ================================================ ================================================ FILE: .clang-format ================================================ Language: Cpp Standard: Latest BasedOnStyle: LLVM ColumnLimit: 100 IndentWidth: 4 TabWidth: 4 UseTab: ForContinuationAndIndentation BreakBeforeBraces: Allman AccessModifierOffset: -4 PointerAlignment: Left AlignAfterOpenBracket: AlwaysBreak BinPackArguments: false BinPackParameters: false ================================================ FILE: .github/FUNDING.yml ================================================ github: H-M-H liberapay: HMH ================================================ FILE: .github/workflows/build.yml ================================================ name: Build on: push: branches: [ '*' ] tags: - v* pull_request: branches: [ master ] jobs: build-docker: runs-on: ubuntu-latest container: docker://hhmhh/weylus_build:latest steps: - uses: actions/checkout@v6 - uses: actions/cache@v5 with: path: deps/dist* key: ${{ runner.os }}-deps-${{ hashFiles('deps/*.sh', 'deps/*.patch') }} - uses: actions/cache@v5 with: path: | ~/.cargo/registry ~/.cargo/git target key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} - name: Build run: ./docker_build.sh shell: bash - name: Artifacts1 uses: actions/upload-artifact@v6 with: name: linux path: packages/weylus-linux.zip - name: Artifacts2 uses: actions/upload-artifact@v6 with: name: linux-deb path: packages/Weylus*.deb - name: Artifacts3 uses: actions/upload-artifact@v6 with: name: windows path: packages/weylus-windows.zip - name: Publish uses: softprops/action-gh-release@v2 if: startsWith(github.ref, 'refs/tags/') with: files: | packages/weylus-linux.zip packages/Weylus*.deb packages/weylus-windows.zip prerelease: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} build-docker-alpine: runs-on: ubuntu-latest container: docker://hhmhh/weylus_build_alpine:latest steps: - uses: actions/checkout@v6 - uses: actions/cache@v5 with: path: deps/dist* key: ${{ runner.os }}-alpine-deps-${{ hashFiles('deps/*.sh', 'deps/*.patch') }} - uses: actions/cache@v5 with: path: | ~/.cargo/registry ~/.cargo/git target key: ${{ runner.os }}-alpine-cargo-${{ hashFiles('Cargo.lock') }} - name: Build run: RUSTFLAGS='-C target-feature=-crt-static' cargo build --release && cd target/release && tar czf weylus-linux-alpine-musl.tar.gz weylus shell: bash - name: Artifacts1 uses: actions/upload-artifact@v6 with: name: linux-alpine-musl path: target/release/weylus-linux-alpine-musl.tar.gz - name: Publish uses: softprops/action-gh-release@v2 if: startsWith(github.ref, 'refs/tags/') with: files: | target/release/weylus-linux-alpine-musl.tar.gz prerelease: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} build-macos: strategy: matrix: os: [macos-latest, macos-15-intel] # -latest is for Apple Silicon, -15-intel is for Intel runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v6 - uses: actions/cache@v5 with: path: deps/dist key: ${{ runner.os }}-deps-${{ hashFiles('deps/*.sh', 'deps/*.patch') }} - uses: actions/cache@v5 with: path: | ~/.cargo/registry ~/.cargo/git target key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} - name: Download deps run: | npm install -g typescript brew install nasm cargo install cargo-bundle shell: bash - name: Build # run: MACOSX_DEPLOYMENT_TARGET=10.13 cargo bundle --release run: cargo bundle --release - name: Package run: | MACOS_BUILD_NAME=macos-$([ "${{ matrix.os }}" == "macos-latest" ] && echo "arm" || echo "intel") echo "MACOS_BUILD_NAME=$MACOS_BUILD_NAME" >> $GITHUB_ENV cd target/release/bundle/osx/ && zip -r ${MACOS_BUILD_NAME}.zip Weylus.app - name: Artifacts uses: actions/upload-artifact@v6 with: name: ${{ env.MACOS_BUILD_NAME }} path: target/release/bundle/osx/${{ env.MACOS_BUILD_NAME }}.zip - name: ArtifactsDebug if: failure() uses: actions/upload-artifact@v6 with: name: ${{ runner.os }}-ffbuild path: | deps/ffmpeg/ffbuild - name: Debug via SSH if: failure() uses: luchihoratiu/debug-via-ssh@main with: NGROK_AUTH_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }} SSH_PASS: ${{ secrets.SSH_PASS }} NGROK_REGION: eu - name: Publish uses: softprops/action-gh-release@v2 if: startsWith(github.ref, 'refs/tags/') with: files: | target/release/bundle/osx/macOS.zip prerelease: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .gitignore ================================================ /.vscode /target /deps/ffmpeg /deps/x264 /deps/dist* /deps/nv-codec-headers /deps/libva c_helper/target c_helper/Cargo.lock *.js *.js.map *.tar.gz ================================================ FILE: CONTRIBUTORS ================================================ If you want to contribute to Weylus you have to agree to license your contributions under the 3-Clause BSD License. To do so please add yourself to the list of contributors below. List of Contributors: ************************** Robert Schroll Daniel Rutz Philipp Urlbauer OmegaRogue ************************** 3-Clause BSD License Copyright 2020-Present above Contributors. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: Cargo.toml ================================================ [package] name = "weylus" version = "0.11.4" authors = ["HMH "] license = "AGPL-3.0-or-later" edition = "2021" description = "Use your iPad or Android tablet as graphic tablet." [dependencies] autopilot = { git = "https://github.com/H-M-H/autopilot-rs.git", rev = "63eed09c715bfb665bb23172a3930a528e11691c" } bitflags = { version = "^2.6", features = ["serde"] } bytes = "1.7.1" clap = { version = "4.5.18", features = ["derive"] } clap_complete = "4.5.29" dirs = "^5.0" fastwebsockets = { version = "0.8.0", features = ["upgrade", "unstable-split"] } fltk = { version = "^1.5", features = ["use-wayland"] } fltk-theme = "^0.7.9" handlebars = "^6.1" http-body-util = "0.1.2" hyper = { version = "^1.4", features = ["server", "http1", "http2"] } hyper-util = { version = "0.1.8", features = ["tokio"] } image = { version = "^0.25", features = ["png"], default-features = false } image_autopilot = { package = "image", version = "0.22.5", features = [], default-features = false } percent-encoding = "2.1.0" qrcode = "0.14.0" rand = "0.8.5" serde = { version = "^1.0", features = ["derive"] } serde_json = "^1.0" signal-hook = "0.3.17" tokio = { version = "^1", features = ["fs", "macros", "rt-multi-thread", "sync", "net"] } toml = "^0.9" tracing = "^0.1" tracing-subscriber = { version = "^0.3", features = ["ansi", "json"], default-features = false } url = "^2.5" [target.'cfg(windows)'.dependencies] winapi = { version = "0.3.9", features = ["d3d11", "d3dcommon", "dxgi", "dxgi1_2", "dxgitype"] } wio = "0.2.2" captrs = "^0.3.1" [build-dependencies] cc = "^1.1" num_cpus = "^1.16" [target.'cfg(target_os = "linux")'.dependencies] dbus = "^0.9" gstreamer = "^0.24" gstreamer-app = { version = "^0.24", features = ["v1_16"] } gstreamer-video = "^0.24" [target.'cfg(not(target_os = "windows"))'.dependencies] pnet_datalink = "^0.35" [target.'cfg(target_os = "macos")'.dependencies] core-foundation = "^0.10" core-graphics = "^0.24" [features] bench = [] ffmpeg-system = [] va-static = [] [package.metadata.bundle] name = "Weylus" identifier = "io.github.h-m-h.weylus" [package.metadata.deb] name = "Weylus" section = "graphics" priority = "optional" assets = [ ["target/release/weylus", "usr/bin/weylus", "755"], ["weylus.desktop", "usr/share/applications/weylus.desktop", "755"], ["Readme.md", "usr/share/doc/weylus/README", "644"], ] [profile.release] lto = true opt-level = 3 ================================================ FILE: LICENSE ================================================ Weylus is licensed under the GNU Affero General Public License, version 3 or later. Contributed work is licensed under the 3-Clause BSD License. See CONTRIBUTORS for details. Copyright (C) 2020-Present H-M-H GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU Affero General Public License is a free, copyleft license for software and other kinds of works, specifically designed to ensure cooperation with the community in the case of network server software. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. Developers that use our General Public Licenses protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License which gives you legal permission to copy, distribute and/or modify the software. A secondary benefit of defending all users' freedom is that improvements made in alternate versions of the program, if they receive widespread use, become available for other developers to incorporate. Many developers of free software are heartened and encouraged by the resulting cooperation. However, in the case of software used on network servers, this result may fail to come about. The GNU General Public License permits making a modified version and letting the public access it on a server without ever releasing its source code to the public. The GNU Affero General Public License is designed specifically to ensure that, in such cases, the modified source code becomes available to the community. It requires the operator of a network server to provide the source code of the modified version running there to the users of that server. Therefore, public use of a modified version, on a publicly accessible server, gives the public access to the source code of the modified version. An older license, called the Affero General Public License and published by Affero, was designed to accomplish similar goals. This is a different license, not a version of the Affero GPL, but Affero has released a new version of the Affero GPL which permits relicensing under this license. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU Affero General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Remote Network Interaction; Use with the GNU General Public License. Notwithstanding any other provision of this License, if you modify the Program, your modified version must prominently offer all users interacting with it remotely through a computer network (if your version supports such interaction) an opportunity to receive the Corresponding Source of your version by providing access to the Corresponding Source from a network server at no charge, through some standard or customary means of facilitating copying of software. This Corresponding Source shall include the Corresponding Source for any work covered by version 3 of the GNU General Public License that is incorporated pursuant to the following paragraph. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the work with which it is combined will remain governed by version 3 of the GNU General Public License. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU Affero General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements. You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see . ================================================ FILE: Readme.md ================================================ # Weylus ![Build](https://github.com/H-M-H/Weylus/workflows/Build/badge.svg) Weylus turns your tablet or smart phone into a graphic tablet/touch screen for your computer! Weylus in action with [Xournal++](https://github.com/xournalpp/xournalpp): ![Weylus in action](In_action.gif) ## Table of Contents * [Features](#features) * [Installation](#installation) * [Packages](#packages) * [Running](#running) * [Fullscreen](#fullscreen) * [Keyboard Input](#keyboard-input) * [Automation](#automation) * [Linux](#linux) * [Wayland](#wayland) * [Hardware Acceleration](#hardware-acceleration) * [Weylus as Second Screen](#weylus-as-second-screen) * [Intel GPU on Xorg with Intel drivers](#intel-gpu-on-xorg-with-intel-drivers) * [Dummy Plugs](#dummy-plugs) * [Other Options](#other-options) * [Encryption](#encryption) * [macOS](#macos) * [Hardware Acceleration](#hardware-acceleration-1) * [Windows](#windows) * [Hardware Acceleration](#hardware-acceleration-2) * [Building](#building) * [Docker](#docker) * [How does this work?](#how-does-this-work) * [Stylus/Touch](#stylustouch) * [Screen mirroring & window capturing](#screen-mirroring--window-capturing) * [FAQ](#faq) ## Features - Control your mouse with your tablet - Mirror your screen to your tablet - Send keyboard input using physical keyboards - Hardware accelerated video encoding The above features are available on all Operating Systems but Weylus works best on Linux. Additional features on Linux are: - Support for a stylus/pen (supports pressure and tilt) - Multi-touch: Try it with software that supports multi-touch, like Krita, and see for yourself! - Capturing specific windows and only drawing to them - Faster screen mirroring - Tablet as second screen ## Installation Just grab the latest release for your OS from the [releases page](https://github.com/H-M-H/Weylus/releases) and install it on your computer. No apps except a modern browser (Firefox 80+, iOS/iPadOS 13+) are required on your tablet. **If you run Linux make sure to follow the instructions described [here](#linux) to enable uinput for features like pressure sensitivity and multitouch!** ### Packages AUR packages for Weylus are available here: - From source: [weylus](https://aur.archlinux.org/packages/weylus/) - Prebuilt binary: [weylus-bin](https://aur.archlinux.org/packages/weylus-bin/) ## Running Start Weylus, preferably set an access code in the access code box and press the Start button. This will start a webserver running on your computer. To control your computer with your tablet you need to open the url `http://
:`, if possible Weylus will display to you the url you need to open and show a QR code with the encoded address. If you have a firewall running make sure to open a TCP port for the webserver (1701 by default) and the websocket connection (9001 by default). On many Linux distributions this is done with ufw: ``` sudo ufw allow 1701/tcp sudo ufw allow 9001/tcp ``` Please only run Weylus in networks you trust as there is no encryption to enable minimal latencies. ### Fullscreen You may want to add a bookmark to your home screen on your tablet as this enables running Weylus in full screen mode (on iOS/iPadOS this needs to be done with Safari). If you are not on iOS/iPadOS there is a button to toggle full screen mode. ### Keyboard Input Weylus supports keyboard input for physical keyboards, so if you have a Bluetooth keyboard, just connect it to your tablet and start typing. Due to technical limitations onscreen keyboards are not supported. ### Automation Weylus provides some features to make automation as convenient as possible. There is a command-line interface; `--no-gui` for example starts Weylus in headless mode without a gui. For more options see `weylus --help`. If you want to run a specific script e.g., once a client connects to your computer you can do so by parsing the log Weylus generates. You may want to enable more verbose logging by setting the environment variable `WEYLUS_LOG_LEVEL` to `DEBUG` or `TRACE` as well as `WEYLUS_LOG_JSON` to `true` to enable easily parseable JSON logging. ### Linux Weylus uses the `uinput` interface to simulate input events on Linux. **To enable stylus and multi-touch support `/dev/uinput` needs to be writable by Weylus.** To make `/dev/uinput` permanently writable by your user, run: ```sh sudo groupadd -r uinput sudo usermod -aG uinput $USER echo 'KERNEL=="uinput", MODE="0660", GROUP="uinput", OPTIONS+="static_node=uinput"' \ | sudo tee /etc/udev/rules.d/60-weylus.rules ``` Then, either reboot, or run ```sh sudo udevadm control --reload sudo udevadm trigger ``` then log out and log in again. To undo this, run: ```sh sudo rm /etc/udev/rules.d/60-weylus.rules ``` This allows your user to synthesize input events system-wide, even when another user is logged in. Therefore, untrusted users should not be added to the uinput group. #### Wayland Weylus offers experimental support for Wayland. Installing `pipewire` and `xdg-desktop-portal` as well as one of: - `xdg-desktop-portal-gtk` for GNOME - `xdg-desktop-portal-kde` for KDE - `xdg-desktop-portal-wlr` for wlroots-based compositors like Sway is required. There are still some things that do not work: - input mapping for windows - displaying proper window names - capturing the cursor #### Hardware Acceleration On Linux Weylus supports hardware accelerated video encoding through the Video Acceleration API (VAAPI) or Nvidia's NVENC. By default hardware acceleration is disabled as quality and stability of the hardware encoded video stream varies widely among different hardware and sufficient quality can not be guaranteed. If VAAPI is used it is possible to select a specific driver by setting the environment variable `LIBVA_DRIVER_NAME`. You can find possible values with the command `ls /usr/lib/dri/ | sed -n 's/^\(\S*\)_drv_video.so$/\1/p'`. On some distributions the drivers may not reside in `/usr/lib/dri` but for example in `/usr/lib/x86_64-linux-gnu/dri` and may not be found by Weylus. To force Weylus to search another directory for drivers, the environment variable `LIBVA_DRIVERS_PATH` can be set. Additionally you can specify the VAAPI device to use by setting `WEYLUS_VAAPI_DEVICE`; by default devices can be found in `/dev/dri`. On some systems this is not optional and this variable must be set. If VAAPI doesn't work out of the box for you, have a look into `/dev/dri`, often setting `WEYLUS_VAAPI_DEVICE=/dev/dri/renderD129` is already the solution. Note that you may need to install the driver(s) first. Nvidias NVENC is very fast but delivers a video stream of noticeably lower quality (at least on my GeForce GTX 1050 Mobile GPU) but more recent GPUs should provide higher quality. For this to work nvidia drivers need to be installed. #### Weylus as Second Screen There are a few possibilities to use Weylus to turn your tablet into a second screen. ##### Intel GPU on Xorg with Intel drivers Intel's drivers support creating virtual outputs that can be configured via xrandr. But first a word of warning: The following configuration may break starting the X server. This means you might end up without a graphical login or X may get stuck and just display a black screen. So make sure you know what you are doing or are at least able to recover from a broken X server. You will need to install the `xf86-video-intel` driver and create the file `/etc/X11/xorg.conf.d/20-intel.conf` with the following contents: ```text Section "Device" Identifier "intelgpu0" Driver "intel" # this adds two virtual monitors / devices Option "VirtualHeads" "2" # if your screen is flickering one of the following options might help # Option "TripleBuffer" "true" # Option "TearFree" "true" # Option "DRI" "false" EndSection ``` After a reboot `xrandr` will show two additional monitors `VIRTUAL1` and `VIRTUAL2` and can be used to configure them. To activate `VIRTUAL1` with a screen size of 1112x834 and a refresh rate of 60 fps the following commands can be used: ```console > # this generates all input parameters xrandr needs > #from a given screen resolution and refresh rate > gtf 1112 834 60 # 1112x834 @ 60.00 Hz (GTF) hsync: 51.78 kHz; pclk: 75.81 MHz Modeline "1112x834_60.00" 75.81 1112 1168 1288 1464 834 835 838 863 -HSync +Vsync > # setup the monitor > xrandr --newmode "1112x834_60.00" 75.81 1112 1168 1288 1464 834 835 838 863 -HSync +Vsync > xrandr --addmode VIRTUAL1 1112x834_60.00 > xrandr --output VIRTUAL1 --mode 1112x834_60.00 > # check if everything is in order > xrandr ``` Now you should be able to configure this monitor in your system setting like a regular second monitor and for example set its position relative to your primary monitor. After setting up the virtual monitor start Weylus and select it in the capture menu. You may want to enable displaying the cursor in this case. That is it! ##### Dummy Plugs Weylus detects if you use multiple monitors and you can select the one you want to mirror. So if you want to use Weylus as a second screen you could just buy another monitor. Obviously this is pointless as if you already bought that monitor, there is no need to use Weylus! This is where so called **HDMI/Displayport/VGA Dummy Plugs** come in handy. These are small devices that pretend to be a monitor but only cost a fraction of the price of an actual monitor. Once you have bought one and plugged it into your computer you can configure an additional screen just like you would do with an actual one and then use Weylus to mirror this virtual screen. ##### Other Options The following is untested/incomplete, feel free to do more research and open a pull request to expand documentation on this! - On Wayland with sway there is `create_output` which can be used to [create headless outputs](https://github.com/swaywm/sway/releases/tag/1.5), unfortunately it is not documented how to actually do that: https://github.com/swaywm/sway/issues/5553 - On Wayland with GNOME recently there has been added an option to [create virtual monitors with mutter](https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/1698) #### Encryption By default Weylus comes without encryption and should only be run on networks you trust. If this is not the case it's strongly advised to set up a TLS proxy. One option is to use [hitch](https://hitch-tls.org/), an example script that sets up encryption is located at `weylus_tls.sh`. But any TLS proxy should work just fine. Note that the mentioned script works by creating a self-signed certificate. This means your browser will most likely display a scary looking but completely unfounded message telling you how incredibly dangerous it is to trust the certificate you yourself just created; this can be safely ignored! In case you are using Firefox: There is a [bug](https://bugzilla.mozilla.org/show_bug.cgi?id=1187666) that prevents users from accepting self-signed certificates for websocket connections. A workaround is to directly open the websocket connection via the URL bar and accept the certificate there. After accepting the connection will of course fail as the browser expects https and not wss as protocol. Sadly this solution is anything but frictionless and I am unhappy with the current state of affairs. This is also another reason why encryption is not enabled by default, self-signed certificates are just too painful to handle nowadays. I'd gladly welcome any proposals to improve the situation! ### macOS Weylus needs some permissions to work properly, make sure you enable: - Incoming connections - Screen capturing - Controlling your desktop #### Hardware Acceleration Weylus can make use of the Videotoolbox framework on macOS for hardware acceleration. In my tests the video quality has been considerably worse than that using software encoding and thus Videotoolbox is disabled by default. ### Windows #### Hardware Acceleration Weylus can make use of Nvidias NVENC as well as Microsoft's MediaFoundation for hardware accelerated video encoding. Due to widely varying quality it is disabled by default. ## Building To build Weylus you need to install Rust, Typescript, make, git, a C compiler, nasm and bash. `cargo build` builds the project. By default Weylus is build in debug mode, if you want a release build run `cargo build --release`. On Linux some additional dependencies are required to build Weylus. On Debian or Ubuntu they can be installed via: ```sh apt-get install -y libx11-dev libxext-dev libxft-dev libxinerama-dev libxcursor-dev libxrender-dev \ libxfixes-dev libxtst-dev libxrandr-dev libxcomposite-dev libxi-dev libxv-dev autoconf libtool-bin \ nvidia-cuda-dev pkg-config libdrm-dev libpango1.0-dev libgstreamer1.0-dev \ libgstreamer-plugins-base1.0-dev libdbus-1-dev ``` On Fedora, they can be installed via: ```sh sudo dnf install libXext-devel libXft-devel libXinerama-devel libXcursor-devel libXrender-devel \ libXfixes-devel libXtst-devel libXrandr-devel libXcomposite-devel libXi-devel libXv-devel autoconf libtool \ pkg-config libdrm-devel pango-devel gstreamer1-devel \ gstreamer1-plugins-base-devel dbus-devel nasm npm ``` After npm is installed, typescript must be installed by: ```sh sudo npm install typescript -g ``` Note that building for the first time may take a while as by default ffmpeg needs to be built. On Windows only msvc is supported as C compiler; it is, however, possible to cross compile on Linux for Windows using minGW. In case you do not want to build ffmpeg and libx264 via the supplied build script you can create the directory `deps/dist` yourself and copy static ffmpeg libraries built with support for libx264 and a static version of libx264 into `deps/dist/lib`. Additional `deps/dist/include` needs to be filled with ffmpeg's include header files. For hardware acceleration to work ffmpeg needs to be built with additional flags depending on your OS: Consult the variable `FFMPEG_EXTRA_ARGS` in `deps/build.sh` for details. Furthermore, for VAAPI on Linux a static version of libva is required as well. The build script will only try to build ffmpeg if the directory `deps/dist` does not exist. Alternatively passing `--features ffmpeg-system` to cargo will build Weylus using the system's version of ffmpeg. This is disabled by default for compatibility reasons, on newer systems this should not pose a problem and using the system libraries is advised. ### Docker It is also possible to build the Linux version inside a docker container. The Dockerfile used is located at [docker/Dockerfile](docker/Dockerfile). This is also how the official release is built. Building works like this: ```console docker run -it hhmhh/weylus_build bash root@f02164dbfa18:/# git clone https://github.com/H-M-H/Weylus Cloning into 'Weylus'... remote: Enumerating objects: 10, done. remote: Counting objects: 100% (10/10), done. remote: Compressing objects: 100% (7/7), done. remote: Total 827 (delta 1), reused 6 (delta 0), pack-reused 817 Receiving objects: 100% (827/827), 5.38 MiB | 7.12 MiB/s, done. Resolving deltas: 100% (431/431), done. root@f02164dbfa18:/# cd Weylus/ root@f02164dbfa18:/Weylus# cargo deb Compiling ... ``` Once the build is finished you can for example copy the binary from the container to your file system like this: ```sh docker cp f02164dbfa18:/Weylus/target/release/weylus ~/some/path/weylus ``` The .deb is located at `/Weylus/target/debian/`. Please note that the container ID will most likely not be `f02164dbfa18` if you run this yourself, replace it accordingly. ## How does this work? ### Stylus/Touch Modern browsers expose so called [PointerEvents](https://developer.mozilla.org/en-US/docs/Web/API/PointerEvent) that can convey not only mouse but additionally stylus/pen and touch information. Weylus sets up a webserver with the corresponding javascript code to capture these events. The events are sent back to the server using websockets. Weylus then processes these events using either the generic OS independent backend, which only supports controlling the mouse or on Linux the uinput backend can be used. It makes use of the uinput Linux kernel module which supports creating a wide range of input devices including mouse, stylus and touch input devices. ### Screen mirroring & window capturing Either the generic backend is used which is less efficient and only captures the whole screen or on Linux xlib is used to connect to the X-server and do the necessary work of getting window information and capturing the window/screen. To make things fast the "MIT-SHM - The MIT Shared Memory Extension" is used to create shared memory images using `XShmCreateImage`. If Wayland instead of X11 is running, PipeWire and GStreamer is used to capture the screen. The images captured are then encoded to a video stream using ffmpeg. Fragmented MP4 is used as container format to enable browsers to play the stream via the Media Source Extensions API. The video codec used is H.264 as this is widely supported and allows very fast encoding as opposed to formats like AV1. To minimize dependencies ffmpeg is statically linked into Weylus. ## FAQ Q: Why does the page not load on my tablet and instead I get a timeout?
A: There probably is some kind of firewall running, make sure the ports Weylus uses are opened. Q: Why do I get the error `ERROR Failed to create uinput device: CError: code...`?
A: uinput is probably misconfigured, have you made sure to follow all instructions and logged out and in again? You may also be running a very old kernel that does not support the required features. In that case try to upgrade your system or use a newer one. Q: Why is the "Capture" drop down empty and the screen not mirrored?
A: It is possible that only the port for the webserver but not the websocket has been opened, check that both ports have been opened. Q: Why can I not select any windows in the "Capture" drop down and only see the whole screen.
A: If you are running Weylus on MacOS or Windows this feature is unfortunately not implemented. On Linux it is possible that your window manager does not support [Extended Window Manager Hints](https://specifications.freedesktop.org/wm-spec/latest/) or that you need to activate them first, like for XMonad. Q: Do I have to follow the instructions to setup Weylus as second screen too?
A: No, this is strictly optional. Q: Why am I unable to connect my tablet to the URL displayed by Weylus?
A: It is possible that your computer and WiFi connected tablet are on different networks, make sure they are on the same network. Q: Why does this not run on Firefox for Android?
A: Actually it does, just make sure Firefox version 80+ is installed. Q: Why does this not run under Chrome on my iPad?
A: Chrome lacks some features for video streaming on iPadOS/iOS, try Firefox or Safari. Q: Why won't my cursor move in osu! ?
A: Try disabling raw input. Q: Can I use Weylus even if there is no WiFi?
A: Probably yes! Most tablets permit setting up a WiFi hotspot that can be used to connect your computer and tablet. Alternatively there is USB tethering too, which can be used to setup a peer to peer connection between your tablet and computer over USB. Another method for Android devices is to setup a socket connection with [adb](https://developer.android.com/studio/command-line/adb#Enabling): ```console adb reverse tcp:1701 tcp:1701 adb reverse tcp:9001 tcp:9001 ``` Like that you can connect from your Android device to Weylus with the URL: `http://127.0.0.1:1701`. Weylus only requires that your devices are connected via the Internet Protocol and that doesn't necessarily imply WiFi. --- [![Packaging status]( https://repology.org/badge/vertical-allrepos/weylus.svg )](https://repology.org/project/weylus/versions) ================================================ FILE: build.rs ================================================ use std::env; use std::path::Path; use std::process::Command; fn build_ffmpeg(dist_dir: &Path, enable_libnpp: bool) { if dist_dir.exists() { return; } Command::new("bash") .arg(Path::new("clean.sh")) .current_dir("deps") .status() .expect("Failed to clean ffmpeg build!"); if !Command::new("bash") .arg(Path::new("build.sh")) .current_dir("deps") .env("DIST", dist_dir) .env("ENABLE_LIBNPP", if enable_libnpp { "y" } else { "n" }) .status() .expect("Failed to run bash!") .success() { println!("cargo:warning=Failed to build ffmpeg!"); std::process::exit(1); } } fn main() { let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); let dist_dir = Path::new("deps") .canonicalize() .unwrap() .join(format!("dist_{}", target_os)); let enable_libnpp = env::var("I_AM_BUILDING_THIS_AT_HOME_AND_WANT_LIBNPP").map_or(false, |v| { ["y", "yes", "true", "1"].contains(&v.to_lowercase().as_str()) }); if env::var("CARGO_FEATURE_FFMPEG_SYSTEM").is_err() { build_ffmpeg(&dist_dir, enable_libnpp); } println!("cargo:rerun-if-changed=ts/lib.ts"); #[cfg(not(target_os = "windows"))] let mut tsc_command = Command::new("tsc"); #[cfg(target_os = "windows")] let mut tsc_command = Command::new("bash"); #[cfg(target_os = "windows")] tsc_command.args(&["-c", "tsc"]); let js_needs_update = || -> Result> { Ok(Path::new("ts/lib.ts").metadata()?.modified()? > Path::new("www/static/lib.js").metadata()?.modified()?) }() .unwrap_or(true); if js_needs_update { match tsc_command.status() { Err(err) => { println!("cargo:warning=Failed to call tsc: {}", err); std::process::exit(1); } Ok(status) => { if !status.success() { match status.code() { Some(code) => println!("cargo:warning=tsc failed with exitcode: {}", code), None => println!("cargo:warning=tsc terminated by signal."), }; std::process::exit(2); } } } } println!("cargo:rerun-if-changed=lib/encode_video.c"); let mut cc_video = cc::Build::new(); cc_video.file("lib/encode_video.c"); cc_video.include(dist_dir.join("include")); if ["linux", "windows"].contains(&target_os.as_str()) { cc_video.define("HAS_NVENC", None); } if target_os == "linux" { cc_video.define("HAS_VAAPI", None); } if target_os == "macos" { cc_video.define("HAS_VIDEOTOOLBOX", None); } if target_os == "windows" { cc_video.define("HAS_MEDIAFOUNDATION", None); } if enable_libnpp { cc_video.define("HAS_LIBNPP", None); } cc_video.compile("video"); println!("cargo:rerun-if-changed=lib/error.h"); println!("cargo:rerun-if-changed=lib/error.c"); println!("cargo:rerun-if-changed=lib/log.h"); println!("cargo:rerun-if-changed=lib/log.c"); cc::Build::new().file("lib/error.c").compile("error"); cc::Build::new().file("lib/log.c").compile("log"); let ffmpeg_link_kind = // https://github.com/rust-lang/rust/pull/72785 // https://users.rust-lang.org/t/linking-on-windows-without-wholearchive/49846/3 if cfg!(target_os = "windows") || env::var("CARGO_FEATURE_FFMPEG_SYSTEM").is_ok() { "dylib" } else { "static" }; println!("cargo:rustc-link-lib={}=avdevice", ffmpeg_link_kind); println!("cargo:rustc-link-lib={}=avformat", ffmpeg_link_kind); println!("cargo:rustc-link-lib={}=avfilter", ffmpeg_link_kind); println!("cargo:rustc-link-lib={}=avcodec", ffmpeg_link_kind); println!("cargo:rustc-link-lib={}=swresample", ffmpeg_link_kind); println!("cargo:rustc-link-lib={}=swscale", ffmpeg_link_kind); println!("cargo:rustc-link-lib={}=avutil", ffmpeg_link_kind); println!("cargo:rustc-link-lib={}=x264", ffmpeg_link_kind); if enable_libnpp { if let Ok(lib_paths) = env::var("LIBRARY_PATH") { for lib_path in lib_paths.split(':') { println!("cargo:rustc-link-search={}", lib_path); } } println!("cargo:rustc-link-lib=dylib=nppig"); println!("cargo:rustc-link-lib=dylib=nppicc"); println!("cargo:rustc-link-lib=dylib=nppc"); println!("cargo:rustc-link-lib=dylib=nppidei"); println!("cargo:rustc-link-lib=dylib=nppif"); } if env::var("CARGO_FEATURE_FFMPEG_SYSTEM").is_err() { println!( "cargo:rustc-link-search={}", dist_dir.join("lib").to_string_lossy() ); } if target_os == "linux" { linux(); } if target_os == "macos" { println!("cargo:rustc-link-lib=framework=VideoToolbox"); println!("cargo:rustc-link-lib=framework=CoreMedia"); } if target_os == "windows" { println!("cargo:rustc-link-lib=dylib=mfplat"); println!("cargo:rustc-link-lib=dylib=mfuuid"); println!("cargo:rustc-link-lib=dylib=ole32"); println!("cargo:rustc-link-lib=dylib=strmiids"); println!("cargo:rustc-link-lib=dylib=vfw32"); println!("cargo:rustc-link-lib=dylib=shlwapi"); println!("cargo:rustc-link-lib=dylib=bcrypt"); } } fn linux() { println!("cargo:rerun-if-changed=lib/linux/uniput.c"); println!("cargo:rerun-if-changed=lib/linux/xcapture.c"); println!("cargo:rerun-if-changed=lib/linux/xhelper.c"); println!("cargo:rerun-if-changed=lib/linux/xhelper.h"); cc::Build::new() .file("lib/linux/uinput.c") .file("lib/linux/xcapture.c") .file("lib/linux/xhelper.c") .compile("linux"); println!("cargo:rustc-link-lib=X11"); println!("cargo:rustc-link-lib=Xext"); println!("cargo:rustc-link-lib=Xrandr"); println!("cargo:rustc-link-lib=Xfixes"); println!("cargo:rustc-link-lib=Xcomposite"); println!("cargo:rustc-link-lib=Xi"); let va_link_kind = if env::var("CARGO_FEATURE_VA_STATIC").is_ok() { "static" } else { "dylib" }; println!("cargo:rustc-link-lib={}=va", va_link_kind); println!("cargo:rustc-link-lib={}=va-drm", va_link_kind); println!("cargo:rustc-link-lib={}=va-x11", va_link_kind); println!("cargo:rustc-link-lib=drm"); println!("cargo:rustc-link-lib=xcb-dri3"); println!("cargo:rustc-link-lib=X11-xcb"); println!("cargo:rustc-link-lib=xcb"); } ================================================ FILE: build_in_local_container.sh ================================================ #!/usr/bin/env sh set -ex rm -f docker/archive.tar.gz git ls-files | tar Tczf - docker/archive.tar.gz podman run --replace -d --name weylus_build hhmhh/weylus_build:latest sleep infinity podman cp docker/archive.tar.gz weylus_build:/ podman exec weylus_build sh -c "mkdir /weylus && tar xf archive.tar.gz --directory=/weylus && cd weylus && ./docker_build.sh" podman run --replace -d --name weylus_build_alpine hhmhh/weylus_build_alpine:latest sleep infinity podman cp docker/archive.tar.gz weylus_build_alpine:/ podman exec weylus_build_alpine sh -c "mkdir /weylus && tar xf archive.tar.gz --directory=/weylus && cd weylus && RUSTFLAGS='-C target-feature=-crt-static' cargo build --release" ================================================ FILE: compile_flags.txt ================================================ -lX11 -lXext -Wall -Wextra -Ideps/dist/include -DHAS_NVENC -DHAS_LIBNPP -DHAS_VAAPI ================================================ FILE: deps/awk.patch ================================================ diff --git a/configure b/configure index 8569a60bf8..928b19db69 100755 --- a/configure +++ b/configure @@ -4690,7 +4690,7 @@ probe_cc(){ else _ident=$($_cc --version 2>/dev/null | head -n1 | tr -d '\r') fi - _DEPCMD='$(DEP$(1)) $(DEP$(1)FLAGS) $($(1)DEP_FLAGS) $< 2>&1 | awk '\''/including/ { sub(/^.*file: */, ""); gsub(/\\/, "/"); if (!match($$0, / /)) print "$@:", $$0 }'\'' > $(@:.o=.d)' + _DEPCMD='$(DEP$(1)) $(DEP$(1)FLAGS) $($(1)DEP_FLAGS) $< 2>&1 | awk -f ./msvc_dep.awk > $(@:.o=.d)' _DEPFLAGS='$(CPPFLAGS) $(CFLAGS) -showIncludes -Zs' _cflags_speed="-O2" _cflags_size="-O1" diff --git a/msvc_dep.awk b/msvc_dep.awk new file mode 100644 index 0000000000..a791efe000 --- /dev/null +++ b/msvc_dep.awk @@ -0,0 +1 @@ +/including/ { sub(/^.*file: */, ""); gsub(/\\/, "/"); if (!match($$0, / /)) print "$@:", $$0 } ================================================ FILE: deps/build.sh ================================================ #!/usr/bin/env bash set -ex export TARGET_OS="$CARGO_CFG_TARGET_OS" if [ "$OSTYPE" == "linux-gnu" ]; then export HOST_OS="linux" fi if [[ "$OSTYPE" == "darwin"* ]]; then export HOST_OS="macos" fi if [ "$OS" == "Windows_NT" ]; then export HOST_OS="windows" fi [ -z "$DIST" ] && export DIST="$PWD/dist" [ -z "$TARGET_OS" ] && export TARGET_OS="$HOST_OS" export NPROCS="$(nproc || echo 4)" ./download.sh if [ "$TARGET_OS" == "windows" ]; then if [ "$HOST_OS" == "linux" ]; then export CROSS_COMPILE="x86_64-w64-mingw32-" export FFMPEG_EXTRA_ARGS="--arch=x86_64 --target-os=mingw64 \ --cross-prefix=x86_64-w64-mingw32- --enable-nvenc --enable-ffnvcodec \ --enable-cuda-llvm --enable-mediafoundation --pkg-config=pkg-config --enable-d3d11va" export FFMPEG_CFLAGS="-I$DIST/include" export FFMPEG_LIBRARY_PATH="-L$DIST/lib" else export CC="cl" export FFMPEG_EXTRA_ARGS="--toolchain=msvc --enable-nvenc --enable-ffnvcodec \ --enable-cuda-llvm --enable-mediafoundation --enable-d3d11va" export FFMPEG_CFLAGS="-I$DIST/include" export FFMPEG_LIBRARY_PATH="-LIBPATH:$DIST/lib" fi else export FFMPEG_CFLAGS="-I$DIST/include" export FFMPEG_LIBRARY_PATH="-L$DIST/lib" if [ "$TARGET_OS" == "linux" ]; then export FFMPEG_EXTRA_ARGS="--enable-nvenc \ --enable-ffnvcodec \ --enable-cuda-llvm \ --enable-vaapi \ --enable-libdrm \ --enable-xlib" fi if [ "$TARGET_OS" == "macos" ]; then export FFMPEG_EXTRA_ARGS="--enable-videotoolbox" fi fi if [ "$ENABLE_LIBNPP" == "y" ]; then export FFMPEG_EXTRA_ARGS="$FFMPEG_EXTRA_ARGS --enable-libnpp --enable-nonfree" fi if [ "$TARGET_OS" == "windows" ] && [ "$HOST_OS" == "linux" ]; then export X264_EXTRA_ARGS="--cross-prefix=x86_64-w64-mingw32- --host=x86_64-w64-mingw32" fi ./x264.sh if [ "$TARGET_OS" == "linux" ]; then ./nv-codec-headers.sh ./libva.sh fi if [ "$TARGET_OS" == "windows" ]; then ./nv-codec-headers.sh fi ./ffmpeg.sh if [ "$TARGET_OS" == "windows" ] && [ "$HOST_OS" == "windows" ]; then cd "$DIST/lib" for l in *.a; do d=${l#lib} cp "$l" "${d%.a}.lib" done cp libx264.lib x264.lib fi ================================================ FILE: deps/clean.sh ================================================ #!/usr/bin/env bash set -x for d in ffmpeg x264 nv-codec-headers libva; do test -d "$d" || continue (cd "$d" && git clean -dfx && git reset --hard HEAD) done ================================================ FILE: deps/clean_all.sh ================================================ #!/usr/bin/env bash set -ex rm -rf ffmpeg x264 nv-codec-headers libva dist* ================================================ FILE: deps/command_limit.patch ================================================ diff --git a/ffbuild/library.mak b/ffbuild/library.mak index ad09f20..e63196d 100644 --- a/ffbuild/library.mak +++ b/ffbuild/library.mak @@ -35,7 +35,9 @@ OBJS += $(SHLIBOBJS) endif $(SUBDIR)$(LIBNAME): $(OBJS) $(STLIBOBJS) $(RM) $@ - $(AR) $(ARFLAGS) $(AR_O) $^ + $(file >$@.ar.txt, $^) + $(AR) $(ARFLAGS) $(AR_O) @$@.ar.txt + $(RM) $@.ar.txt $(RANLIB) $@ install-headers: install-lib$(NAME)-headers install-lib$(NAME)-pkgconfig ================================================ FILE: deps/download.sh ================================================ #!/usr/bin/env bash set -ex SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/refs.sh" source "$SCRIPT_DIR/hashes.sh" clone_at_commit() { local url="$1" dir="$2" commit="$3" if [ ! -d "$dir" ]; then git clone --filter=blob:none "$url" "$dir" git -C "$dir" checkout "$commit" fi } clone_at_commit "$X264_URL" x264 "$X264_COMMIT" clone_at_commit "$FFMPEG_URL" ffmpeg "$FFMPEG_COMMIT" if [ "$TARGET_OS" == "linux" ]; then clone_at_commit "$NV_CODEC_URL" nv-codec-headers "$NV_CODEC_COMMIT" clone_at_commit "$LIBVA_URL" libva "$LIBVA_COMMIT" fi if [ "$TARGET_OS" == "windows" ]; then clone_at_commit "$NV_CODEC_URL" nv-codec-headers "$NV_CODEC_COMMIT" fi if [ "$TARGET_OS" == "windows" ] && [ "$HOST_OS" == "windows" ]; then cd ffmpeg git apply ../command_limit.patch git apply ../awk.patch fi ================================================ FILE: deps/ffmpeg.sh ================================================ #!/usr/bin/env bash set -ex cd ffmpeg PKG_CONFIG_PATH="$DIST/lib/pkgconfig" ./configure \ --prefix="$DIST" \ --disable-debug \ --enable-static \ --disable-shared \ --enable-pic \ --enable-stripping \ --disable-programs \ --enable-gpl \ --enable-libx264 \ --disable-autodetect \ --extra-cflags="$FFMPEG_CFLAGS" \ --extra-ldflags="$FFMPEG_LIBRARY_PATH" \ $FFMPEG_EXTRA_ARGS make -j$NPROCS make install ================================================ FILE: deps/hashes.sh ================================================ #!/usr/bin/env bash # Generated by update_hashes.sh — do not edit manually. # Run ./update_hashes.sh to regenerate. X264_COMMIT="b35605ace3ddf7c1a5d67a2eb553f034aef41d55" # refs/heads/stable FFMPEG_COMMIT="a4044e04486d1136022498891088a90baf5b2775" # refs/tags/n8.0 NV_CODEC_COMMIT="876af32a202d0de83bd1d36fe74ee0f7fcf86b0d" # HEAD LIBVA_COMMIT="e85b1569b738fd8866cb9fa2452319f7148d663f" # refs/tags/2.23.0 ================================================ FILE: deps/libva.sh ================================================ #!/usr/bin/env bash set -ex cd libva # required to make ffmpeg's configure work sed -i -e "s/-lva$/-lva -ldrm -ldl/" pkgconfig/libva.pc.in sed -i -e 's/-lva-\${display}$/-lva-\${display} -lX11 -lXext -lXfixes -ldrm/' pkgconfig/libva-x11.pc.in sed -i -e 's/-lva-\${display}$/-lva-\${display} -ldrm/' pkgconfig/libva-drm.pc.in ./autogen.sh --prefix=$(readlink -f "$DIST") \ --enable-static=yes \ --enable-shared=yes \ --enable-drm \ --enable-x11 \ --enable-glx \ --with-drivers-path="/usr/lib/dri" make -j$NPROCS make install ================================================ FILE: deps/nv-codec-headers.sh ================================================ #!/usr/bin/env bash set -ex cd nv-codec-headers make PREFIX="$DIST" make install PREFIX="$DIST" ================================================ FILE: deps/refs.sh ================================================ #!/usr/bin/env bash # Dependency URLs and the branch/tag/ref to pin. # After editing, run ./update_hashes.sh to regenerate hashes.sh. X264_URL="https://code.videolan.org/videolan/x264.git" X264_REF="refs/heads/stable" FFMPEG_URL="https://code.ffmpeg.org/FFmpeg/FFmpeg.git" FFMPEG_REF="refs/tags/n8.0" NV_CODEC_URL="https://git.videolan.org/git/ffmpeg/nv-codec-headers.git" NV_CODEC_REF="HEAD" LIBVA_URL="https://github.com/intel/libva" LIBVA_REF="refs/tags/2.23.0" ================================================ FILE: deps/update_hashes.sh ================================================ #!/usr/bin/env bash # Resolves each ref in refs.sh to a commit hash and writes hashes.sh. set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/refs.sh" resolve() { local name="$1" url="$2" ref="$3" local hash hash=$(git ls-remote "$url" "$ref" | awk 'END { print $1 }') if [ -z "$hash" ]; then echo "ERROR: could not resolve $name $ref from $url" >&2 exit 1 fi echo "${name}_COMMIT=\"${hash}\" # ${ref}" } cat > "$SCRIPT_DIR/hashes.sh" < #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "error.h" #include "log.h" #ifdef HAS_VAAPI #include #include #endif const AVRational TIME_BASE = (AVRational){1, 1000}; typedef struct ScaleContext { AVFilterGraph* filter_graph_scale; AVFilterContext* buffersink_scale_ctx; AVFilterContext* buffersrc_scale_ctx; AVFrame* frame_in; AVFrame* frame_out; } ScaleContext; typedef struct Scalers { ScaleContext bgr0; ScaleContext rgb0; ScaleContext rgb; AVBufferRef* hw_frames_ctx; AVFrame* frame_out; } Scalers; typedef struct VideoContext { AVFormatContext* oc; AVCodecContext* c; // pointer to the frame to be encoded, one of frame_out in scalers.bgr0/rgb0/rgb AVFrame* frame; Scalers scalers; AVBufferRef* hw_device_ctx; AVPacket* pkt; AVStream* st; int width_out; int height_out; int width_in; int height_in; void* buf; void* rust_ctx; int pts; int initialized; int frame_allocated; int try_vaapi; int try_nvenc; int try_videotoolbox; int try_mediafoundation; } VideoContext; // this is a rust function and lives in src/video.rs int write_video_packet(void* rust_ctx, const uint8_t* buf, int buf_size); #if defined(__clang__) || defined(__GNUC__) void log_callback(__attribute__((unused)) void* _ptr, int level, const char* fmt_orig, va_list args) #else void log_callback(void* _ptr, int level, const char* fmt_orig, va_list args) #endif { char fmt[256] = {0}; strncpy(fmt, fmt_orig, sizeof(fmt) - 1); int done = 0; // strip whitespaces from end for (int i = sizeof(fmt) - 1; i >= 0 && !done; --i) switch (fmt[i]) { case ' ': case '\n': case '\t': case '\r': fmt[i] = '\0'; break; case '\0': break; default: done = 1; } char buf[2048]; vsnprintf(buf, sizeof(buf), fmt, args); switch (level) { case AV_LOG_FATAL: case AV_LOG_ERROR: case AV_LOG_PANIC: log_error("%s", buf); break; case AV_LOG_INFO: log_info("%s", buf); break; case AV_LOG_WARNING: log_warn("%s", buf); break; case AV_LOG_QUIET: break; case AV_LOG_VERBOSE: log_debug("%s", buf); break; case AV_LOG_DEBUG: log_trace("%s", buf); break; } } // called in src/log.rs void init_ffmpeg_logger() { av_log_set_callback(log_callback); } void set_codec_params(VideoContext* ctx) { /* resolution must be a multiple of two */ ctx->c->width = ctx->width_out; ctx->c->height = ctx->height_out; ctx->c->time_base = TIME_BASE; ctx->c->framerate = (AVRational){0, 1}; ctx->c->gop_size = 12; // no B-frames to reduce latency ctx->c->max_b_frames = 0; if (ctx->oc->oformat->flags & AVFMT_GLOBALHEADER) ctx->c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; } void destroy_scale_ctx(ScaleContext* ctx) { avfilter_graph_free(&ctx->filter_graph_scale); if (ctx->frame_in) av_frame_free(&ctx->frame_in); } void init_scaler( ScaleContext* ctx, int width_in, int height_in, int width_out, int height_out, enum AVPixelFormat pix_fmt_in, enum AVPixelFormat pix_fmt_out, AVBufferRef* hw_device_ctx, enum AVPixelFormat pix_fmt_sw_out, AVFrame* frame_out, Error* err) { int ret = 0; ctx->frame_in = av_frame_alloc(); if (!ctx->frame_in) ERROR(err, 1, "Failed to allocate frame_in for scale filter!"); ctx->frame_out = frame_out; ctx->frame_in->format = pix_fmt_in; ctx->frame_in->width = width_in; ctx->frame_in->height = height_in; ret = av_frame_get_buffer(ctx->frame_in, 0); if (ret) { destroy_scale_ctx(ctx); ERROR( err, 1, "Failed to allocate buffer for frame_in for scale filter: %s!", av_err2str(ret)); } char args[512]; const AVFilter* buffersrc = avfilter_get_by_name("buffer"); const AVFilter* buffersink = avfilter_get_by_name("buffersink"); AVFilterInOut* outputs = avfilter_inout_alloc(); AVFilterInOut* inputs = avfilter_inout_alloc(); ctx->filter_graph_scale = avfilter_graph_alloc(); if (!outputs || !inputs || !ctx->filter_graph_scale) { ret = AVERROR(ENOMEM); goto end; } avfilter_graph_set_auto_convert(ctx->filter_graph_scale, AVFILTER_AUTO_CONVERT_NONE); /* buffer video source: the decoded frames from the decoder will be inserted here. */ snprintf( args, sizeof(args), "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d", width_in, height_in, pix_fmt_in, TIME_BASE.num, TIME_BASE.den, 1, 1); ret = avfilter_graph_create_filter( &ctx->buffersrc_scale_ctx, buffersrc, "in", args, NULL, ctx->filter_graph_scale); if (ret < 0) { log_warn("Cannot create buffer source"); goto end; } /* buffer video sink: to terminate the filter chain. */ ctx->buffersink_scale_ctx = avfilter_graph_alloc_filter(ctx->filter_graph_scale, buffersink, "out"); if (ctx->buffersink_scale_ctx == NULL) { log_warn("Cannot allocate buffer sink"); goto end; } ret = av_opt_set_array( ctx->buffersink_scale_ctx, "pixel_formats", AV_OPT_SEARCH_CHILDREN, 0, 1, AV_OPT_TYPE_PIXEL_FMT, &pix_fmt_out); if (ret < 0) { log_warn("Cannot set output pixel format: %s", av_err2str(ret)); goto end; } ret = avfilter_init_dict(ctx->buffersink_scale_ctx, NULL); if (ret < 0) { log_warn("Cannot init buffer sink"); goto end; } outputs->name = av_strdup("in"); outputs->filter_ctx = ctx->buffersrc_scale_ctx; outputs->pad_idx = 0; outputs->next = NULL; inputs->name = av_strdup("out"); inputs->filter_ctx = ctx->buffersink_scale_ctx; inputs->pad_idx = 0; inputs->next = NULL; switch (pix_fmt_out) { case AV_PIX_FMT_CUDA: if (pix_fmt_in == AV_PIX_FMT_RGB24) { snprintf( args, sizeof(args), "scale=w=%d:h=%d:flags=fast_bilinear,hwupload_cuda", width_out, height_out); } else { snprintf( args, sizeof(args), #ifdef HAS_LIBNPP "scale,format=nv12,hwupload_cuda,scale_npp=w=%d:h=%d:format=%s:interp_algo=nn", #else "hwupload_cuda,scale_cuda=w=%d:h=%d:format=%s:interp_algo=nearest", #endif width_out, height_out, av_get_pix_fmt_name(pix_fmt_sw_out)); } break; case AV_PIX_FMT_VAAPI: if (pix_fmt_in == AV_PIX_FMT_RGB24) snprintf( args, sizeof(args), "scale=w=%d:h=%d:flags=fast_bilinear,hwupload", width_out, height_out); else snprintf( args, sizeof(args), "hwupload,scale_vaapi=w=%d:h=%d:format=%s:mode=fast", width_out, height_out, av_get_pix_fmt_name(pix_fmt_sw_out)); break; default: snprintf(args, sizeof(args), "scale=w=%d:h=%d:flags=fast_bilinear", width_out, height_out); } if ((ret = avfilter_graph_parse_ptr(ctx->filter_graph_scale, args, &inputs, &outputs, NULL)) < 0) { log_warn("Failed to parse filter"); goto end; } for (unsigned int i = 0; i < ctx->filter_graph_scale->nb_filters; i++) { AVFilterContext* filt = ctx->filter_graph_scale->filters[i]; if (strcmp(filt->filter->name, "hwupload") == 0) { filt->hw_device_ctx = av_buffer_ref(hw_device_ctx); } } if ((ret = avfilter_graph_config(ctx->filter_graph_scale, NULL)) < 0) { log_warn("Failed to configure filter graph"); goto end; } end: avfilter_inout_free(&inputs); avfilter_inout_free(&outputs); if (ret != 0) { destroy_scale_ctx(ctx); ERROR( err, 1, "Setting up scale filter %s -> %s (sw: %s) failed!", av_get_pix_fmt_name(pix_fmt_in), av_get_pix_fmt_name(pix_fmt_out), av_get_pix_fmt_name(pix_fmt_sw_out)); } else { log_debug( "Scale filter set %s -> %s (sw: %s) up!", av_get_pix_fmt_name(pix_fmt_in), av_get_pix_fmt_name(pix_fmt_out), av_get_pix_fmt_name(pix_fmt_sw_out)); } } void destroy_scalers(Scalers* s) { destroy_scale_ctx(&s->bgr0); destroy_scale_ctx(&s->rgb0); destroy_scale_ctx(&s->rgb); if (s->frame_out) av_frame_free(&s->frame_out); } void init_scalers( Scalers* ctx, int width_in, int height_in, int width_out, int height_out, enum AVPixelFormat pix_fmt_out, enum AVPixelFormat pix_fmt_sw_out, AVBufferRef* hw_device_ctx, Error* err) { int ret; ctx->frame_out = av_frame_alloc(); if (!ctx->frame_out) { destroy_scalers(ctx); ERROR(err, 1, "Failed to allocate frame_out for scale filter!"); } if (hw_device_ctx != NULL) { AVBufferRef* hw_frames_ref; AVHWFramesContext* frames_ctx = NULL; if (!(hw_frames_ref = av_hwframe_ctx_alloc(hw_device_ctx))) { destroy_scalers(ctx); ERROR(err, 1, "Failed to create HW frame context."); } frames_ctx = (AVHWFramesContext*)(hw_frames_ref->data); frames_ctx->format = pix_fmt_out; frames_ctx->sw_format = pix_fmt_sw_out; frames_ctx->width = width_out; frames_ctx->height = height_out; frames_ctx->initial_pool_size = 20; if ((ret = av_hwframe_ctx_init(hw_frames_ref)) < 0) { av_buffer_unref(&hw_frames_ref); destroy_scalers(ctx); ERROR( err, 1, "Failed to initialize HW frame context." "Error code: %s", av_err2str(ret)); } ctx->hw_frames_ctx = av_buffer_ref(hw_frames_ref); ret = av_hwframe_get_buffer(ctx->hw_frames_ctx, ctx->frame_out, 0); if (ret < 0) { av_buffer_unref(&hw_frames_ref); destroy_scalers(ctx); ERROR( err, 1, "Could not allocate video hardware frame data for scaling: %s", av_err2str(ret)); } av_buffer_unref(&hw_frames_ref); } enum AVPixelFormat pix_fmts[] = {AV_PIX_FMT_BGR0, AV_PIX_FMT_RGB0, AV_PIX_FMT_RGB24}; ScaleContext* scalers[] = {&ctx->bgr0, &ctx->rgb0, &ctx->rgb}; for (int i = 0; i < 3; i++) { init_scaler( scalers[i], width_in, height_in, width_out, height_out, pix_fmts[i], pix_fmt_out, hw_device_ctx, pix_fmt_sw_out, ctx->frame_out, err); OK_OR_ABORT(err); } } void scale_frame(ScaleContext* ctx, Error* err) { int ret; if ((ret = av_buffersrc_add_frame_flags( ctx->buffersrc_scale_ctx, ctx->frame_in, AV_BUFFERSRC_FLAG_KEEP_REF) < 0)) { ERROR(err, ret, "Error adding frame to buffer source: %s.", av_err2str(ret)); } av_frame_unref(ctx->frame_out); while (1) { int ret = av_buffersink_get_frame(ctx->buffersink_scale_ctx, ctx->frame_out); if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) break; if (ret < 0) { ERROR(err, ret, "Error reading frame from buffer sink: %s.", av_err2str(ret)); } } } void open_video(VideoContext* ctx, Error* err) { if (ctx->width_out <= 1 || ctx->height_out <= 1) ERROR( err, 1, "Invalid size for video: width = %d, height = %d", ctx->width_out, ctx->height_out); const AVCodec* codec; int ret; avformat_alloc_output_context2(&ctx->oc, NULL, "mp4", NULL); if (!ctx->oc) { ERROR(err, 1, "Could not find output format mp4."); } int using_hw = 0; #ifdef HAS_VAAPI char* vaapi_device = getenv("WEYLUS_VAAPI_DEVICE"); if (ctx->try_vaapi && av_hwdevice_ctx_create( &ctx->hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, vaapi_device, NULL, 0) == 0) { if (ctx->hw_device_ctx) { AVHWFramesConstraints* cst = av_hwdevice_get_hwframe_constraints(ctx->hw_device_ctx, NULL); if (cst) { for (enum AVPixelFormat* fmt = cst->valid_sw_formats; *fmt != AV_PIX_FMT_NONE; ++fmt) { log_debug("VAAPI: valid pix_fmt: %s", av_get_pix_fmt_name(*fmt)); } av_hwframe_constraints_free(&cst); } } codec = avcodec_find_encoder_by_name("h264_vaapi"); if (codec) { ctx->c = avcodec_alloc_context3(codec); if (ctx->c) { Error err = {0}; init_scalers( &ctx->scalers, ctx->width_in, ctx->height_in, ctx->width_out, ctx->height_out, AV_PIX_FMT_VAAPI, AV_PIX_FMT_NV12, ctx->hw_device_ctx, &err); if (err.code) { log_warn("Failed to initialize scaler: %s", err.error_str); avcodec_free_context(&ctx->c); } else { ctx->c->pix_fmt = AV_PIX_FMT_VAAPI; ctx->c->hw_frames_ctx = ctx->scalers.hw_frames_ctx; av_opt_set(ctx->c->priv_data, "quality", "7", 0); av_opt_set(ctx->c->priv_data, "qp", "23", 0); set_codec_params(ctx); if ((ret = avcodec_open2(ctx->c, codec, NULL) == 0)) using_hw = 1; else { log_debug("Could not open codec: %s!", av_err2str(ret)); avcodec_free_context(&ctx->c); av_buffer_unref(&ctx->hw_device_ctx); destroy_scalers(&ctx->scalers); } } } } else av_buffer_unref(&ctx->hw_device_ctx); } #endif #ifdef HAS_MEDIAFOUNDATION if (ctx->try_mediafoundation && !using_hw) { codec = avcodec_find_encoder_by_name("h264_mf"); if (codec) { ctx->c = avcodec_alloc_context3(codec); if (ctx->c) { Error err = {0}; init_scalers( &ctx->scalers, ctx->width_in, ctx->height_in, ctx->width_out, ctx->height_out, AV_PIX_FMT_NV12, AV_PIX_FMT_NV12, NULL, &err); if (err.code) { log_warn("Failed to initialize scaler: %s", err.error_str); avcodec_free_context(&ctx->c); } else { ctx->c->pix_fmt = AV_PIX_FMT_NV12; av_opt_set(ctx->c->priv_data, "rate_control", "ld_vbr", 0); av_opt_set(ctx->c->priv_data, "scenario", "display_remoting", 0); av_opt_set(ctx->c->priv_data, "quality", "100", 0); set_codec_params(ctx); int ret = avcodec_open2(ctx->c, codec, NULL); if (ret == 0) using_hw = 1; else { log_debug("Could not open codec: %s!", av_err2str(ret)); avcodec_free_context(&ctx->c); destroy_scalers(&ctx->scalers); } } } else log_debug("Could not allocate video codec context for 'h264_mf'!"); } else log_debug("Codec 'h264_mf' not found!"); } #endif #ifdef HAS_NVENC if (ctx->try_nvenc && !using_hw && av_hwdevice_ctx_create(&ctx->hw_device_ctx, AV_HWDEVICE_TYPE_CUDA, NULL, NULL, 0) == 0) { codec = avcodec_find_encoder_by_name("h264_nvenc"); if (codec) { ctx->c = avcodec_alloc_context3(codec); if (ctx->c) { Error err = {0}; init_scalers( &ctx->scalers, ctx->width_in, ctx->height_in, ctx->width_out, ctx->height_out, AV_PIX_FMT_CUDA, #ifdef HAS_LIBNPP AV_PIX_FMT_NV12, #else AV_PIX_FMT_BGR0, #endif ctx->hw_device_ctx, &err); if (err.code) { log_warn("Failed to initialize scaler: %s", err.error_str); avcodec_free_context(&ctx->c); } else { ctx->c->pix_fmt = AV_PIX_FMT_CUDA; ctx->c->hw_frames_ctx = ctx->scalers.hw_frames_ctx; av_opt_set(ctx->c->priv_data, "preset", "p1", 0); av_opt_set(ctx->c->priv_data, "zerolatency", "1", 0); av_opt_set(ctx->c->priv_data, "tune", "ull", 0); av_opt_set(ctx->c->priv_data, "rc", "cbr", 0); av_opt_set(ctx->c->priv_data, "cq", "21", 0); av_opt_set(ctx->c->priv_data, "delay", "0", 0); set_codec_params(ctx); int ret = avcodec_open2(ctx->c, codec, NULL); if (ret == 0) using_hw = 1; else { log_debug("Could not open codec: %s!", av_err2str(ret)); avcodec_free_context(&ctx->c); destroy_scalers(&ctx->scalers); } } } else log_debug("Could not allocate video codec context for 'h264_nvenc'!"); } else log_debug("Codec 'h264_nvenc' not found!"); } #endif #ifdef HAS_VIDEOTOOLBOX if (ctx->try_videotoolbox && !using_hw) { codec = avcodec_find_encoder_by_name("h264_videotoolbox"); if (codec) { ctx->c = avcodec_alloc_context3(codec); if (ctx->c) { Error err = {0}; init_scalers( &ctx->scalers, ctx->width_in, ctx->height_in, ctx->width_out, ctx->height_out, AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P, ctx->hw_device_ctx, &err); if (err.code) { log_warn("Failed to initialize scaler: %s", err.error_str); avcodec_free_context(&ctx->c); } else { ctx->c->pix_fmt = AV_PIX_FMT_YUV420P; av_opt_set(ctx->c->priv_data, "realtime", "true", 0); av_opt_set(ctx->c->priv_data, "allow_sw", "true", 0); av_opt_set(ctx->c->priv_data, "profile", "extended", 0); av_opt_set(ctx->c->priv_data, "level", "5.2", 0); set_codec_params(ctx); if (avcodec_open2(ctx->c, codec, NULL) == 0) using_hw = 1; else { log_debug("Could not open codec: %s!", av_err2str(ret)); avcodec_free_context(&ctx->c); destroy_scalers(&ctx->scalers); } } } } } #endif if (!using_hw) { codec = avcodec_find_encoder_by_name("libx264"); if (!codec) { ERROR(err, 1, "Codec 'libx264' not found"); } ctx->c = avcodec_alloc_context3(codec); if (!ctx->c) { ERROR(err, 1, "Could not allocate video codec context"); } init_scalers( &ctx->scalers, ctx->width_in, ctx->height_in, ctx->width_out, ctx->height_out, AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P, NULL, err); if (err->code) { avcodec_free_context(&ctx->c); return; } ctx->c->pix_fmt = AV_PIX_FMT_YUV420P; av_opt_set(ctx->c->priv_data, "preset", "ultrafast", 0); av_opt_set(ctx->c->priv_data, "tune", "zerolatency", 0); av_opt_set(ctx->c->priv_data, "crf", "23", 0); set_codec_params(ctx); ret = avcodec_open2(ctx->c, codec, NULL); if (ret < 0) { avcodec_free_context(&ctx->c); ERROR(err, 1, "Could not open codec: %s", av_err2str(ret)); } } ctx->st = avformat_new_stream(ctx->oc, NULL); avcodec_parameters_from_context(ctx->st->codecpar, ctx->c); ctx->pkt = av_packet_alloc(); if (!ctx->pkt) ERROR(err, 1, "Failed to allocate packet"); int buf_size = 1024 * 1024; ctx->buf = av_malloc(buf_size); ctx->oc->pb = avio_alloc_context( ctx->buf, buf_size, AVIO_FLAG_WRITE, ctx->rust_ctx, NULL, write_video_packet, NULL); if (!ctx->oc->pb) ERROR(err, 1, "Failed to allocate avio context"); AVDictionary* opt = NULL; // enable writing fragmented mp4 av_dict_set(&opt, "movflags", "frag_custom+empty_moov+default_base_moof", 0); ret = avformat_write_header(ctx->oc, &opt); if (ret < 0) log_warn("Video: failed to write header!"); av_dict_free(&opt); if (av_pix_fmt_desc_get(ctx->c->pix_fmt)->flags & AV_PIX_FMT_FLAG_HWACCEL && ctx->c->hw_frames_ctx) { const char* pix_fmt_sw = av_get_pix_fmt_name(((AVHWFramesContext*)ctx->c->hw_frames_ctx->data)->sw_format); log_info( "Video: %dx%d@%s pix_fmt: %s (%s)", ctx->width_out, ctx->height_out, ctx->c->codec->name, av_get_pix_fmt_name(ctx->c->pix_fmt), pix_fmt_sw); } else log_info( "Video: %dx%d@%s pix_fmt: %s", ctx->width_out, ctx->height_out, ctx->c->codec->name, av_get_pix_fmt_name(ctx->c->pix_fmt)); ctx->initialized = 1; } void destroy_video_encoder(VideoContext* ctx) { if (ctx->initialized) { av_write_trailer(ctx->oc); avio_context_free(&ctx->oc->pb); avformat_free_context(ctx->oc); avcodec_free_context(&ctx->c); av_packet_free(&ctx->pkt); av_free(ctx->buf); destroy_scalers(&ctx->scalers); } if (ctx->hw_device_ctx) av_buffer_unref(&ctx->hw_device_ctx); free(ctx); } void encode_video_frame(VideoContext* ctx, int millis, Error* err) { int ret; AVFrame* frame = ctx->frame; if (!frame) ERROR(err, 1, "Frame not initialized!"); frame->pts = millis; ret = avcodec_send_frame(ctx->c, frame); if (ret < 0) ERROR(err, 1, "Error sending a frame for encoding: %s", av_err2str(ret)); while (ret >= 0) { ret = avcodec_receive_packet(ctx->c, ctx->pkt); if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) return; else if (ret < 0) { ERROR(err, 1, "Error during encoding"); } av_packet_rescale_ts(ctx->pkt, ctx->c->time_base, ctx->st->time_base); av_write_frame(ctx->oc, ctx->pkt); av_packet_unref(ctx->pkt); // new fragment on every frame for lowest latency av_write_frame(ctx->oc, NULL); } } VideoContext* init_video_encoder( void* rust_ctx, int width_in, int height_in, int width_out, int height_out, int try_vaapi, int try_nvenc, int try_videotoolbox, int try_mediafoundation) { VideoContext* ctx = malloc(sizeof(VideoContext)); ctx->rust_ctx = rust_ctx; ctx->width_out = width_out - width_out % 2; ctx->height_out = height_out - height_out % 2; ctx->width_in = width_in; ctx->height_in = height_in; ctx->pts = 0; ctx->initialized = 0; ctx->frame_allocated = 0; ctx->try_vaapi = try_vaapi; ctx->try_nvenc = try_nvenc; ctx->try_videotoolbox = try_videotoolbox; ctx->try_mediafoundation = try_mediafoundation; ctx->hw_device_ctx = NULL; // make sure all scalers are zero initialized so that destroy can always be called memset(&ctx->scalers, 0, sizeof(Scalers)); return ctx; } void fill_bgr0(VideoContext* ctx, const void* data, int stride, Error* err) { ctx->frame = NULL; ScaleContext* scaler = &ctx->scalers.bgr0; scaler->frame_in->data[0] = (uint8_t*)data; scaler->frame_in->linesize[0] = stride; scale_frame(scaler, err); OK_OR_ABORT(err) ctx->frame = scaler->frame_out; } void fill_rgb(VideoContext* ctx, const void* data, Error* err) { ctx->frame = NULL; ScaleContext* scaler = &ctx->scalers.rgb; ctx->frame = NULL; scaler->frame_in->data[0] = (uint8_t*)data; scaler->frame_in->linesize[0] = ctx->width_in * 3; scale_frame(scaler, err); OK_OR_ABORT(err) ctx->frame = scaler->frame_out; } void fill_rgb0(VideoContext* ctx, const void* data, Error* err) { ctx->frame = NULL; ScaleContext* scaler = &ctx->scalers.rgb0; scaler->frame_in->data[0] = (uint8_t*)data; scaler->frame_in->linesize[0] = ctx->width_in * 4; scale_frame(scaler, err); OK_OR_ABORT(err) ctx->frame = scaler->frame_out; } ================================================ FILE: lib/error.c ================================================ #include "error.h" void fill_error(Error* err, int code, const char* fmt, ...) { if (!err) return; err->code = code; va_list args; va_start(args, fmt); vsnprintf(err->error_str, sizeof(err->error_str), fmt, args); } ================================================ FILE: lib/error.h ================================================ #pragma once #include #include struct Error { int code; char error_str[1024]; }; typedef struct Error Error; #if defined(__clang__) || defined(__GNUC__) __attribute__((__format__ (__printf__, 3, 4))) #endif void fill_error(Error* err, int code, const char* fmt, ...); #define ERROR(err, code, fmt, ...) \ { \ fill_error(err, code, fmt, ##__VA_ARGS__); \ return; \ } #define OK_OR_ABORT(err) \ { \ if (err->code) \ return; \ } ================================================ FILE: lib/linux/uinput.c ================================================ #include #include #include #include #include #include #include #include #include #include #include #include #include "../error.h" #define ABS_MAXVAL 65535 #ifndef REL_WHEEL_HI_RES #define REL_WHEEL_HI_RES 0x0b #endif #ifndef REL_HWHEEL_HI_RES #define REL_HWHEEL_HI_RES 0x0c #endif void setup_abs(int fd, int code, int minimum, int maximum, int resolution, Error* err) { if (ioctl(fd, UI_SET_ABSBIT, code) < 0) ERROR(err, 1, "error: ioctl UI_SET_ABSBIT, code %#x", code); struct uinput_abs_setup abs_setup; memset(&abs_setup, 0, sizeof(abs_setup)); abs_setup.code = code; abs_setup.absinfo.value = 0; abs_setup.absinfo.minimum = minimum; abs_setup.absinfo.maximum = maximum; abs_setup.absinfo.fuzz = 0; abs_setup.absinfo.flat = 0; // units/mm abs_setup.absinfo.resolution = resolution; if (ioctl(fd, UI_ABS_SETUP, &abs_setup) < 0) ERROR(err, 1, "error: UI_ABS_SETUP, code: %#x", code); } void setup(int fd, const char* name, Error* err) { struct uinput_setup setup; memset(&setup, 0, sizeof(setup)); strncpy(setup.name, name, UINPUT_MAX_NAME_SIZE - 1); setup.id.bustype = BUS_VIRTUAL; setup.id.vendor = 0x1701; setup.id.product = 0x1701; setup.id.version = 0x0001; setup.ff_effects_max = 0; if (ioctl(fd, UI_DEV_SETUP, &setup) < 0) ERROR(err, 1, "error: UI_DEV_SETUP"); } void init_keyboard(int fd, const char* name, Error* err) { // enable synchronization if (ioctl(fd, UI_SET_EVBIT, EV_SYN) < 0) ERROR(err, 1, "error: ioctl UI_SET_EVBIT EV_SYN"); // enable keys if (ioctl(fd, UI_SET_EVBIT, EV_KEY) < 0) ERROR(err, 1, "error: ioctl UI_SET_EVBIT EV_KEY"); // enable all the keys! for (int keycode = KEY_ESC; keycode <= KEY_MICMUTE; ++keycode) if (ioctl(fd, UI_SET_KEYBIT, keycode) < 0) ERROR(err, 1, "error: ioctl UI_SET_KEYBIT %x", keycode); // TODO: figure if scancodes are needed // if (ioctl(fd, UI_SET_EVBIT, EV_MSC) < 0) // ERROR(err, 1, "error: ioctl UI_SET_EVBIT EV_MSC"); // if (ioctl(fd, UI_SET_MSCBIT, MSC_SCAN) < 0) // ERROR(err, 1, "error: ioctl UI_SET_MSCBIT MSC_SCAN"); setup(fd, name, err); OK_OR_ABORT(err); if (ioctl(fd, UI_DEV_CREATE) < 0) ERROR(err, 1, "error: ioctl"); } void init_mouse(int fd, const char* name, Error* err) { // enable synchronization if (ioctl(fd, UI_SET_EVBIT, EV_SYN) < 0) ERROR(err, 1, "error: ioctl UI_SET_EVBIT EV_SYN"); if (ioctl(fd, UI_SET_PROPBIT, INPUT_PROP_DIRECT) < 0) ERROR(err, 1, "error: ioctl UI_SET_PROPBIT INPUT_PROP_DIRECT"); // enable buttons if (ioctl(fd, UI_SET_EVBIT, EV_KEY) < 0) ERROR(err, 1, "error: ioctl UI_SET_EVBIT EV_KEY"); if (ioctl(fd, UI_SET_KEYBIT, BTN_LEFT) < 0) ERROR(err, 1, "error: ioctl UI_SET_KEYBIT BTN_LEFT"); if (ioctl(fd, UI_SET_KEYBIT, BTN_RIGHT) < 0) ERROR(err, 1, "error: ioctl UI_SET_KEYBIT BTN_RIGHT"); if (ioctl(fd, UI_SET_KEYBIT, BTN_MIDDLE) < 0) ERROR(err, 1, "error: ioctl UI_SET_KEYBIT BTN_MIDDLE"); // enable scrolling if (ioctl(fd, UI_SET_EVBIT, EV_REL) < 0) ERROR(err, 1, "error: ioctl UI_SET_EVBIT EV_REL"); if (ioctl(fd, UI_SET_RELBIT, REL_WHEEL) < 0) ERROR(err, 1, "error: ioctl UI_SET_RELBIT REL_WHEEL"); if (ioctl(fd, UI_SET_RELBIT, REL_HWHEEL) < 0) ERROR(err, 1, "error: ioctl UI_SET_RELBIT REL_HWHEEL"); if (ioctl(fd, UI_SET_RELBIT, REL_WHEEL_HI_RES) < 0) ERROR(err, 1, "error: ioctl UI_SET_RELBIT REL_WHEEL_HI_RES"); if (ioctl(fd, UI_SET_RELBIT, REL_HWHEEL_HI_RES) < 0) ERROR(err, 1, "error: ioctl UI_SET_RELBIT REL_HWHEEL_HI_RES"); // setup sending timestamps if (ioctl(fd, UI_SET_EVBIT, EV_MSC) < 0) ERROR(err, 1, "error: ioctl UI_SET_EVBIT EV_MSC"); if (ioctl(fd, UI_SET_MSCBIT, MSC_TIMESTAMP) < 0) ERROR(err, 1, "error: ioctl UI_SET_MSCBIT MSC_TIMESTAMP"); if (ioctl(fd, UI_SET_EVBIT, EV_ABS) < 0) ERROR(err, 1, "error: ioctl UI_SET_EVBIT EV_ABS"); setup_abs(fd, ABS_X, 0, ABS_MAXVAL, 0, err); OK_OR_ABORT(err); setup_abs(fd, ABS_Y, 0, ABS_MAXVAL, 0, err); OK_OR_ABORT(err); setup(fd, name, err); OK_OR_ABORT(err); if (ioctl(fd, UI_DEV_CREATE) < 0) ERROR(err, 1, "error: ioctl"); } void init_stylus(int fd, const char* name, Error* err) { // enable synchronization if (ioctl(fd, UI_SET_EVBIT, EV_SYN) < 0) ERROR(err, 1, "error: ioctl UI_SET_EVBIT EV_SYN"); if (ioctl(fd, UI_SET_PROPBIT, INPUT_PROP_DIRECT) < 0) ERROR(err, 1, "error: ioctl UI_SET_PROPBIT INPUT_PROP_DIRECT"); // enable buttons if (ioctl(fd, UI_SET_EVBIT, EV_KEY) < 0) ERROR(err, 1, "error: ioctl UI_SET_EVBIT EV_KEY"); if (ioctl(fd, UI_SET_KEYBIT, BTN_TOOL_PEN) < 0) ERROR(err, 1, "error: ioctl UI_SET_KEYBIT BTN_TOOL_PEN"); if (ioctl(fd, UI_SET_KEYBIT, BTN_TOOL_RUBBER) < 0) ERROR(err, 1, "error: ioctl UI_SET_KEYBIT BTN_TOOL_RUBBER"); if (ioctl(fd, UI_SET_KEYBIT, BTN_TOUCH) < 0) ERROR(err, 1, "error: ioctl UI_SET_KEYBIT BTN_TOUCH"); // setup sending timestamps if (ioctl(fd, UI_SET_EVBIT, EV_MSC) < 0) ERROR(err, 1, "error: ioctl UI_SET_EVBIT EV_MSC"); if (ioctl(fd, UI_SET_MSCBIT, MSC_TIMESTAMP) < 0) ERROR(err, 1, "error: ioctl UI_SET_MSCBIT MSC_TIMESTAMP"); if (ioctl(fd, UI_SET_EVBIT, EV_ABS) < 0) ERROR(err, 1, "error: ioctl UI_SET_EVBIT EV_ABS"); setup_abs(fd, ABS_X, 0, ABS_MAXVAL, 12, err); OK_OR_ABORT(err); setup_abs(fd, ABS_Y, 0, ABS_MAXVAL, 12, err); OK_OR_ABORT(err); setup_abs(fd, ABS_PRESSURE, 0, ABS_MAXVAL, 12, err); OK_OR_ABORT(err); setup_abs(fd, ABS_TILT_X, -90, 90, 12, err); OK_OR_ABORT(err); setup_abs(fd, ABS_TILT_Y, -90, 90, 12, err); OK_OR_ABORT(err); setup(fd, name, err); OK_OR_ABORT(err); if (ioctl(fd, UI_DEV_CREATE) < 0) ERROR(err, 1, "error: ioctl"); } void init_touch(int fd, const char* name, Error* err) { // enable synchronization if (ioctl(fd, UI_SET_EVBIT, EV_SYN) < 0) ERROR(err, 1, "error: ioctl UI_SET_EVBIT EV_SYN"); if (ioctl(fd, UI_SET_PROPBIT, INPUT_PROP_DIRECT) < 0) ERROR(err, 1, "error: ioctl UI_SET_PROPBIT INPUT_PROP_DIRECT"); if (ioctl(fd, UI_SET_EVBIT, EV_KEY) < 0) ERROR(err, 1, "error: ioctl UI_SET_EVBIT EV_KEY"); if (ioctl(fd, UI_SET_KEYBIT, BTN_TOUCH) < 0) ERROR(err, 1, "error: ioctl UI_SET_KEYBIT"); if (ioctl(fd, UI_SET_KEYBIT, BTN_TOOL_FINGER) < 0) ERROR(err, 1, "error: ioctl UI_SET_KEYBIT"); if (ioctl(fd, UI_SET_KEYBIT, BTN_TOOL_DOUBLETAP) < 0) ERROR(err, 1, "error: ioctl UI_SET_KEYBIT"); if (ioctl(fd, UI_SET_KEYBIT, BTN_TOOL_TRIPLETAP) < 0) ERROR(err, 1, "error: ioctl UI_SET_KEYBIT"); if (ioctl(fd, UI_SET_KEYBIT, BTN_TOOL_QUADTAP) < 0) ERROR(err, 1, "error: ioctl UI_SET_KEYBIT"); if (ioctl(fd, UI_SET_KEYBIT, BTN_TOOL_QUINTTAP) < 0) ERROR(err, 1, "error: ioctl UI_SET_KEYBIT"); // setup sending timestamps if (ioctl(fd, UI_SET_EVBIT, EV_MSC) < 0) ERROR(err, 1, "error: ioctl UI_SET_EVBIT EV_MSC"); if (ioctl(fd, UI_SET_MSCBIT, MSC_TIMESTAMP) < 0) ERROR(err, 1, "error: ioctl UI_SET_MSCBIT MSC_TIMESTAMP"); if (ioctl(fd, UI_SET_EVBIT, EV_ABS) < 0) ERROR(err, 1, "error: ioctl UI_SET_EVBIT EV_ABS"); setup_abs(fd, ABS_X, 0, ABS_MAXVAL, 200, err); OK_OR_ABORT(err); setup_abs(fd, ABS_Y, 0, ABS_MAXVAL, 200, err); OK_OR_ABORT(err); // 5 fingers 5 multitouch slots. setup_abs(fd, ABS_MT_SLOT, 0, 4, 0, err); OK_OR_ABORT(err); setup_abs(fd, ABS_MT_TRACKING_ID, 0, 4, 0, err); OK_OR_ABORT(err); setup_abs(fd, ABS_MT_POSITION_X, 0, ABS_MAXVAL, 200, err); OK_OR_ABORT(err); setup_abs(fd, ABS_MT_POSITION_Y, 0, ABS_MAXVAL, 200, err); OK_OR_ABORT(err); setup_abs(fd, ABS_MT_PRESSURE, 0, ABS_MAXVAL, 0, err); OK_OR_ABORT(err); setup_abs(fd, ABS_MT_TOUCH_MAJOR, 0, ABS_MAXVAL, 12, err); OK_OR_ABORT(err); setup_abs(fd, ABS_MT_TOUCH_MINOR, 0, ABS_MAXVAL, 12, err); OK_OR_ABORT(err); // PointerEvent only gives partial orientation of the touch ellipse setup_abs(fd, ABS_MT_ORIENTATION, 0, 1, 0, err); OK_OR_ABORT(err); setup(fd, name, err); OK_OR_ABORT(err); if (ioctl(fd, UI_DEV_CREATE) < 0) ERROR(err, 1, "error: ioctl"); } int init_uinput_keyboard(const char* name, Error* err) { int device; if ((device = open("/dev/uinput", O_WRONLY | O_NONBLOCK)) < 0) fill_error(err, 101, "error: failed to open /dev/uinput"); else { init_keyboard(device, name, err); } return device; } int init_uinput_stylus(const char* name, Error* err) { int device; if ((device = open("/dev/uinput", O_WRONLY | O_NONBLOCK)) < 0) fill_error(err, 101, "error: failed to open /dev/uinput"); else { init_stylus(device, name, err); } return device; } int init_uinput_mouse(const char* name, Error* err) { int device; if ((device = open("/dev/uinput", O_WRONLY | O_NONBLOCK)) < 0) fill_error(err, 101, "error: failed to open /dev/uinput"); else { init_mouse(device, name, err); } return device; } int init_uinput_touch(const char* name, Error* err) { int device; if ((device = open("/dev/uinput", O_WRONLY | O_NONBLOCK)) < 0) fill_error(err, 101, "error: failed to open /dev/uinput"); else { init_touch(device, name, err); } return device; } void destroy_uinput_device(int fd) { ioctl(fd, UI_DEV_DESTROY); close(fd); } void send_uinput_event(int device, int type, int code, int value, Error* err) { struct input_event ev; ev.type = type; ev.code = code; ev.value = value; if (write(device, &ev, sizeof(ev)) < 0) ERROR(err, 1, "error writing to device, filedescriptor: %d)", device); } ================================================ FILE: lib/linux/uinput_info.md ================================================ # Some References on how to develop things using uinput - uinput: https://www.kernel.org/doc/html/latest/input/uinput.html - event codes: https://www.kernel.org/doc/html/latest/input/event-codes.html - multi-touch protocol: https://www.kernel.org/doc/html/latest/input/multi-touch-protocol.html ## Other projects using uinput - https://github.com/rfc2822/GfxTablet - https://github.com/bsteinsbo/rpi_touch_driver ## Debugging - libinput debug-events - evtest ================================================ FILE: lib/linux/xcapture.c ================================================ #include #include #include #include #include #include #include #include #include #include #include #include "../error.h" #include "../log.h" #include "xhelper.h" int clamp(int x, int lb, int ub) { if (x < lb) return lb; if (x > ub) return ub; return x; } struct CaptureContext { Capturable cap; XImage* ximg; XShmSegmentInfo shminfo; int has_xfixes; int has_offscreen; int wayland; Bool last_img_return; }; typedef struct CaptureContext CaptureContext; struct Image { char* data; unsigned int width; unsigned int height; }; void* start_capture(Capturable* cap, CaptureContext* ctx, Error* err) { if (XShmQueryExtension(cap->disp) != True) { fill_error(err, 1, "XShmExtension is not available but required!"); return NULL; } if (!ctx) { ctx = malloc(sizeof(CaptureContext)); int major, minor; Bool pixmaps = False; XShmQueryVersion(cap->disp, &major, &minor, &pixmaps); ctx->has_offscreen = pixmaps == True; if (ctx->has_offscreen && cap->type == WINDOW && cap->c.winfo.is_regular_window) { int event_base, error_base; ctx->has_offscreen = XCompositeQueryExtension(cap->disp, &event_base, &error_base) == True; if (ctx->has_offscreen) XCompositeRedirectWindow(cap->disp, cap->c.winfo.win, False); } const char* session_type = getenv("XDG_SESSION_TYPE"); if (session_type && strcmp(session_type, "wayland") == 0) ctx->wayland = 1; else ctx->wayland = 0; } ctx->cap = *cap; ctx->last_img_return = True; if (&ctx->cap != cap) strncpy(ctx->cap.name, cap->name, sizeof(ctx->cap.name)); int event_base, error_base; ctx->has_xfixes = XFixesQueryExtension(cap->disp, &event_base, &error_base) == True; int x, y; unsigned int width, height; get_geometry(cap, &x, &y, &width, &height, err); ctx->ximg = XShmCreateImage( cap->disp, DefaultVisualOfScreen(cap->screen), DefaultDepthOfScreen(cap->screen), ZPixmap, NULL, &ctx->shminfo, width, height); ctx->shminfo.shmid = shmget(IPC_PRIVATE, ctx->ximg->bytes_per_line * ctx->ximg->height, IPC_CREAT | 0777); ctx->shminfo.shmaddr = ctx->ximg->data = (char*)shmat(ctx->shminfo.shmid, 0, 0); ctx->shminfo.readOnly = False; if (ctx->shminfo.shmid < 0) { fill_error(err, 1, "Fatal shminfo error!"); free(ctx); return NULL; } if (!XShmAttach(cap->disp, &ctx->shminfo)) { fill_error(err, 1, "XShmAttach() failed"); free(ctx); return NULL; } return ctx; } void stop_capture(CaptureContext* ctx, Error* err) { XShmDetach(ctx->cap.disp, &ctx->shminfo); XDestroyImage(ctx->ximg); if (shmdt(ctx->shminfo.shmaddr) != 0) { fill_error(err, 1, "Failed to detach shared memory!"); } shmctl(ctx->shminfo.shmid, IPC_RMID, NULL); if (ctx->has_offscreen && ctx->cap.type == WINDOW && ctx->cap.c.winfo.is_regular_window) XCompositeUnredirectWindow(ctx->cap.disp, ctx->cap.c.winfo.win, False); free(ctx); } void capture_screen(CaptureContext* ctx, struct Image* img, int capture_cursor, Error* err) { Window root = DefaultRootWindow(ctx->cap.disp); int x, y; unsigned int width, height; get_geometry(&ctx->cap, &x, &y, &width, &height, err); OK_OR_ABORT(err); // if window resized, create new cap... if (width != (unsigned int)ctx->ximg->width || height != (unsigned int)ctx->ximg->height) { XShmDetach(ctx->cap.disp, &ctx->shminfo); XDestroyImage(ctx->ximg); shmdt(ctx->shminfo.shmaddr); shmctl(ctx->shminfo.shmid, IPC_RMID, NULL); CaptureContext* new_ctx = start_capture(&ctx->cap, ctx, err); if (!new_ctx) { return; } } Bool get_img_ret = False; switch (ctx->cap.type) { case WINDOW: { Window* active_window; unsigned long size; int is_offscreen = ctx->cap.c.winfo.is_regular_window && (x < 0 || y < 0 || x + (int)width > ctx->cap.screen->width || y + (int)height > ctx->cap.screen->height); active_window = (Window*)get_property(ctx->cap.disp, root, XA_WINDOW, "_NET_ACTIVE_WINDOW", &size, err); if (!ctx->wayland && *active_window == ctx->cap.c.winfo.win && !is_offscreen) { // cap window within its root so menus are visible as strictly speaking menus do not // belong to the window itself ... // But don't do this on (X)Wayland as the root window is just black in that case. get_img_ret = XShmGetImage(ctx->cap.disp, root, ctx->ximg, x, y, 0x00ffffff); } else { // ... but only if it is the active window as we might be recording the wrong thing // otherwise. If it is not active just record the window itself. // also if pixmaps are supported use those as they support capturing windows even if // they are offscreen if (is_offscreen) { if (ctx->has_offscreen) { Pixmap pm = XCompositeNameWindowPixmap(ctx->cap.disp, ctx->cap.c.winfo.win); get_img_ret = XShmGetImage(ctx->cap.disp, pm, ctx->ximg, 0, 0, 0x00ffffff); XFreePixmap(ctx->cap.disp, pm); } else ERROR( err, 1, "Can not capture window as it is off screen and Xcomposite is " "unavailable!"); } else get_img_ret = XShmGetImage(ctx->cap.disp, ctx->cap.c.winfo.win, ctx->ximg, 0, 0, 0x00ffffff); } free(active_window); break; } case RECT: get_img_ret = XShmGetImage(ctx->cap.disp, root, ctx->ximg, x, y, 0x00ffffff); break; } Bool last_img_return = ctx->last_img_return; ctx->last_img_return = get_img_ret; // only print an error once and do not repeat this message if consecutive calls to XShmGetImage // fail to avoid spamming the logs. if (get_img_ret != True) { if (last_img_return != get_img_ret) { ERROR(err, 1, "XShmGetImage failed!"); } else { ERROR(err, 2, "XShmGetImage failed!"); } } // capture cursor if requested and if XFixes is available if (capture_cursor && ctx->has_xfixes) { XFixesCursorImage* cursor_img = XFixesGetCursorImage(ctx->cap.disp); if (cursor_img != NULL) { uint32_t* data = (uint32_t*)ctx->ximg->data; // coordinates of cursor inside ximg int x0 = cursor_img->x - cursor_img->xhot - x; int y0 = cursor_img->y - cursor_img->yhot - y; // clamp part of cursor image to draw to the part of the cursor that is inside // the captured area int i0 = clamp(0, -x0, width - x0); int i1 = clamp(cursor_img->width, -x0, width - x0); int j0 = clamp(0, -y0, height - y0); int j1 = clamp(cursor_img->height, -y0, height - y0); // paint cursor image into captured image for (int j = j0; j < j1; ++j) for (int i = i0; i < i1; ++i) { uint32_t c_pixel = cursor_img->pixels[j * cursor_img->width + i]; unsigned char a = (c_pixel & 0xff000000) >> 24; if (a) { uint32_t d_pixel = data[(j + y0) * width + i + x0]; unsigned char c1 = (c_pixel & 0x00ff0000) >> 16; unsigned char c2 = (c_pixel & 0x0000ff00) >> 8; unsigned char c3 = (c_pixel & 0x000000ff) >> 0; unsigned char d1 = (d_pixel & 0x00ff0000) >> 16; unsigned char d2 = (d_pixel & 0x0000ff00) >> 8; unsigned char d3 = (d_pixel & 0x000000ff) >> 0; // colors from the cursor image are premultiplied with the alpha channel unsigned char f1 = c1 + d1 * (255 - a) / 255; unsigned char f2 = c2 + d2 * (255 - a) / 255; unsigned char f3 = c3 + d3 * (255 - a) / 255; data[(j + y0) * width + i + x0] = (f1 << 16) | (f2 << 8) | (f3 << 0); } } XFree(cursor_img); } else { log_warn( "Failed to obtain cursor image, XFixesGetCursorImage has returned a null pointer."); } } img->width = ctx->ximg->width; img->height = ctx->ximg->height; img->data = ctx->ximg->data; } ================================================ FILE: lib/linux/xhelper.c ================================================ #include #include #include #include #include #include #include #include #include "../error.h" #include "../log.h" #include "xhelper.h" int x11_error_handler(Display* disp, XErrorEvent* err) { char buf1[128], buf2[128], message_selector[64]; XGetErrorText(disp, err->error_code, buf1, sizeof(buf1)); snprintf(message_selector, sizeof(message_selector), "XRequest.%d", err->request_code); XGetErrorDatabaseText(disp, "", message_selector, message_selector, buf2, sizeof(buf2)); log_debug("X11 error: %s: %s 0x%lx", buf1, buf2, err->resourceid); return 0; } void x11_set_error_handler() { // setting an error handler is required as otherwise xlib may just exit the process, even though // the error was recoverable. XSetErrorHandler(x11_error_handler); } int locale_to_utf8(char* src, char* dest, size_t size) { iconv_t icd = iconv_open("UTF-8//IGNORE", ""); size_t src_size = size; size_t outbytes_left = MAX_PROPERTY_VALUE_LEN - 1; int ret = iconv(icd, &src, &src_size, &dest, &outbytes_left); iconv_close(icd); if (ret < 0) { return -1; } dest[src_size - 1 - outbytes_left] = '\0'; return 0; } char* get_property( Display* disp, Window win, Atom xa_prop_type, char* prop_name, unsigned long* size, Error* err) { Atom xa_prop_name; Atom xa_ret_type; int ret_format; unsigned long ret_nitems; unsigned long ret_bytes_after; unsigned long tmp_size; unsigned char* ret_prop; char* ret; xa_prop_name = XInternAtom(disp, prop_name, False); /* MAX_PROPERTY_VALUE_LEN / 4 explanation (XGetWindowProperty manpage): * * long_length = Specifies the length in 32-bit multiples of the * data to be retrieved. */ if (XGetWindowProperty( disp, win, xa_prop_name, 0, MAX_PROPERTY_VALUE_LEN / 4, False, xa_prop_type, &xa_ret_type, &ret_format, &ret_nitems, &ret_bytes_after, &ret_prop) != Success) { fill_error(err, 1, "Cannot get %s property.", prop_name); return NULL; } if (xa_ret_type != xa_prop_type) { fill_error(err, 1, "Invalid type of %s property.", prop_name); XFree(ret_prop); return NULL; } /* null terminate the result to make string handling easier */ tmp_size = (ret_format / 8) * ret_nitems; /* Correct 64 Architecture implementation of 32 bit data */ if (ret_format == 32) tmp_size *= sizeof(long) / 4; ret = malloc(tmp_size + 1); memcpy(ret, ret_prop, tmp_size); ret[tmp_size] = '\0'; if (size) { *size = tmp_size; } XFree(ret_prop); return ret; } char* get_window_title(Display* disp, Window win, Error* err) { char* title_utf8; char* wm_name; char* net_wm_name; Error err_wm; Error err_net_wm; wm_name = get_property(disp, win, XA_STRING, "WM_NAME", NULL, &err_wm); net_wm_name = get_property( disp, win, XInternAtom(disp, "UTF8_STRING", False), "_NET_WM_NAME", NULL, &err_net_wm); if (net_wm_name) { title_utf8 = strdup(net_wm_name); } else { if (wm_name) { title_utf8 = malloc(MAX_PROPERTY_VALUE_LEN); if (locale_to_utf8(wm_name, title_utf8, MAX_PROPERTY_VALUE_LEN) != 0) { fill_error(err, 1, "Failed to convert windowname to UTF-8!"); free(title_utf8); title_utf8 = NULL; } } else { fill_error( err, 1, "Could not get window name: (%s) (%s)", err_net_wm.error_str, err_wm.error_str); title_utf8 = NULL; } } free(wm_name); free(net_wm_name); return title_utf8; } Window* get_client_list(Display* disp, unsigned long* size, Error* err) { Window* client_list; Error err_net; Error err_win; if ((client_list = (Window*)get_property( disp, DefaultRootWindow(disp), XA_WINDOW, "_NET_CLIENT_LIST", size, &err_net)) == NULL) { if ((client_list = (Window*)get_property( disp, DefaultRootWindow(disp), XA_CARDINAL, "_WIN_CLIENT_LIST", size, &err_win)) == NULL) { fill_error( err, 2, "Cannot get client list properties. " "_NET_CLIENT_LIST: %s or _WIN_CLIENT_LIST: %s", err_net.error_str, err_win.error_str); return NULL; } } return client_list; } int create_capturables( Display* disp, Capturable** capturables, int* num_monitors, int size, Error* err) { if (size <= 0) return 0; int screen = DefaultScreen(disp); Window root = RootWindow(disp, screen); int event_base, error_base, major, minor; *num_monitors = 0; XRRMonitorInfo* monitors = NULL; if (XRRQueryExtension(disp, &event_base, &error_base) && XRRQueryVersion(disp, &major, &minor)) { monitors = XRRGetMonitors(disp, root, True, num_monitors); if (*num_monitors < 0) { *num_monitors = 0; fill_error(err, 2, "Failed to query monitor info via xrandr."); } } else { fill_error(err, 2, "Xrandr is unsupported on this X server."); } Window* client_list; unsigned long client_list_size; size_t num_windows = ((client_list = get_client_list(disp, &client_list_size, err)) == NULL) ? 0 : client_list_size / sizeof(Window); size_t i = 0; Capturable* c = malloc(sizeof(Capturable)); capturables[i] = c; c->disp = disp; c->screen = ScreenOfDisplay(disp, screen); strncpy(c->name, "Desktop", sizeof(c->name) - 1); c->type = WINDOW; c->c.winfo.win = root; c->c.winfo.is_regular_window = 0; ++i; for (; i < (size_t)*num_monitors + 1 && i < (size_t)size; ++i) { Capturable* c = malloc(sizeof(Capturable)); capturables[i] = c; XRRMonitorInfo* m = &monitors[i - 1]; c->disp = disp; c->screen = ScreenOfDisplay(disp, screen); char* name = XGetAtomName(disp, m->name); snprintf(c->name, sizeof(c->name) - 1, "Monitor: %s", name); XFree(name); c->type = RECT; c->c.rinfo.x = m->x; c->c.rinfo.y = m->y; c->c.rinfo.width = m->width; c->c.rinfo.height = m->height; } for (; i < num_windows + *num_monitors + 1 && i < (size_t)size; ++i) { size_t j = i - *num_monitors - 1; char* title_utf8 = get_window_title(disp, client_list[j], NULL); if (title_utf8 == NULL) { title_utf8 = malloc(32); snprintf(title_utf8, 32, "UNKNOWN %lu", j); } Capturable* c = malloc(sizeof(Capturable)); capturables[i] = c; c->disp = disp; c->screen = ScreenOfDisplay(disp, screen); c->type = WINDOW; strncpy(c->name, title_utf8, sizeof(c->name) - 1); c->c.winfo.win = client_list[j]; c->c.winfo.is_regular_window = 1; free(title_utf8); } free(client_list); XRRFreeMonitors(monitors); return i; } void* clone_capturable(Capturable* c) { Capturable* c2 = malloc(sizeof(Capturable)); *c2 = *c; memcpy(c2->name, c->name, sizeof(c2->name)); return c2; } void destroy_capturable(Capturable* c) { free(c); } void get_window_geometry( Display* disp, Window win, int* x, int* y, unsigned int* width, unsigned int* height, Error* err) { Window junkroot; int junkx, junky; unsigned int bw, depth; if (!XGetGeometry(disp, win, &junkroot, &junkx, &junky, width, height, &bw, &depth)) { ERROR(err, 1, "Failed to get window geometry!"); } XTranslateCoordinates(disp, win, junkroot, 0, 0, x, y, &junkroot); } void get_geometry( Capturable* cap, int* x, int* y, unsigned int* width, unsigned int* height, Error* err) { switch (cap->type) { case WINDOW: get_window_geometry(cap->disp, cap->c.winfo.win, x, y, width, height, err); return; case RECT: *x = cap->c.rinfo.x; *y = cap->c.rinfo.y; *width = cap->c.rinfo.width; *height = cap->c.rinfo.height; return; } } void get_geometry_relative( Capturable* cap, float* x, float* y, float* width, float* height, Error* err) { int x_tmp, y_tmp; unsigned int width_tmp, height_tmp; get_geometry(cap, &x_tmp, &y_tmp, &width_tmp, &height_tmp, err); OK_OR_ABORT(err); *x = x_tmp / (float)cap->screen->width; *y = y_tmp / (float)cap->screen->height; *width = width_tmp / (float)cap->screen->width; *height = height_tmp / (float)cap->screen->height; } void client_msg( Display* disp, Window win, char* msg, unsigned long data0, unsigned long data1, unsigned long data2, unsigned long data3, unsigned long data4, Error* err) { XEvent event; long mask = SubstructureRedirectMask | SubstructureNotifyMask; event.xclient.type = ClientMessage; event.xclient.serial = 0; event.xclient.send_event = True; event.xclient.message_type = XInternAtom(disp, msg, False); event.xclient.window = win; event.xclient.format = 32; event.xclient.data.l[0] = data0; event.xclient.data.l[1] = data1; event.xclient.data.l[2] = data2; event.xclient.data.l[3] = data3; event.xclient.data.l[4] = data4; if (!XSendEvent(disp, DefaultRootWindow(disp), False, mask, &event)) { ERROR(err, 1, "Cannot send %s event.", msg); } } void activate_window(Display* disp, WindowInfo* winfo, Error* err) { // do not activate windows like the root window or root windows of a screen if (!winfo->is_regular_window) return; Window* active_window = 0; unsigned long size; active_window = (Window*)get_property( disp, DefaultRootWindow(disp), XA_WINDOW, "_NET_ACTIVE_WINDOW", &size, err); if (*active_window == winfo->win) { // nothing to do window is active already free(active_window); return; } free(active_window); unsigned long* desktop; /* desktop ID */ if ((desktop = (unsigned long*)get_property( disp, winfo->win, XA_CARDINAL, "_NET_WM_DESKTOP", NULL, err)) == NULL) { if ((desktop = (unsigned long*)get_property( disp, winfo->win, XA_CARDINAL, "_WIN_WORKSPACE", NULL, err)) == NULL) { ERROR(err, 1, "Cannot find desktop ID of the window."); } } client_msg(disp, DefaultRootWindow(disp), "_NET_CURRENT_DESKTOP", *desktop, 0, 0, 0, 0, err); free(desktop); OK_OR_ABORT(err); client_msg(disp, winfo->win, "_NET_ACTIVE_WINDOW", 0, 0, 0, 0, 0, err); OK_OR_ABORT(err); XMapRaised(disp, winfo->win); } void capturable_before_input(Capturable* cap, Error* err) { switch (cap->type) { case WINDOW: activate_window(cap->disp, &cap->c.winfo, err); break; case RECT: break; } } const char* get_capturable_name(Capturable* c) { return c->name; } void map_input_device_to_entire_screen(Display* disp, const char* device_name, int pen, Error* err) { // for some reason a device simualting a stylus does NOT create a single device in // XListInputDevices but actually two: One with the original name and the other one with // "Pen (0)" appended to it. The problem is that the original device does NOT permit setting // "Coordinate Transformation Matrix". This can only be done for the device with "Pen (0)" // appended. So this here is a dirty workaround assuming the configurable stylus/pen device is // always called original name + "Pen" + whatever. char pen_name[256]; if (pen) snprintf(pen_name, sizeof(pen_name), "%s Pen", device_name); XID device_id; int num_devices = 0; XDeviceInfo* devices = XListInputDevices(disp, &num_devices); int found = 0; for (int i = 0; i < num_devices; ++i) { if ((!pen && strcmp(device_name, devices[i].name) == 0) || (pen && strncmp(pen_name, devices[i].name, strlen(pen_name)) == 0)) { device_id = devices[i].id; found = 1; break; } } XFreeDeviceList(devices); if (!found) ERROR(err, 2, "Device with name: %s not found!", device_name); Atom prop_float, prop_matrix; union { unsigned char* c; float* f; } data; int format_return; Atom type_return; unsigned long nitems; unsigned long bytes_after; int rc; prop_float = XInternAtom(disp, "FLOAT", False); prop_matrix = XInternAtom(disp, "Coordinate Transformation Matrix", False); if (!prop_float) { ERROR(err, 1, "Float atom not found. This server is too old."); } if (!prop_matrix) { ERROR( err, 1, "Coordinate transformation matrix not found. This " "server is too old."); } rc = XIGetProperty( disp, device_id, prop_matrix, 0, 9, False, prop_float, &type_return, &format_return, &nitems, &bytes_after, &data.c); if (rc != Success || prop_float != type_return || format_return != 32 || nitems != 9 || bytes_after != 0) { ERROR(err, 1, "Failed to retrieve current property values."); } data.f[0] = 1.0; data.f[1] = 0.0; data.f[2] = 0.0; data.f[3] = 0.0; data.f[4] = 1.0; data.f[5] = 0.0; data.f[6] = 0.0; data.f[7] = 0.0; data.f[8] = 1.0; XIChangeProperty( disp, device_id, prop_matrix, prop_float, format_return, PropModeReplace, data.c, nitems); XFree(data.c); } ================================================ FILE: lib/linux/xhelper.h ================================================ #pragma once #include #include #include #include #include #include #include #include "../error.h" #define MAX_PROPERTY_VALUE_LEN 4096 typedef struct WindowInfo { Window win; int is_regular_window; } WindowInfo; typedef struct RectInfo { int x; int y; unsigned int width; unsigned int height; } RectInfo; typedef enum CaptureType { WINDOW, RECT } CaptureType; typedef struct Capturable { CaptureType type; char name[128]; Display* disp; Screen* screen; union { WindowInfo winfo; RectInfo rinfo; } c; } Capturable; char* get_property( Display* disp, Window win, Atom xa_prop_type, char* prop_name, unsigned long* size, Error* err); void get_geometry( Capturable* cap, int* x, int* y, unsigned int* width, unsigned int* height, Error* err); void get_geometry_relative( Capturable* cap, float* x, float* y, float* width, float* height, Error* err); ================================================ FILE: lib/log.c ================================================ #include "log.h" // rust functions living in log.rs void log_error_rust(const char*); void log_debug_rust(const char*); void log_info_rust(const char*); void log_trace_rust(const char*); void log_warn_rust(const char*); void log_error(const char* fmt, ...) { va_list args; va_start(args, fmt); char buf[2048]; vsnprintf(buf, sizeof(buf), fmt, args); log_error_rust(buf); } void log_debug(const char* fmt, ...) { va_list args; va_start(args, fmt); char buf[2048]; vsnprintf(buf, sizeof(buf), fmt, args); log_debug_rust(buf); } void log_info(const char* fmt, ...) { va_list args; va_start(args, fmt); char buf[2048]; vsnprintf(buf, sizeof(buf), fmt, args); log_info_rust(buf); } void log_trace(const char* fmt, ...) { va_list args; va_start(args, fmt); char buf[2048]; vsnprintf(buf, sizeof(buf), fmt, args); log_trace_rust(buf); } void log_warn(const char* fmt, ...) { va_list args; va_start(args, fmt); char buf[2048]; vsnprintf(buf, sizeof(buf), fmt, args); log_warn_rust(buf); } ================================================ FILE: lib/log.h ================================================ #pragma once #include #include #if defined(__clang__) || defined(__GNUC__) __attribute__((__format__ (__printf__, 1, 2))) void log_error(const char* fmt, ...); __attribute__((__format__ (__printf__, 1, 2))) void log_debug(const char* fmt, ...); __attribute__((__format__ (__printf__, 1, 2))) void log_info(const char* fmt, ...); __attribute__((__format__ (__printf__, 1, 2))) void log_trace(const char* fmt, ...); __attribute__((__format__ (__printf__, 1, 2))) void log_warn(const char* fmt, ...); #else void log_error(const char* fmt, ...); void log_debug(const char* fmt, ...); void log_info(const char* fmt, ...); void log_trace(const char* fmt, ...); void log_warn(const char* fmt, ...); #endif ================================================ FILE: src/capturable/captrs_capture.rs ================================================ use crate::capturable::{Capturable, Recorder}; use captrs::Capturer; use std::boxed::Box; use std::error::Error; use winapi::shared::windef::RECT; use super::Geometry; #[derive(Clone)] pub struct CaptrsCapturable { id: u8, name: String, screen: RECT, virtual_screen: RECT, } impl CaptrsCapturable { pub fn new(id: u8, name: String, screen: RECT, virtual_screen: RECT) -> CaptrsCapturable { CaptrsCapturable { id, name, screen, virtual_screen, } } } impl Capturable for CaptrsCapturable { fn name(&self) -> String { format!("Desktop {} (captrs)", self.name).into() } fn before_input(&mut self) -> Result<(), Box> { Ok(()) } fn recorder(&self, _capture_cursor: bool) -> Result, Box> { Ok(Box::new(CaptrsRecorder::new(self.id)?)) } fn geometry(&self) -> Result> { Ok(Geometry::VirtualScreen( self.screen.left - self.virtual_screen.left, self.screen.top - self.virtual_screen.top, (self.screen.right - self.screen.left) as u32, (self.screen.bottom - self.screen.top) as u32, self.screen.left, self.screen.top, )) } } #[derive(Debug)] pub struct CaptrsError(String); impl std::fmt::Display for CaptrsError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let Self(s) = self; write!(f, "{}", s) } } impl Error for CaptrsError {} pub struct CaptrsRecorder { capturer: Capturer, } impl CaptrsRecorder { pub fn new(id: u8) -> Result> { Ok(CaptrsRecorder { capturer: Capturer::new(id.into())?, }) } } impl Recorder for CaptrsRecorder { fn capture(&mut self) -> Result> { self.capturer .capture_store_frame() .map_err(|_e| CaptrsError("Captrs failed to capture frame".into()))?; let (w, h) = self.capturer.geometry(); Ok(crate::video::PixelProvider::BGR0( w as usize, h as usize, unsafe { std::mem::transmute(self.capturer.get_stored_frame().unwrap()) }, )) } } ================================================ FILE: src/capturable/core_graphics.rs ================================================ use std::boxed::Box; use std::error::Error; use std::ffi::c_void; use std::time::{Duration, Instant}; use core_foundation::{ array::CFArray, base::{TCFType, ToVoid}, data::CFData, dictionary::{CFDictionary, CFDictionaryRef}, number::{CFNumber, CFNumberRef}, string::{CFString, CFStringRef}, }; use core_graphics::{ display, display::{CGDisplay, CGRect}, image::CGImage, window, window::CGWindowID, }; use crate::capturable::{Capturable, Geometry, Recorder}; #[derive(Debug)] pub struct CGError(String); impl std::fmt::Display for CGError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let Self(s) = self; write!(f, "{}", s) } } impl Error for CGError {} #[derive(Clone)] pub struct CGDisplayCapturable { display: CGDisplay, } impl CGDisplayCapturable { pub fn new(display: CGDisplay) -> Self { Self { display } } } impl Capturable for CGDisplayCapturable { fn name(&self) -> String { format!( "Monitor (CG, {}x{})", self.display.pixels_wide(), self.display.pixels_high() ) } fn geometry(&self) -> Result> { let bounds = self.display.bounds(); let (x0, y0, w, h) = screen_coordsys()?; Ok(Geometry::Relative( (bounds.origin.x - x0) / w, (bounds.origin.y - y0) / h, bounds.size.width / w, bounds.size.height / h, )) } fn before_input(&mut self) -> Result<(), Box> { Ok(()) } fn recorder(&self, capture_cursor: bool) -> Result, Box> { Ok(Box::new(RecorderCGDisplay::new( self.display, capture_cursor, ))) } } pub struct RecorderCGDisplay { img_data: Option, display: CGDisplay, capture_cursor: bool, } impl RecorderCGDisplay { pub fn new(display: CGDisplay, capture_cursor: bool) -> Self { Self { img_data: None, display, capture_cursor, } } } fn check_pixelformat(img: &CGImage) -> Result<(), Box> { // for now assume that the pixels are always in BGR0 format // do some basic checks to verify this if img.bits_per_pixel() != 32 { Err(CGError(format!( "Only BGR0 with 32 bits per pixel is supported, not {} bits!", img.bits_per_pixel() )))? } if img.bits_per_component() != 8 { Err(CGError(format!( "Only BGR0 with 8 bits per component is supported, not {} bits!", img.bits_per_component() )))? } Ok(()) } impl Recorder for RecorderCGDisplay { fn capture(&mut self) -> Result> { let img = if self.capture_cursor { CGDisplay::screenshot(self.display.bounds(), 0, 0, 0) } else { self.display.image() }; if let Some(img) = img { check_pixelformat(&img)?; let w = img.width() as usize; let h = img.height() as usize; // extract raw image data self.img_data = Some(img.data()); Ok(crate::video::PixelProvider::BGR0S( w, h, img.bytes_per_row(), self.img_data.as_ref().unwrap().bytes(), )) } else { Err(Box::new(CGError( "Failed to capture screen using CoreGraphics.".into(), ))) } } } #[derive(Clone)] pub struct CGWindowCapturable { id: CGWindowID, name: String, cursor_id: CGWindowID, bounds: CGRect, geometry_relative: (f64, f64, f64, f64), last_geometry_update: Instant, } impl CGWindowCapturable { fn update_geometry(&mut self) -> Result<(), Box> { if Instant::now() - self.last_geometry_update > Duration::from_secs(1) { self.bounds = get_window_infos() .iter() .find(|w| w.id == self.id) .ok_or_else(|| { CGError(format!( "Could not find information for current window {}.", self.id )) })? .bounds; let (x0, y0, w, h) = screen_coordsys()?; self.geometry_relative = ( (self.bounds.origin.x - x0) / w, (self.bounds.origin.y - y0) / h, self.bounds.size.width / w, self.bounds.size.height / h, ); self.last_geometry_update = Instant::now(); } Ok(()) } } impl Capturable for CGWindowCapturable { fn name(&self) -> String { self.name.clone() } fn geometry(&self) -> Result> { let (x, y, w, h) = self.geometry_relative; Ok(Geometry::Relative(x, y, w, h)) } fn before_input(&mut self) -> Result<(), Box> { self.update_geometry() } fn recorder(&self, capture_cursor: bool) -> Result, Box> { Ok(Box::new(RecorderCGWindow { img_data: None, capture_cursor, win: self.clone(), })) } } pub struct RecorderCGWindow { img_data: Option, capture_cursor: bool, win: CGWindowCapturable, } impl Recorder for RecorderCGWindow { fn capture(&mut self) -> Result> { self.win.update_geometry()?; let img = CGDisplay::screenshot_from_windows( self.win.bounds, if self.capture_cursor { CFArray::from_copyable(&[ self.win.cursor_id as *const c_void, self.win.id as *const c_void, ]) } else { CFArray::from_copyable(&[self.win.id as *const c_void]) }, 0, ); if let Some(img) = img { check_pixelformat(&img)?; let w = img.width() as usize; let h = img.height() as usize; // extract raw image data self.img_data = Some(img.data()); Ok(crate::video::PixelProvider::BGR0S( w, h, img.bytes_per_row(), self.img_data.as_ref().unwrap().bytes(), )) } else { Err(Box::new(CGError( "Failed to capture window using CoreGraphics.".into(), ))) } } } #[derive(Debug)] struct WindowInfo { pub id: CGWindowID, pub name: String, pub bounds: CGRect, } fn get_window_infos() -> Vec { let mut win_infos = vec![]; let wins = CGDisplay::window_list_info( display::kCGWindowListExcludeDesktopElements | display::kCGWindowListOptionOnScreenOnly, None, ); if let Some(wins) = wins { for w in wins.iter() { let w: CFDictionary<*const c_void, *const c_void> = unsafe { CFDictionary::wrap_under_get_rule(*w as CFDictionaryRef) }; let id = w.get(unsafe { window::kCGWindowNumber }.to_void()); let id = unsafe { CFNumber::wrap_under_get_rule(*id as CFNumberRef) } .to_i64() .unwrap() as CGWindowID; let bounds = w.get(unsafe { window::kCGWindowBounds }.to_void()); let bounds = unsafe { CFDictionary::wrap_under_get_rule(*bounds as CFDictionaryRef) }; let bounds = CGRect::from_dict_representation(&bounds).unwrap(); let name = match w.find(unsafe { window::kCGWindowName }.to_void()) { Some(n) => n, None => continue, }; let name = unsafe { CFString::wrap_under_get_rule(*name as CFStringRef) }; win_infos.push(WindowInfo { id, name: name.to_string(), bounds, }); } } win_infos } pub fn screen_coordsys() -> Result<(f64, f64, f64, f64), Box> { let display_ids = CGDisplay::active_displays() .map_err(|err| CGError(format!("Failed to obtain displays, CGError code: {}", err)))?; let rects: Vec = display_ids .iter() .map(|id| CGDisplay::new(*id).bounds()) .collect(); let mut x0 = 0.0; let mut x1 = 0.0; let mut y0 = 0.0; let mut y1 = 0.0; for r in rects.iter() { let r_x0 = r.origin.x; let r_x1 = r_x0 + r.size.width; let r_y0 = r.origin.y; let r_y1 = r_y0 + r.size.height; x0 = f64::min(x0, r_x0); x1 = f64::max(x1, r_x1); y0 = f64::min(y0, r_y0); y1 = f64::max(y1, r_y1); } Ok((x0, y0, x1 - x0, y1 - y0)) } pub fn get_displays() -> Result, Box> { let display_ids = CGDisplay::active_displays() .map_err(|err| CGError(format!("Failed to obtain displays, CGError code: {}", err)))?; Ok(display_ids .iter() .map(|id| CGDisplayCapturable::new(CGDisplay::new(*id))) .collect()) } pub fn get_windows() -> Result, Box> { let window_infos = get_window_infos(); let cursor_id = window_infos .iter() .find(|w| w.name == "Cursor") .ok_or_else(|| CGError("No Cursor found!".into()))? .id; Ok(window_infos .iter() .filter(|w| w.id != cursor_id) .map(|w| CGWindowCapturable { id: w.id, name: w.name.clone(), cursor_id, bounds: w.bounds, geometry_relative: (0.0, 0.0, 1.0, 1.0), last_geometry_update: Instant::now() - Duration::from_secs(2), }) .collect()) } ================================================ FILE: src/capturable/mod.rs ================================================ use std::boxed::Box; use std::error::Error; use tracing::warn; #[cfg(target_os = "macos")] pub mod core_graphics; #[cfg(target_os = "linux")] pub mod pipewire; #[cfg(target_os = "linux")] #[allow(dead_code)] pub mod remote_desktop_dbus; pub mod testsrc; #[cfg(target_os = "windows")] pub mod captrs_capture; #[cfg(target_os = "windows")] pub mod win_ctx; #[cfg(target_os = "linux")] pub mod x11; pub trait Recorder { fn capture(&mut self) -> Result, Box>; } pub trait BoxCloneCapturable { fn box_clone(&self) -> Box; } impl BoxCloneCapturable for T where T: Clone + Capturable + 'static, { fn box_clone(&self) -> Box { Box::new(self.clone()) } } /// Relative: x, y, width, height of the Capturable as floats relative to the absolute size of the /// screen. For example x=0.5, y=0.0, width=0.5, height=1.0 means the right half of the screen. /// VirtualScreen: offset_x, offset_y, width, height for a capturable using a virtual screen. (Windows) pub enum Geometry { Relative(f64, f64, f64, f64), #[cfg(target_os = "windows")] VirtualScreen(i32, i32, u32, u32, i32, i32), } pub trait Capturable: Send + BoxCloneCapturable { /// Name of the Capturable, for example the window title, if it is a window. fn name(&self) -> String; /// Return Geometry of the Capturable. fn geometry(&self) -> Result>; /// Callback that is called right before input is simulated. /// Useful to focus the window on input. fn before_input(&mut self) -> Result<(), Box>; /// Return a Recorder that can record the current capturable. fn recorder(&self, capture_cursor: bool) -> Result, Box>; } impl Clone for Box { fn clone(&self) -> Self { self.box_clone() } } pub fn get_capturables( #[cfg(target_os = "linux")] wayland_support: bool, #[cfg(target_os = "linux")] capture_cursor: bool, ) -> Vec> { let mut capturables: Vec> = vec![]; #[cfg(target_os = "linux")] { if wayland_support { use crate::capturable::pipewire::get_capturables as get_capturables_pw; match get_capturables_pw(capture_cursor) { Ok(captrs) => { for c in captrs { capturables.push(Box::new(c)); } } Err(err) => warn!( "Failed to get list of capturables via dbus/pipewire: {}", err ), } } use crate::capturable::x11::X11Context; let x11ctx = X11Context::new(); if let Some(mut x11ctx) = x11ctx { match x11ctx.capturables() { Ok(captrs) => { for c in captrs { capturables.push(Box::new(c)); } } Err(err) => warn!("Failed to get list of capturables via X11: {}", err), } }; } #[cfg(target_os = "macos")] { use crate::capturable::core_graphics::get_displays as get_displays_cg; use crate::capturable::core_graphics::get_windows as get_windows_cg; match get_displays_cg() { Ok(captrs) => { for c in captrs { capturables.push(Box::new(c)); } } Err(err) => warn!("Failed to get list of displays via CoreGraphics: {}", err), } match get_windows_cg() { Ok(mut captrs) => { captrs.sort_by(|a, b| a.name().to_lowercase().cmp(&b.name().to_lowercase())); for c in captrs { capturables.push(Box::new(c)); } } Err(err) => warn!("Failed to get list of windows via CoreGraphics: {}", err), } } #[cfg(target_os = "windows")] { use crate::capturable::captrs_capture::CaptrsCapturable; use crate::capturable::win_ctx::WinCtx; let winctx = WinCtx::new(); for (i, o) in winctx.get_outputs().iter().enumerate() { let captr = CaptrsCapturable::new( i as u8, String::from_utf16_lossy(o.DeviceName.as_ref()), o.DesktopCoordinates, winctx.get_union_rect().clone(), ); capturables.push(Box::new(captr)); } } if crate::log::get_log_level() >= tracing::Level::DEBUG { for (width, height) in [ (200, 200), (800, 600), (1080, 720), (1920, 1080), (3840, 2160), (15360, 2160), ] .iter() { use testsrc::PixelFormat; for pixel_format in [PixelFormat::BGR0, PixelFormat::RGB0, PixelFormat::RGB] { capturables.push(Box::new(testsrc::TestCapturable { width: *width, height: *height, pixel_format, })); } } } capturables } ================================================ FILE: src/capturable/pipewire.rs ================================================ use std::collections::HashMap; use std::error::Error; use std::os::unix::io::AsRawFd; use std::sync::{Arc, Mutex}; use std::time::Duration; use tracing::{debug, trace, warn}; use dbus::{ arg::{OwnedFd, PropMap, RefArg, Variant}, blocking::{Proxy, SyncConnection}, message::{MatchRule, MessageType}, Message, }; use gstreamer as gst; use gstreamer::prelude::*; use gstreamer_app::AppSink; use crate::capturable::{Capturable, Geometry, Recorder}; use crate::video::PixelProvider; use crate::capturable::remote_desktop_dbus::{ OrgFreedesktopPortalRemoteDesktop, OrgFreedesktopPortalRequestResponse, OrgFreedesktopPortalScreenCast, }; #[derive(Debug, Clone, Copy)] struct PwStreamInfo { path: u64, source_type: u64, } #[derive(Debug)] pub struct DBusError(String); impl std::fmt::Display for DBusError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let Self(s) = self; write!(f, "{}", s) } } impl Error for DBusError {} #[derive(Debug)] pub struct GStreamerError(String); impl std::fmt::Display for GStreamerError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let Self(s) = self; write!(f, "{}", s) } } impl Error for GStreamerError {} #[derive(Clone)] pub struct PipeWireCapturable { // connection needs to be kept alive for recording dbus_conn: Arc, fd: OwnedFd, path: u64, source_type: u64, } impl PipeWireCapturable { fn new(conn: Arc, fd: OwnedFd, stream: PwStreamInfo) -> Self { Self { dbus_conn: conn, fd, path: stream.path, source_type: stream.source_type, } } } impl std::fmt::Debug for PipeWireCapturable { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, "PipeWireCapturable {{dbus: {}, fd: {}, path: {}, source_type: {}}}", self.dbus_conn.unique_name(), self.fd.as_raw_fd(), self.path, self.source_type ) } } impl Capturable for PipeWireCapturable { fn name(&self) -> String { let type_str = match self.source_type { 1 => "Desktop", 2 => "Window", _ => "Unknown", }; format!("Pipewire {}, path: {}", type_str, self.path) } fn geometry(&self) -> Result> { Ok(Geometry::Relative(0.0, 0.0, 1.0, 1.0)) } fn before_input(&mut self) -> Result<(), Box> { Ok(()) } fn recorder(&self, _capture_cursor: bool) -> Result, Box> { Ok(Box::new(PipeWireRecorder::new(self.clone())?)) } } pub struct PipeWireRecorder { buffer: Option>, buffer_cropped: Vec, pix_fmt: String, is_cropped: bool, pipeline: gst::Pipeline, appsink: AppSink, width: usize, height: usize, } impl PipeWireRecorder { pub fn new(capturable: PipeWireCapturable) -> Result> { let pipeline = gst::Pipeline::new(); let src = gst::ElementFactory::make("pipewiresrc").build()?; src.set_property("fd", &capturable.fd.as_raw_fd()); src.set_property("path", &format!("{}", capturable.path)); // For some reason pipewire blocks on destruction of AppSink if this is not set to true, // see: https://gitlab.freedesktop.org/pipewire/pipewire/-/issues/982 src.set_property("always-copy", &true); let sink = gst::ElementFactory::make("appsink").build()?; sink.set_property("drop", &true); sink.set_property("max-buffers", &1u32); pipeline.add_many(&[&src, &sink])?; src.link(&sink)?; let appsink = sink .dynamic_cast::() .map_err(|_| GStreamerError("Sink element is expected to be an appsink!".into()))?; let mut caps = gst::Caps::new_empty(); caps.merge_structure(gst::structure::Structure::from_iter( "video/x-raw", [("format", "BGRx".into())], )); caps.merge_structure(gst::structure::Structure::from_iter( "video/x-raw", [("format", "RGBx".into())], )); appsink.set_caps(Some(&caps)); pipeline.set_state(gst::State::Playing)?; Ok(Self { pipeline, appsink, buffer: None, pix_fmt: "".into(), width: 0, height: 0, buffer_cropped: vec![], is_cropped: false, }) } } impl Recorder for PipeWireRecorder { fn capture(&mut self) -> Result, Box> { if let Some(sample) = self .appsink .try_pull_sample(gst::ClockTime::from_mseconds(16)) { let cap = sample.caps().unwrap().structure(0).unwrap(); let w: i32 = cap.value("width")?.get()?; let h: i32 = cap.value("height")?.get()?; self.pix_fmt = cap.value("format")?.get()?; let w = w as usize; let h = h as usize; let buf = sample .buffer_owned() .ok_or_else(|| GStreamerError("Failed to get owned buffer.".into()))?; let mut crop = buf .meta::() .map(|m| m.rect()); // only crop if necessary if Some((0, 0, w as u32, h as u32)) == crop { crop = None; } let buf = buf .into_mapped_buffer_readable() .map_err(|_| GStreamerError("Failed to map buffer.".into()))?; let buf_size = buf.size(); // BGRx is 4 bytes per pixel if buf_size != (w * h * 4) { // for some reason the width and height of the caps do not guarantee correct buffer // size, so ignore those buffers, see: // https://gitlab.freedesktop.org/pipewire/pipewire/-/issues/985 trace!( "Size of mapped buffer: {} does NOT match size of capturable {}x{}@BGRx, \ dropping it!", buf_size, w, h ); } else { // Copy region specified by crop into self.buffer_cropped // TODO: Figure out if ffmpeg provides a zero copy alternative if let Some((x_off, y_off, w_crop, h_crop)) = crop { let x_off = x_off as usize; let y_off = y_off as usize; let w_crop = w_crop as usize; let h_crop = h_crop as usize; self.buffer_cropped.clear(); let data = buf.as_slice(); // BGRx is 4 bytes per pixel self.buffer_cropped.reserve(w_crop * h_crop * 4); for y in y_off..(y_off + h_crop) { let i = 4 * (w * y + x_off); self.buffer_cropped.extend(&data[i..i + 4 * w_crop]); } self.width = w_crop; self.height = h_crop; } else { self.width = w; self.height = h; } self.is_cropped = crop.is_some(); self.buffer = Some(buf); } } else { trace!("No new buffer available, falling back to previous one."); } if self.buffer.is_none() { return Err(Box::new(GStreamerError("No buffer available!".into()))); } let buf = if self.is_cropped { self.buffer_cropped.as_slice() } else { self.buffer.as_ref().unwrap().as_slice() }; match self.pix_fmt.as_str() { "BGRx" => Ok(PixelProvider::BGR0(self.width, self.height, buf)), "RGBx" => Ok(PixelProvider::RGB0(self.width, self.height, buf)), _ => unreachable!(), } } } impl Drop for PipeWireRecorder { fn drop(&mut self) { if let Err(err) = self.pipeline.set_state(gst::State::Null) { warn!("Failed to stop GStreamer pipeline: {}.", err); } } } fn handle_response( portal: Proxy<&SyncConnection>, path: dbus::Path<'static>, context: Arc>, mut f: F, ) -> Result where F: FnMut( OrgFreedesktopPortalRequestResponse, Proxy<&SyncConnection>, &Message, Arc>, ) -> Result<(), Box> + Send + Sync + 'static, { let mut m = MatchRule::new(); m.path = Some(path); m.msg_type = Some(MessageType::Signal); m.sender = Some("org.freedesktop.portal.Desktop".into()); m.interface = Some("org.freedesktop.portal.Request".into()); portal .connection .add_match(m, move |r: OrgFreedesktopPortalRequestResponse, c, m| { let portal = get_portal(c); debug!("Response from DBus: response: {:?}, message: {:?}", r, m); match r.response { 0 => {} 1 => { context.lock().unwrap().failure = true; warn!("DBus response: User cancelled interaction."); return true; } c => { context.lock().unwrap().failure = true; warn!("DBus response: Unknown error, code: {}.", c); return true; } } if let Err(err) = f(r, portal, m, context.clone()) { context.lock().unwrap().failure = true; warn!("Error requesting screen capture via dbus: {}", err); } true }) } fn get_portal(conn: &SyncConnection) -> Proxy<'_, &SyncConnection> { conn.with_proxy( "org.freedesktop.portal.Desktop", "/org/freedesktop/portal/desktop", Duration::from_millis(1000), ) } fn streams_from_response(response: &OrgFreedesktopPortalRequestResponse) -> Vec { (move || { Some( response .results .get("streams")? .as_iter()? .next()? .as_iter()? .filter_map(|stream| { let mut itr = stream.as_iter()?; let path = itr.next()?.as_u64()?; let (keys, values): (Vec<(usize, &dyn RefArg)>, Vec<(usize, &dyn RefArg)>) = itr.next()? .as_iter()? .enumerate() .partition(|(i, _)| i % 2 == 0); let attributes = keys .iter() .filter_map(|(_, key)| Some(key.as_str()?.to_owned())) .zip( values .iter() .map(|(_, arg)| *arg) .collect::>(), ) .collect::>(); Some(PwStreamInfo { path, source_type: attributes .get("source_type") .map_or(Some(0), |v| v.as_u64())?, }) }) .collect::>(), ) })() .unwrap_or_default() } // mostly inspired by https://gitlab.gnome.org/snippets/19 and // https://gitlab.gnome.org/-/snippets/39 struct CallBackContext { capture_cursor: bool, session: dbus::Path<'static>, streams: Vec, fd: Option, restore_token: Option, has_remote_desktop: bool, failure: bool, } fn on_create_session_response( r: OrgFreedesktopPortalRequestResponse, portal: Proxy<&SyncConnection>, _msg: &Message, context: Arc>, ) -> Result<(), Box> { debug!("on_create_session_response"); let session: dbus::Path = r .results .get("session_handle") .ok_or_else(|| { DBusError(format!( "Failed to obtain session_handle from response: {:?}", r )) })? .as_str() .ok_or_else(|| DBusError("Failed to convert session_handle to string.".into()))? .to_string() .into(); context.lock().unwrap().session = session.clone(); if context.lock().unwrap().has_remote_desktop { select_devices(portal, context) } else { select_sources(portal, context) } } fn select_devices( portal: Proxy<&SyncConnection>, context: Arc>, ) -> Result<(), Box> { let mut args: PropMap = HashMap::new(); let t: usize = rand::random(); args.insert( "handle_token".to_string(), Variant(Box::new(format!("weylus{t}"))), ); // TODO //args.insert( // "restore_token".to_string(), // Variant(Box::new(format!("weylus{t}"))), //); // persist modes: // 0: Do not persist (default) // 1: Permissions persist as long as the application is running // 2: Permissions persist until explicitly revoked args.insert("persist_mode".to_string(), Variant(Box::new(2 as u32))); // device types // 1: KEYBOARD // 2: POINTER // 4: TOUCHSCREEN let device_types = portal.available_device_types()?; debug!("Available device types: {device_types}."); args.insert("types".to_string(), Variant(Box::new(device_types))); let path = portal.select_devices(context.lock().unwrap().session.clone(), args)?; handle_response(portal, path, context, |_, portal, _, context| { select_sources(portal, context) })?; Ok(()) } fn select_sources( portal: Proxy<&SyncConnection>, context: Arc>, ) -> Result<(), Box> { debug!("select_sources"); let mut args: PropMap = HashMap::new(); let t: usize = rand::random(); args.insert( "handle_token".to_string(), Variant(Box::new(format!("weylus{t}"))), ); // https://flatpak.github.io/xdg-desktop-portal/docs/doc-org.freedesktop.portal.ScreenCast.html#org-freedesktop-portal-screencast-selectsources // allow multiple sources args.insert("multiple".into(), Variant(Box::new(true))); // 1: MONITOR // 2: WINDOW // 4: VIRTUAL let source_types = portal.available_source_types()?; debug!("Available source types: {source_types}."); args.insert("types".into(), Variant(Box::new(source_types))); let capture_cursor = context.lock().unwrap().capture_cursor; // 1: Hidden. The cursor is not part of the screen cast stream. // 2: Embedded: The cursor is embedded as part of the stream buffers. // 4: Metadata: The cursor is not part of the screen cast stream, but sent as PipeWire stream metadata. let cursor_mode = if capture_cursor { 2u32 } else { 1u32 }; let is_plasma = std::env::var("DESKTOP_SESSION").map_or(false, |s| s.contains("plasma")); if is_plasma && capture_cursor { // Warn the user if capturing the cursor is tried on kde as this can crash // kwin_wayland and tear down the plasma desktop, see: // https://bugs.kde.org/show_bug.cgi?id=435042 warn!( "You are attempting to capture the cursor under KDE Plasma, this may crash your \ desktop, see https://bugs.kde.org/show_bug.cgi?id=435042 for details! \ You have been warned." ); } args.insert("cursor_mode".into(), Variant(Box::new(cursor_mode))); let path = portal.select_sources(context.lock().unwrap().session.clone(), args)?; handle_response(portal, path, context, on_select_sources_response)?; Ok(()) } fn on_select_sources_response( _r: OrgFreedesktopPortalRequestResponse, portal: Proxy<&SyncConnection>, _msg: &Message, context: Arc>, ) -> Result<(), Box> { debug!("on_select_sources_response"); let mut args: PropMap = HashMap::new(); let t: usize = rand::random(); args.insert( "handle_token".to_string(), Variant(Box::new(format!("weylus{t}"))), ); let path = if context.lock().unwrap().has_remote_desktop { OrgFreedesktopPortalRemoteDesktop::start( &portal, context.lock().unwrap().session.clone(), "", args, )? } else { OrgFreedesktopPortalScreenCast::start( &portal, context.lock().unwrap().session.clone(), "", args, )? }; handle_response(portal, path, context, on_start_response)?; Ok(()) } fn on_start_response( r: OrgFreedesktopPortalRequestResponse, portal: Proxy<&SyncConnection>, _msg: &Message, context: Arc>, ) -> Result<(), Box> { debug!("on_start_response"); let mut context = context.lock().unwrap(); context.streams.append(&mut streams_from_response(&r)); let session = context.session.clone(); context .fd .replace(portal.open_pipe_wire_remote(session.clone(), HashMap::new())?); if let Some(Some(t)) = r.results.get("restore_token").map(|t| t.as_str()) { context.restore_token = Some(t.to_string()); } dbg!(&context.restore_token); if context.has_remote_desktop { debug!("Remote Desktop Session started"); } else { debug!("Screen Cast Session started"); } Ok(()) } fn request_remote_desktop( capture_cursor: bool, ) -> Result<(SyncConnection, OwnedFd, Vec), Box> { let conn = SyncConnection::new_session()?; let portal = get_portal(&conn); // Disabled for KDE plasma due to https://bugs.kde.org/show_bug.cgi?id=484996 // List of supported DEs: https://wiki.archlinux.org/title/XDG_Desktop_Portal#List_of_backends_and_interfaces let has_remote_desktop = std::env::var("DESKTOP_SESSION").map_or(false, |s| s.contains("gnome")); let context = CallBackContext { capture_cursor, session: Default::default(), streams: Default::default(), fd: None, restore_token: None, has_remote_desktop, failure: false, }; let context = Arc::new(Mutex::new(context)); let mut args: PropMap = HashMap::new(); let t1: usize = rand::random(); let t2: usize = rand::random(); args.insert( "session_handle_token".to_string(), Variant(Box::new(format!("weylus{t1}"))), ); args.insert( "handle_token".to_string(), Variant(Box::new(format!("weylus{t2}"))), ); let path = if has_remote_desktop { OrgFreedesktopPortalRemoteDesktop::create_session(&portal, args)? } else { OrgFreedesktopPortalScreenCast::create_session(&portal, args)? }; handle_response(portal, path, context.clone(), on_create_session_response)?; // wait 3 minutes for user interaction for _ in 0..1800 { conn.process(Duration::from_millis(100))?; let context = context.lock().unwrap(); // Once we got a file descriptor we are done! if context.fd.is_some() { break; } if context.failure { break; } } let context = context.lock().unwrap(); if context.fd.is_some() && !context.streams.is_empty() { Ok((conn, context.fd.clone().unwrap(), context.streams.clone())) } else { Err(Box::new(DBusError( "Failed to obtain screen capture.".into(), ))) } } pub fn get_capturables(capture_cursor: bool) -> Result, Box> { let (conn, fd, streams) = request_remote_desktop(capture_cursor)?; let conn = Arc::new(conn); Ok(streams .into_iter() .map(|s| PipeWireCapturable::new(conn.clone(), fd.clone(), s)) .collect()) } ================================================ FILE: src/capturable/remote_desktop_dbus.rs ================================================ // This code was autogenerated with `dbus-codegen-rust -c blocking -m None`, see https://github.com/diwic/dbus-rs // XML source: // https://github.com/flatpak/xdg-desktop-portal/raw/refs/tags/1.18.4/data/org.freedesktop.portal.Request.xml // https://github.com/flatpak/xdg-desktop-portal/raw/refs/tags/1.18.4/data/org.freedesktop.portal.ScreenCast.xml // https://github.com/flatpak/xdg-desktop-portal/raw/refs/tags/1.18.4/data/org.freedesktop.portal.RemoteDesktop.xml use dbus; #[allow(unused_imports)] use dbus::arg; use dbus::blocking; pub trait OrgFreedesktopPortalRequest { fn close(&self) -> Result<(), dbus::Error>; } #[derive(Debug)] pub struct OrgFreedesktopPortalRequestResponse { pub response: u32, pub results: arg::PropMap, } impl arg::AppendAll for OrgFreedesktopPortalRequestResponse { fn append(&self, i: &mut arg::IterAppend) { arg::RefArg::append(&self.response, i); arg::RefArg::append(&self.results, i); } } impl arg::ReadAll for OrgFreedesktopPortalRequestResponse { fn read(i: &mut arg::Iter) -> Result { Ok(OrgFreedesktopPortalRequestResponse { response: i.read()?, results: i.read()?, }) } } impl dbus::message::SignalArgs for OrgFreedesktopPortalRequestResponse { const NAME: &'static str = "Response"; const INTERFACE: &'static str = "org.freedesktop.portal.Request"; } impl<'a, T: blocking::BlockingSender, C: ::std::ops::Deref> OrgFreedesktopPortalRequest for blocking::Proxy<'a, C> { fn close(&self) -> Result<(), dbus::Error> { self.method_call("org.freedesktop.portal.Request", "Close", ()) } } pub trait OrgFreedesktopPortalScreenCast { fn create_session(&self, options: arg::PropMap) -> Result, dbus::Error>; fn select_sources( &self, session_handle: dbus::Path, options: arg::PropMap, ) -> Result, dbus::Error>; fn start( &self, session_handle: dbus::Path, parent_window: &str, options: arg::PropMap, ) -> Result, dbus::Error>; fn open_pipe_wire_remote( &self, session_handle: dbus::Path, options: arg::PropMap, ) -> Result; fn available_source_types(&self) -> Result; fn available_cursor_modes(&self) -> Result; fn version(&self) -> Result; } impl<'a, T: blocking::BlockingSender, C: ::std::ops::Deref> OrgFreedesktopPortalScreenCast for blocking::Proxy<'a, C> { fn create_session(&self, options: arg::PropMap) -> Result, dbus::Error> { self.method_call( "org.freedesktop.portal.ScreenCast", "CreateSession", (options,), ) .and_then(|r: (dbus::Path<'static>,)| Ok(r.0)) } fn select_sources( &self, session_handle: dbus::Path, options: arg::PropMap, ) -> Result, dbus::Error> { self.method_call( "org.freedesktop.portal.ScreenCast", "SelectSources", (session_handle, options), ) .and_then(|r: (dbus::Path<'static>,)| Ok(r.0)) } fn start( &self, session_handle: dbus::Path, parent_window: &str, options: arg::PropMap, ) -> Result, dbus::Error> { self.method_call( "org.freedesktop.portal.ScreenCast", "Start", (session_handle, parent_window, options), ) .and_then(|r: (dbus::Path<'static>,)| Ok(r.0)) } fn open_pipe_wire_remote( &self, session_handle: dbus::Path, options: arg::PropMap, ) -> Result { self.method_call( "org.freedesktop.portal.ScreenCast", "OpenPipeWireRemote", (session_handle, options), ) .and_then(|r: (arg::OwnedFd,)| Ok(r.0)) } fn available_source_types(&self) -> Result { ::get( self, "org.freedesktop.portal.ScreenCast", "AvailableSourceTypes", ) } fn available_cursor_modes(&self) -> Result { ::get( self, "org.freedesktop.portal.ScreenCast", "AvailableCursorModes", ) } fn version(&self) -> Result { ::get( self, "org.freedesktop.portal.ScreenCast", "version", ) } } pub trait OrgFreedesktopPortalRemoteDesktop { fn create_session(&self, options: arg::PropMap) -> Result, dbus::Error>; fn select_devices( &self, session_handle: dbus::Path, options: arg::PropMap, ) -> Result, dbus::Error>; fn start( &self, session_handle: dbus::Path, parent_window: &str, options: arg::PropMap, ) -> Result, dbus::Error>; fn notify_pointer_motion( &self, session_handle: dbus::Path, options: arg::PropMap, dx: f64, dy: f64, ) -> Result<(), dbus::Error>; fn notify_pointer_motion_absolute( &self, session_handle: dbus::Path, options: arg::PropMap, stream: u32, x_: f64, y_: f64, ) -> Result<(), dbus::Error>; fn notify_pointer_button( &self, session_handle: dbus::Path, options: arg::PropMap, button: i32, state: u32, ) -> Result<(), dbus::Error>; fn notify_pointer_axis( &self, session_handle: dbus::Path, options: arg::PropMap, dx: f64, dy: f64, ) -> Result<(), dbus::Error>; fn notify_pointer_axis_discrete( &self, session_handle: dbus::Path, options: arg::PropMap, axis: u32, steps: i32, ) -> Result<(), dbus::Error>; fn notify_keyboard_keycode( &self, session_handle: dbus::Path, options: arg::PropMap, keycode: i32, state: u32, ) -> Result<(), dbus::Error>; fn notify_keyboard_keysym( &self, session_handle: dbus::Path, options: arg::PropMap, keysym: i32, state: u32, ) -> Result<(), dbus::Error>; fn notify_touch_down( &self, session_handle: dbus::Path, options: arg::PropMap, stream: u32, slot: u32, x_: f64, y_: f64, ) -> Result<(), dbus::Error>; fn notify_touch_motion( &self, session_handle: dbus::Path, options: arg::PropMap, stream: u32, slot: u32, x_: f64, y_: f64, ) -> Result<(), dbus::Error>; fn notify_touch_up( &self, session_handle: dbus::Path, options: arg::PropMap, slot: u32, ) -> Result<(), dbus::Error>; fn connect_to_eis( &self, session_handle: dbus::Path, options: arg::PropMap, ) -> Result; fn available_device_types(&self) -> Result; fn version(&self) -> Result; } impl<'a, T: blocking::BlockingSender, C: ::std::ops::Deref> OrgFreedesktopPortalRemoteDesktop for blocking::Proxy<'a, C> { fn create_session(&self, options: arg::PropMap) -> Result, dbus::Error> { self.method_call( "org.freedesktop.portal.RemoteDesktop", "CreateSession", (options,), ) .and_then(|r: (dbus::Path<'static>,)| Ok(r.0)) } fn select_devices( &self, session_handle: dbus::Path, options: arg::PropMap, ) -> Result, dbus::Error> { self.method_call( "org.freedesktop.portal.RemoteDesktop", "SelectDevices", (session_handle, options), ) .and_then(|r: (dbus::Path<'static>,)| Ok(r.0)) } fn start( &self, session_handle: dbus::Path, parent_window: &str, options: arg::PropMap, ) -> Result, dbus::Error> { self.method_call( "org.freedesktop.portal.RemoteDesktop", "Start", (session_handle, parent_window, options), ) .and_then(|r: (dbus::Path<'static>,)| Ok(r.0)) } fn notify_pointer_motion( &self, session_handle: dbus::Path, options: arg::PropMap, dx: f64, dy: f64, ) -> Result<(), dbus::Error> { self.method_call( "org.freedesktop.portal.RemoteDesktop", "NotifyPointerMotion", (session_handle, options, dx, dy), ) } fn notify_pointer_motion_absolute( &self, session_handle: dbus::Path, options: arg::PropMap, stream: u32, x_: f64, y_: f64, ) -> Result<(), dbus::Error> { self.method_call( "org.freedesktop.portal.RemoteDesktop", "NotifyPointerMotionAbsolute", (session_handle, options, stream, x_, y_), ) } fn notify_pointer_button( &self, session_handle: dbus::Path, options: arg::PropMap, button: i32, state: u32, ) -> Result<(), dbus::Error> { self.method_call( "org.freedesktop.portal.RemoteDesktop", "NotifyPointerButton", (session_handle, options, button, state), ) } fn notify_pointer_axis( &self, session_handle: dbus::Path, options: arg::PropMap, dx: f64, dy: f64, ) -> Result<(), dbus::Error> { self.method_call( "org.freedesktop.portal.RemoteDesktop", "NotifyPointerAxis", (session_handle, options, dx, dy), ) } fn notify_pointer_axis_discrete( &self, session_handle: dbus::Path, options: arg::PropMap, axis: u32, steps: i32, ) -> Result<(), dbus::Error> { self.method_call( "org.freedesktop.portal.RemoteDesktop", "NotifyPointerAxisDiscrete", (session_handle, options, axis, steps), ) } fn notify_keyboard_keycode( &self, session_handle: dbus::Path, options: arg::PropMap, keycode: i32, state: u32, ) -> Result<(), dbus::Error> { self.method_call( "org.freedesktop.portal.RemoteDesktop", "NotifyKeyboardKeycode", (session_handle, options, keycode, state), ) } fn notify_keyboard_keysym( &self, session_handle: dbus::Path, options: arg::PropMap, keysym: i32, state: u32, ) -> Result<(), dbus::Error> { self.method_call( "org.freedesktop.portal.RemoteDesktop", "NotifyKeyboardKeysym", (session_handle, options, keysym, state), ) } fn notify_touch_down( &self, session_handle: dbus::Path, options: arg::PropMap, stream: u32, slot: u32, x_: f64, y_: f64, ) -> Result<(), dbus::Error> { self.method_call( "org.freedesktop.portal.RemoteDesktop", "NotifyTouchDown", (session_handle, options, stream, slot, x_, y_), ) } fn notify_touch_motion( &self, session_handle: dbus::Path, options: arg::PropMap, stream: u32, slot: u32, x_: f64, y_: f64, ) -> Result<(), dbus::Error> { self.method_call( "org.freedesktop.portal.RemoteDesktop", "NotifyTouchMotion", (session_handle, options, stream, slot, x_, y_), ) } fn notify_touch_up( &self, session_handle: dbus::Path, options: arg::PropMap, slot: u32, ) -> Result<(), dbus::Error> { self.method_call( "org.freedesktop.portal.RemoteDesktop", "NotifyTouchUp", (session_handle, options, slot), ) } fn connect_to_eis( &self, session_handle: dbus::Path, options: arg::PropMap, ) -> Result { self.method_call( "org.freedesktop.portal.RemoteDesktop", "ConnectToEIS", (session_handle, options), ) .and_then(|r: (arg::OwnedFd,)| Ok(r.0)) } fn available_device_types(&self) -> Result { ::get( self, "org.freedesktop.portal.RemoteDesktop", "AvailableDeviceTypes", ) } fn version(&self) -> Result { ::get( self, "org.freedesktop.portal.RemoteDesktop", "version", ) } } ================================================ FILE: src/capturable/testsrc.rs ================================================ use crate::capturable::{Capturable, Geometry, Recorder}; use crate::video::PixelProvider; use std::error::Error; #[derive(Debug, Clone, Copy)] pub enum PixelFormat { BGR0, RGB0, RGB, } #[derive(Debug, Clone, Copy)] pub struct TestCapturable { pub width: usize, pub height: usize, pub pixel_format: PixelFormat, } impl TestCapturable { fn pixel_size(&self) -> usize { match self.pixel_format { PixelFormat::BGR0 => 4, PixelFormat::RGB0 => 4, PixelFormat::RGB => 3, } } fn set_default_pixel(&self, buf: &mut [u8], x: usize, y: usize) { let w = self.width; let i = x * 8 / w; let pos = (x + y * w) * self.pixel_size(); let (pos_r, pos_g, pos_b) = match self.pixel_format { PixelFormat::BGR0 => (pos + 2, pos + 1, pos), PixelFormat::RGB0 | PixelFormat::RGB => (pos, pos + 1, pos + 2), }; buf[pos_b] = if i & 1 != 0 { 255 } else { 0 }; buf[pos_g] = if i & 2 != 0 { 255 } else { 0 }; buf[pos_r] = if i & 4 != 0 { 255 } else { 0 }; } } pub struct TestRecorder { capturable: TestCapturable, buf: Vec, i: usize, } impl TestRecorder { fn new(capturable: TestCapturable) -> Self { let mut buf = vec![0; capturable.width * capturable.height * capturable.pixel_size()]; let buf_ref = buf.as_mut(); for y in 0..capturable.height { for x in 0..capturable.width { capturable.set_default_pixel(buf_ref, x, y); } } Self { capturable, buf, i: 0, } } } impl Capturable for TestCapturable { fn name(&self) -> String { format!( "Test Source {}x{}@{:?}", self.width, self.height, self.pixel_format ) } fn geometry(&self) -> Result> { Ok(Geometry::Relative(0.0, 0.0, 1.0, 1.0)) } fn before_input(&mut self) -> Result<(), Box> { Ok(()) } fn recorder(&self, _: bool) -> Result, Box> { Ok(Box::new(TestRecorder::new(*self))) } } impl Recorder for TestRecorder { fn capture(&mut self) -> Result, Box> { const N: usize = 120; let dh = self.capturable.height / N; let buf_ref = self.buf.as_mut(); let w = self.capturable.width; for y in self.i * dh..(self.i + 1) * dh { for x in 0..w { self.capturable.set_default_pixel(buf_ref, x, y); } } self.i = (self.i + 1) % N; for y in self.i * dh..(self.i + 1) * dh { for x in 0..w { let pos = (x + y * w) * self.capturable.pixel_size(); let (pos_r, pos_g, pos_b) = match self.capturable.pixel_format { PixelFormat::BGR0 => (pos + 2, pos + 1, pos), PixelFormat::RGB0 | PixelFormat::RGB => (pos, pos + 1, pos + 2), }; buf_ref[pos_b] = ((self.i + N * x / w) % N * 256 / N) as u8; buf_ref[pos_g] = ((self.i + N * x / w + N / 3) % N * 256 / N) as u8; buf_ref[pos_r] = ((self.i + N * x / w + 2 * N / 3) % N * 256 / N) as u8; } } Ok(match self.capturable.pixel_format { PixelFormat::BGR0 => PixelProvider::BGR0( self.capturable.width, self.capturable.height, self.buf.as_slice(), ), PixelFormat::RGB0 => PixelProvider::RGB0( self.capturable.width, self.capturable.height, self.buf.as_slice(), ), PixelFormat::RGB => PixelProvider::RGB( self.capturable.width, self.capturable.height, self.buf.as_slice(), ), }) } } ================================================ FILE: src/capturable/win_ctx.rs ================================================ use std::mem::zeroed; use std::{mem, ptr}; use winapi::shared::dxgi::{ CreateDXGIFactory1, IDXGIAdapter1, IDXGIFactory1, IDXGIOutput, IID_IDXGIFactory1, DXGI_OUTPUT_DESC, }; use winapi::shared::windef::*; use winapi::shared::winerror::*; use winapi::um::winuser::*; use wio::com::ComPtr; // from https://github.com/bryal/dxgcap-rs/blob/009b746d1c19c4c10921dd469eaee483db6aa002/src/lib.r fn hr_failed(hr: HRESULT) -> bool { hr < 0 } fn create_dxgi_factory_1() -> ComPtr { unsafe { let mut factory = ptr::null_mut(); let hr = CreateDXGIFactory1(&IID_IDXGIFactory1, &mut factory); if hr_failed(hr) { panic!("Failed to create DXGIFactory1, {:x}", hr) } else { ComPtr::from_raw(factory as *mut IDXGIFactory1) } } } fn get_adapter_outputs(adapter: &IDXGIAdapter1) -> Vec> { let mut outputs = Vec::new(); for i in 0.. { unsafe { let mut output = ptr::null_mut(); if hr_failed(adapter.EnumOutputs(i, &mut output)) { break; } else { let mut out_desc = zeroed(); (*output).GetDesc(&mut out_desc); if out_desc.AttachedToDesktop != 0 { outputs.push(ComPtr::from_raw(output)) } else { break; } } } } outputs } #[derive(Clone)] pub struct WinCtx { outputs: Vec, union_rect: RECT, } impl WinCtx { pub fn new() -> WinCtx { let mut desktops: Vec = Vec::new(); let mut union: RECT; unsafe { union = mem::zeroed(); let factory = create_dxgi_factory_1(); let mut adapter = ptr::null_mut(); if factory.EnumAdapters1(0, &mut adapter) != DXGI_ERROR_NOT_FOUND { let adp = ComPtr::from_raw(adapter); let outputs = get_adapter_outputs(&adp); for o in outputs { let mut desc: DXGI_OUTPUT_DESC = mem::zeroed(); o.GetDesc(ptr::addr_of_mut!(desc)); desktops.push(desc); UnionRect( ptr::addr_of_mut!(union), ptr::addr_of!(union), ptr::addr_of!(desc.DesktopCoordinates), ); } } } WinCtx { outputs: desktops, union_rect: union, } } pub fn get_outputs(&self) -> &Vec { &self.outputs } pub fn get_union_rect(&self) -> &RECT { &self.union_rect } } ================================================ FILE: src/capturable/x11.rs ================================================ use crate::capturable::{Capturable, Geometry, Recorder}; use crate::cerror::CError; use crate::video::PixelProvider; use std::ffi::{CStr, CString}; use std::os::raw::{c_char, c_float, c_int, c_uint, c_void}; use std::slice::from_raw_parts; use std::sync::Arc; use std::{error::Error, fmt}; use tracing::debug; extern "C" { fn XOpenDisplay(name: *const c_char) -> *mut c_void; fn XCloseDisplay(disp: *mut c_void) -> c_int; fn XInitThreads() -> c_int; fn XLockDisplay(disp: *mut c_void); fn XUnlockDisplay(disp: *mut c_void); fn x11_set_error_handler(); fn create_capturables( disp: *mut c_void, handles: *mut *mut c_void, num_monitors: *mut c_int, size: c_int, err: *mut CError, ) -> c_int; fn clone_capturable(handle: *const c_void) -> *mut c_void; fn destroy_capturable(handle: *mut c_void); fn get_capturable_name(handle: *const c_void) -> *const c_char; fn capturable_before_input(handle: *mut c_void, err: *mut CError); fn get_geometry_relative( handle: *const c_void, x: *mut c_float, y: *mut c_float, width: *mut c_float, height: *mut c_float, err: *mut CError, ); fn map_input_device_to_entire_screen( disp: *mut c_void, device_name: *const c_char, libinput: c_int, err: *mut CError, ); fn start_capture(handle: *const c_void, ctx: *mut c_void, err: *mut CError) -> *mut c_void; fn capture_screen( handle: *mut c_void, img: *mut CImage, capture_cursor: c_int, err: *mut CError, ); fn stop_capture(handle: *mut c_void, err: *mut CError); } pub fn x11_init() { unsafe { XInitThreads(); x11_set_error_handler(); } } pub struct X11Capturable { handle: *mut c_void, // keep a reference to the display so it is not closed while a capturable still exists disp: Arc, } impl Clone for X11Capturable { fn clone(&self) -> Self { let handle = unsafe { clone_capturable(self.handle) }; Self { handle, disp: self.disp.clone(), } } } unsafe impl Send for X11Capturable {} impl X11Capturable { pub unsafe fn handle(&mut self) -> *mut c_void { self.handle } } impl Capturable for X11Capturable { fn name(&self) -> String { unsafe { CStr::from_ptr(get_capturable_name(self.handle)) .to_string_lossy() .into() } } fn geometry(&self) -> Result> { let mut x: c_float = 0.0; let mut y: c_float = 0.0; let mut width: c_float = 0.0; let mut height: c_float = 0.0; let mut err = CError::new(); self.disp.lock(); unsafe { get_geometry_relative( self.handle, &mut x, &mut y, &mut width, &mut height, &mut err, ); } self.disp.unlock(); if err.is_err() { return Err(Box::new(err)); } Ok(Geometry::Relative( x.into(), y.into(), width.into(), height.into(), )) } fn before_input(&mut self) -> Result<(), Box> { let mut err = CError::new(); self.disp.lock(); unsafe { capturable_before_input(self.handle, &mut err) }; self.disp.unlock(); if err.is_err() { Err(Box::new(err)) } else { Ok(()) } } fn recorder(&self, capture_cursor: bool) -> Result, Box> { match RecorderX11::new(self.clone(), capture_cursor) { Ok(recorder) => Ok(Box::new(recorder)), Err(err) => Err(Box::new(err)), } } } impl fmt::Display for X11Capturable { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.name()) } } impl Drop for X11Capturable { fn drop(&mut self) { unsafe { destroy_capturable(self.handle); } } } struct XDisplay { handle: *mut c_void, } impl XDisplay { pub fn new() -> Option { let handle = unsafe { XOpenDisplay(std::ptr::null()) }; if handle.is_null() { return None; } Some(Self { handle }) } pub fn lock(&self) { unsafe { XLockDisplay(self.handle) } } pub fn unlock(&self) { unsafe { XUnlockDisplay(self.handle) } } } impl Drop for XDisplay { fn drop(&mut self) { self.lock(); unsafe { XCloseDisplay(self.handle) }; self.unlock(); } } pub struct X11Context { disp: Arc, } impl X11Context { pub fn new() -> Option { let disp = XDisplay::new()?; Some(Self { disp: Arc::new(disp), }) } pub fn capturables(&mut self) -> Result, CError> { let mut err = CError::new(); let mut handles = [std::ptr::null_mut::(); 128]; let mut num_monitors: c_int = 0; self.disp.lock(); let size = unsafe { create_capturables( self.disp.handle, handles.as_mut_ptr(), &mut num_monitors, handles.len() as c_int, &mut err, ) }; self.disp.unlock(); if err.is_err() { if err.code() == 2 { debug!("{}", err); } else { return Err(err); } } let mut capturables: Vec = handles[0..size as usize] .iter() .map(|handle| X11Capturable { handle: *handle, disp: self.disp.clone(), }) .collect(); // The first capturable is always the whole desktop, after that there is num_monitors // monitors and finally windows. let win = &mut capturables[(num_monitors as usize + 1)..(size as usize)]; win.sort_by(|a, b| a.name().to_lowercase().cmp(&b.name().to_lowercase())); Ok(capturables) } pub fn map_input_device_to_entire_screen(&mut self, device_name: &str, pen: bool) -> CError { let mut err = CError::new(); let device_name_c_str = CString::new(device_name).unwrap(); self.disp.lock(); unsafe { map_input_device_to_entire_screen( self.disp.handle, device_name_c_str.as_ptr(), pen.into(), &mut err, ) }; self.disp.unlock(); if err.is_err() { debug!("Failed to map input device to screen: {}", &err); } err } } #[repr(C)] struct CImage { data: *const u8, width: c_uint, height: c_uint, } impl CImage { pub fn new() -> Self { Self { data: std::ptr::null(), width: 0, height: 0, } } pub fn size(&self) -> usize { (self.width * self.height * 4) as usize } pub fn data(&self) -> &[u8] { unsafe { from_raw_parts(self.data, self.size()) } } } pub struct RecorderX11 { handle: *mut c_void, // keep a reference to the capturable so it is not destroyed until we are done #[allow(dead_code)] capturable: X11Capturable, img: CImage, capture_cursor: bool, } impl RecorderX11 { pub fn new(mut capturable: X11Capturable, capture_cursor: bool) -> Result { let mut err = CError::new(); capturable.disp.lock(); let handle = unsafe { start_capture(capturable.handle(), std::ptr::null_mut(), &mut err) }; capturable.disp.unlock(); if err.is_err() { Err(err) } else { Ok(Self { handle, capturable, img: CImage::new(), capture_cursor, }) } } } impl Drop for RecorderX11 { fn drop(&mut self) { let mut err = CError::new(); self.capturable.disp.lock(); unsafe { stop_capture(self.handle, &mut err); } self.capturable.disp.unlock(); } } impl Recorder for RecorderX11 { fn capture(&mut self) -> Result, Box> { let mut err = CError::new(); self.capturable.disp.lock(); unsafe { capture_screen( self.handle, &mut self.img, self.capture_cursor.into(), &mut err, ); } self.capturable.disp.unlock(); if err.is_err() { self.img.data = std::ptr::null(); Err(err.into()) } else { Ok(PixelProvider::BGR0( self.img.width as usize, self.img.height as usize, self.img.data(), )) } } } ================================================ FILE: src/cerror.rs ================================================ use std::error::Error; use std::ffi::CStr; use std::fmt; use std::os::raw::{c_char, c_int}; #[repr(C)] pub struct CError { code: c_int, error_str: [c_char; 1024], } pub enum CErrorCode { NoError, GenericError, UInputNotAccessible, } impl CError { pub fn new() -> Self { Self { code: 0, error_str: [0; 1024], } } pub fn is_err(&self) -> bool { self.code != 0 } pub fn code(&self) -> i32 { self.code as i32 } pub fn to_enum(&self) -> CErrorCode { match self.code { 0 => CErrorCode::NoError, 101 => CErrorCode::UInputNotAccessible, _ => CErrorCode::GenericError, } } } impl fmt::Display for CError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "CError: code: {} message: {}", self.code, unsafe { CStr::from_ptr(self.error_str.as_ptr()).to_string_lossy() }) } } impl fmt::Debug for CError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(self, f) } } impl Error for CError {} ================================================ FILE: src/config.rs ================================================ use std::net::IpAddr; use std::{fs, path::PathBuf}; use clap::Parser; use serde::{Deserialize, Serialize}; use tracing::{debug, warn}; #[derive(clap::ValueEnum, Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] pub enum ThemeType { Aero, AquaClassic, Blue, Classic, Dark, Greybird, HighContrast, Metro, } const THEME_LIST: [ThemeType; 8] = [ ThemeType::Aero, ThemeType::AquaClassic, ThemeType::Blue, ThemeType::Classic, ThemeType::Dark, ThemeType::Greybird, ThemeType::HighContrast, ThemeType::Metro, ]; impl Default for ThemeType { fn default() -> Self { Self::Greybird } } impl ThemeType { pub fn apply(&self) { let theme = match self { ThemeType::Classic => fltk_theme::ThemeType::Classic, ThemeType::Aero => fltk_theme::ThemeType::Aero, ThemeType::Metro => fltk_theme::ThemeType::Metro, ThemeType::AquaClassic => fltk_theme::ThemeType::AquaClassic, ThemeType::Greybird => fltk_theme::ThemeType::Greybird, ThemeType::Blue => fltk_theme::ThemeType::Blue, ThemeType::Dark => fltk_theme::ThemeType::Dark, ThemeType::HighContrast => fltk_theme::ThemeType::HighContrast, }; let theme = fltk_theme::WidgetTheme::new(theme); theme.apply(); } pub fn name(&self) -> String { format!("{self:?}") } pub fn to_index(&self) -> i32 { THEME_LIST.iter().position(|th| th == self).unwrap() as i32 } pub fn from_index(i: i32) -> Self { let i = i.clamp(0, THEME_LIST.len() as i32 - 1) as usize; THEME_LIST[i] } pub fn themes() -> &'static [ThemeType] { &THEME_LIST } } #[derive(Serialize, Deserialize, Parser, Debug, Clone)] #[command(version, about, long_about = None)] pub struct Config { #[arg(long, help = "Access code")] pub access_code: Option, #[arg(long, default_value = "0.0.0.0", help = "Bind address")] pub bind_address: IpAddr, #[arg(long, default_value = "1701", help = "Web port")] pub web_port: u16, #[cfg(target_os = "linux")] #[arg( long, help = "Try to use hardware acceleration through the Video Acceleration API." )] pub try_vaapi: bool, #[cfg(any(target_os = "linux", target_os = "windows"))] #[arg(long, help = "Try to use Nvidia's NVENC to encode the video via GPU.")] #[serde(default)] pub try_nvenc: bool, #[cfg(target_os = "macos")] #[arg( long, help = "Try to use hardware acceleration through the VideoToolbox API." )] #[serde(default)] pub try_videotoolbox: bool, #[cfg(target_os = "windows")] #[arg( long, help = "Try to use hardware acceleration through the MediaFoundation API." )] #[serde(default)] pub try_mediafoundation: bool, #[arg(long, help = "Start Weylus server immediately on program start.")] #[serde(default)] pub auto_start: bool, #[arg(long, help = "Gui Theme")] pub gui_theme: Option, #[arg(long, help = "Run Weylus without gui and start immediately.")] #[serde(default)] pub no_gui: bool, #[cfg(target_os = "linux")] #[arg(long, help = "Wayland/PipeWire Support.")] #[serde(default)] pub wayland_support: bool, #[arg(long, help = "Print template of index.html served by Weylus.")] #[serde(skip)] pub print_index_html: bool, #[arg(long, help = "Print access.html served by Weylus.")] #[serde(skip)] pub print_access_html: bool, #[arg(long, help = "Print style.css served by Weylus.")] #[serde(skip)] pub print_style_css: bool, #[arg(long, help = "Print lib.js served by Weylus.")] #[serde(skip)] pub print_lib_js: bool, #[arg( long, help = "Use custom template of index.html to be served by Weylus." )] #[serde(skip)] pub custom_index_html: Option, #[arg(long, help = "Use custom access.html to be served by Weylus.")] #[serde(skip)] pub custom_access_html: Option, #[arg(long, help = "Use custom style.css to be served by Weylus.")] #[serde(skip)] pub custom_style_css: Option, #[arg(long, help = "Use custom lib.js to be served by Weylus.")] #[serde(skip)] pub custom_lib_js: Option, #[arg(long, help = "Print shell completions for given shell.")] #[serde(skip)] pub completions: Option, } pub fn read_config() -> Option { if let Some(mut config_path) = dirs::config_dir() { config_path.push("weylus"); config_path.push("weylus.toml"); match fs::read_to_string(&config_path) { Ok(s) => match toml::from_str(&s) { Ok(c) => Some(c), Err(e) => { warn!("Failed to read configuration file: {}", e); None } }, Err(err) => { match err.kind() { std::io::ErrorKind::NotFound => { debug!("Failed to read configuration file: {}", err) } _ => warn!("Failed to read configuration file: {}", err), } None } } } else { None } } pub fn write_config(conf: &Config) { match dirs::config_dir() { Some(mut config_path) => { config_path.push("weylus"); if !config_path.exists() { if let Err(err) = fs::create_dir_all(&config_path) { warn!("Failed create directory for configuration: {}", err); return; } } config_path.push("weylus.toml"); if let Err(err) = fs::write( config_path, &toml::to_string_pretty(&conf).expect("Failed to encode config to toml."), ) { warn!("Failed to write configuration file: {}", err); } } None => { warn!("Failed to find configuration directory!"); } } } pub fn get_config() -> Config { let args = std::env::args(); if let Some(mut config) = read_config() { if args.len() > 1 { config.update_from(args); } config } else { Config::parse() } } ================================================ FILE: src/gui.rs ================================================ use std::cmp::min; use std::io::Cursor; use std::iter::Iterator; use std::net::{IpAddr, SocketAddr}; use std::sync::atomic::AtomicBool; use fltk::app; use fltk::enums::{FrameType, LabelType}; use fltk::image::PngImage; use fltk::menu::Choice; use std::sync::{mpsc, Arc, Mutex}; use tracing::{error, info, warn}; use fltk::{ app::{awake_callback, App}, button::{Button, CheckButton}, frame::Frame, input::{Input, IntInput}, output::Output, prelude::*, text::{TextBuffer, TextDisplay}, window::Window, }; #[cfg(not(target_os = "windows"))] use pnet_datalink as datalink; use crate::config::{write_config, Config, ThemeType}; use crate::protocol::{CustomInputAreas, Rect}; use crate::web::Web2UiMessage::UInputInaccessible; pub fn run(config: &Config, log_receiver: mpsc::Receiver) { let width = 200; let height = 30; let padding = 10; let app = App::default().with_scheme(fltk::app::AppScheme::Gtk); config.gui_theme.map(|th| th.apply()); let mut wind = Window::default() .with_size(660, 600) .center_screen() .with_label(&format!("Weylus - {}", env!("CARGO_PKG_VERSION"))); wind.set_xclass("weylus"); wind.set_callback(move |_win| app.quit()); let mut input_access_code = Input::default() .with_pos(130, 30) .with_size(width, height) .with_label("Access code"); input_access_code.set_tooltip( "Restrict who can control your computer with an access code. Note that this does NOT do \ any kind of encryption and it is advised to only run Weylus inside trusted networks! Do \ NOT reuse any of your passwords! If left blank, no code is required to access Weylus \ remotely.", ); if let Some(code) = config.access_code.as_ref() { input_access_code.set_value(code); } let mut input_bind_addr = Input::default() .with_size(width, height) .below_of(&input_access_code, padding) .with_label("Bind Address"); input_bind_addr.set_value(&config.bind_address.to_string()); let mut input_port = IntInput::default() .with_size(width, height) .below_of(&input_bind_addr, padding) .with_label("Port"); input_port.set_value(&config.web_port.to_string()); let mut check_auto_start = CheckButton::default() .with_size(70, height) .below_of(&input_port, padding + 5) .with_label("Auto Start"); check_auto_start.set_tooltip("Start Weylus server immediately on program start."); check_auto_start.set_checked(config.auto_start); #[cfg(target_os = "linux")] let mut check_wayland = CheckButton::default() .with_size(70, height) .right_of(&check_auto_start, 3 * padding) .with_label("Wayland/\nPipeWire\nSupport"); #[cfg(target_os = "linux")] { check_wayland.set_tooltip( "EXPERIMENTAL! This may crash your desktop! Enables screen \ capturing for Wayland using PipeWire and GStreamer.", ); check_wayland.set_checked(config.wayland_support); } let mut label_hw_accel = Frame::default() .with_size(width, height) .below_of(&check_auto_start, padding) .with_label("Try Hardware acceleration"); label_hw_accel.set_tooltip( "On many systems video encoding can be done with hardware \ acceleration. By default this is disabled as the quality and stability of video encoding \ varies greatly among hardware and drivers. Currently this is only supported on Linux.", ); let mut check_native_hw_accel = CheckButton::default() .with_size(70, height) .below_of(&label_hw_accel, 0); #[cfg(target_os = "linux")] { check_native_hw_accel.set_label("VAAPI"); check_native_hw_accel .set_tooltip("Try to use hardware acceleration through the Video Acceleration API."); check_native_hw_accel.set_checked(config.try_vaapi); } #[cfg(target_os = "macos")] { check_native_hw_accel.set_label("VideoToolbox"); check_native_hw_accel .set_tooltip("Try to use hardware acceleration through the VideoToolbox API."); check_native_hw_accel.set_checked(config.try_videotoolbox); } #[cfg(target_os = "windows")] { check_native_hw_accel.set_label("Media-\nFoundation"); check_native_hw_accel .set_tooltip("Try to use hardware acceleration through the MediaFoundation API."); check_native_hw_accel.set_checked(config.try_mediafoundation); } let mut check_nvenc = CheckButton::default() .with_size(70, height) .right_of(&check_native_hw_accel, 2 * padding) .with_label("NVENC"); check_nvenc.set_tooltip("Try to use Nvidia's NVENC to encode the video via GPU."); #[cfg(any(target_os = "linux", target_os = "windows"))] check_nvenc.set_checked(config.try_nvenc); #[cfg(not(any(target_os = "linux", target_os = "windows")))] { check_nvenc.deactivate(); check_nvenc.hide(); } let mut but_toggle = Button::default() .with_size(width, height) .below_of(&check_native_hw_accel, 2 * padding) .with_label("Start"); let mut output_server_addr = Output::default() .with_size(500, height) .below_of(&but_toggle, 3 * padding) .with_label("Connect your\ntablet to:"); output_server_addr.hide(); let output_buf = TextBuffer::default(); let mut output = TextDisplay::default().with_size(600, 6 * height).with_pos( 30, output_server_addr.y() + output_server_addr.height() + padding, ); output.set_buffer(output_buf); let output_buf = output.buffer().unwrap(); let mut choice_theme = Choice::default() .with_size(width, height) .right_of(&input_access_code, padding); for theme in ThemeType::themes() { choice_theme.add_choice(&theme.name()); } choice_theme.set_value(config.gui_theme.unwrap_or(ThemeType::default()).to_index()); let mut qr_frame = Frame::default() .with_size(235, 235) .right_of(&input_bind_addr, padding); qr_frame.hide(); wind.make_resizable(true); wind.end(); wind.show(); let output_buf = Arc::new(Mutex::new(output_buf)); std::thread::spawn(move || { while let Ok(log_message) = log_receiver.recv() { let mut output_buf = output_buf.lock().unwrap(); output_buf.append(&log_message); } }); let mut weylus = crate::weylus::Weylus::new(); let mut is_server_running = false; let auto_start = config.auto_start; let config = Arc::new(Mutex::new(config.clone())); { let config = config.clone(); choice_theme.set_callback(move |c| { let v = c.value(); if v >= 0 { ThemeType::from_index(v).apply(); config.lock().unwrap().gui_theme = Some(ThemeType::from_index(v)); write_config(&config.lock().unwrap()); } }); } let mut toggle_server = move |but: &mut Button| { if let Err(err) = || -> Result<(), Box> { let mut config = config.lock().unwrap(); if !is_server_running { { let access_code_string = input_access_code.value(); let access_code = match access_code_string.as_str() { "" => None, code => Some(code), }; let bind_addr: IpAddr = input_bind_addr.value().parse()?; let web_port: u16 = input_port.value().parse()?; config.access_code = access_code.map(|s| s.to_string()); config.web_port = web_port; config.bind_address = bind_addr; config.auto_start = check_auto_start.is_checked(); config.gui_theme = Some(ThemeType::from_index(choice_theme.value())); #[cfg(target_os = "linux")] { config.try_vaapi = check_native_hw_accel.is_checked(); config.wayland_support = check_wayland.is_checked(); } #[cfg(any(target_os = "linux", target_os = "windows"))] { config.try_nvenc = check_nvenc.is_checked(); } #[cfg(target_os = "macos")] { config.try_videotoolbox = check_native_hw_accel.is_checked(); } #[cfg(target_os = "windows")] { config.try_mediafoundation = check_native_hw_accel.is_checked(); } } if !weylus.start(&config, |message| match message { UInputInaccessible => awake_callback(move || { let w = 500; let h = 300; let mut pop_up = Window::default() .with_size(w, h) .center_screen() .with_label("Weylus - UInput inaccessible!"); pop_up.set_xclass("weylus"); let buf = TextBuffer::default(); let mut pop_up_text = TextDisplay::default().with_size(w, h); pop_up_text.set_buffer(buf); pop_up_text.wrap_mode(fltk::text::WrapMode::AtBounds, 5); let mut buf = pop_up_text.buffer().unwrap(); buf.set_text(std::include_str!("strings/uinput_error.txt")); pop_up.end(); pop_up.make_modal(true); pop_up.show(); }), }) { return Ok(()); } is_server_running = true; write_config(&config); let mut web_sock = SocketAddr::new(config.bind_address, config.web_port); #[cfg(not(target_os = "windows"))] { if web_sock.ip().is_unspecified() { // try to guess an ip let mut ips = Vec::::new(); for iface in datalink::interfaces() .iter() .filter(|iface| iface.is_up() && !iface.is_loopback()) { for ipnetw in &iface.ips { if (ipnetw.is_ipv4() && web_sock.ip().is_ipv4()) || (ipnetw.is_ipv6() && web_sock.ip().is_ipv6()) { // filtering ipv6 unicast requires nightly or more fiddling, // lets wait for nightlies to stabilize... ips.push(ipnetw.ip()) } } } if !ips.is_empty() { web_sock.set_ip(ips[0]); } if ips.len() > 1 { info!("Found more than one IP address for browsers to connect to,"); info!("other urls are:"); for ip in &ips[1..] { info!("http://{}", SocketAddr::new(*ip, config.web_port)); } } } } #[cfg(not(target_os = "windows"))] { use image::Luma; use qrcode::QrCode; let addr_string = format!("http://{}", web_sock); output_server_addr.set_value(&addr_string); let mut url_string = addr_string; if let Some(access_code) = &config.access_code { url_string.push_str("?access_code="); url_string.push_str( &percent_encoding::utf8_percent_encode( access_code, percent_encoding::NON_ALPHANUMERIC, ) .to_string(), ); } let cb = move |qr_frame: &mut Frame, _, _, w, h| { let code = QrCode::new(&url_string).unwrap(); let img_buf = code.render::>().build(); let image = image::DynamicImage::ImageLuma8(img_buf); let dims = min(w, h) as u32; let image = image.resize_exact(dims, dims, image::imageops::FilterType::Nearest); let mut buf = vec![]; let mut cursor = Cursor::new(&mut buf); image .write_to(&mut cursor, image::ImageFormat::Png) .unwrap(); let png = PngImage::from_data(&buf).unwrap(); qr_frame.set_image(Some(png)); }; let x = qr_frame.x(); let y = qr_frame.y(); let w = qr_frame.width(); let h = qr_frame.height(); cb(&mut qr_frame, x, y, w, h); qr_frame.resize_callback(cb); qr_frame.show(); } #[cfg(target_os = "windows")] { if web_sock.ip().is_unspecified() { output_server_addr.set_value("http://"); } else { output_server_addr.set_value(&format!("http://{}", web_sock.to_string())); } } output_server_addr.show(); but.set_label("Stop"); } else { weylus.stop(); but.set_label("Start"); output_server_addr.hide(); qr_frame.resize_callback(|_, _, _, _, _| {}); qr_frame.hide(); is_server_running = false; } Ok(()) }() { error!("{}", err); }; }; if auto_start { toggle_server(&mut but_toggle); } but_toggle.set_callback(toggle_server); app.run().expect("Failed to run Gui!"); // TODO: Remove when https://github.com/fltk-rs/fltk-rs/issues/1480 is fixed // this is required to drop the callback and do a graceful shutdown of the web server but_toggle.set_callback(|_| ()); } const BORDER: i32 = 30; static WINCTX: Mutex> = Mutex::new(None); struct InputAreaWindowContext { win: Window, choice_mouse: Choice, choice_touch: Choice, choice_pen: Choice, workspaces: Vec, } pub fn get_input_area( no_gui: bool, output_sender: std::sync::mpsc::Sender, ) { // If no gui is running there is no event loop and windows can not be created. // That's why we initialize the fltk app here one the first call. if no_gui { static GUI_INITIALIZED: AtomicBool = AtomicBool::new(false); if !GUI_INITIALIZED.swap(true, std::sync::atomic::Ordering::Relaxed) { std::thread::spawn(move || { let _app = App::default().with_scheme(fltk::app::AppScheme::Gtk); let mut winctx = create_custom_input_area_window(); custom_input_area_window_handle_events(&mut winctx.win, output_sender.clone()); show_overlay_window(&mut winctx); WINCTX.lock().unwrap().replace(winctx); loop { // calling wait_for ensures that the fltk event loop keeps running even if // there is no window shown if let Err(err) = app::wait_for(1.0) { warn!("Error waiting for fltk events: {err}."); } } }); } else { fltk::app::awake_callback(move || { let mut winctx = WINCTX.lock().unwrap(); let winctx = winctx.as_mut().unwrap(); custom_input_area_window_handle_events(&mut winctx.win, output_sender.clone()); show_overlay_window(winctx); }); } } else { fltk::app::awake_callback(move || { let mut winctx = WINCTX.lock().unwrap(); if winctx.is_none() { winctx.replace(create_custom_input_area_window()); } let winctx = winctx.as_mut().unwrap(); custom_input_area_window_handle_events(&mut winctx.win, output_sender.clone()); show_overlay_window(winctx); }); } } fn create_custom_input_area_window() -> InputAreaWindowContext { let mut win = Window::default().with_size(600, 600).center_screen(); win.make_resizable(true); win.set_border(false); win.set_frame(FrameType::FlatBox); win.set_color(fltk::enums::Color::from_rgb(240, 240, 240)); let mut frame = Frame::default() .with_size(win.w() - 2 * BORDER, win.h() - 2 * BORDER) .center_of_parent() .with_label( "Press Enter to submit\ncurrent selection as\ncustom input area,\nEscape to abort.", ); frame.set_label_type(LabelType::Normal); frame.set_label_size(20); frame.set_color(fltk::enums::Color::Black); frame.set_frame(FrameType::BorderFrame); frame.set_label_font(fltk::enums::Font::HelveticaBold); let width = 200; let height = 30; let padding = 10; let tool_tip = "Some systems may have the input device mapped to a specific screen, this screen has to be selected here. Otherwise input mapping will be wrong. Selecting None disables any mapping."; let mut choice_mouse = Choice::default() .with_size(width, height) .with_pos(padding, 4 * padding) .center_x(&frame) .with_id("choice_mouse") .with_label("Map Mouse from:"); choice_mouse.set_tooltip(tool_tip); let mut choice_touch = Choice::default() .with_size(width, height) .below_of(&choice_mouse, padding) .with_id("choice_touch") .with_label("Map Touch from:"); choice_touch.set_tooltip(tool_tip); let mut choice_pen = Choice::default() .with_size(width, height) .below_of(&choice_touch, padding) .with_id("choice_pen") .with_label("Map Pen from:"); choice_pen.set_tooltip(tool_tip); frame.handle(|frame, event| match event { fltk::enums::Event::Push => { if app::event_clicks() { if let Some(mut win) = frame.window() { win.fullscreen(!win.fullscreen_active()); } true } else { false } } _ => false, }); win.resize_callback(move |_win, _x, _y, w, h| { frame.resize(BORDER, BORDER, w - 2 * BORDER, h - 2 * BORDER) }); win.end(); InputAreaWindowContext { win, choice_mouse, choice_touch, choice_pen, workspaces: Vec::new(), } } fn custom_input_area_window_handle_events( win: &mut Window, sender: std::sync::mpsc::Sender, ) { #[derive(Debug)] enum MouseFlags { All, Edge(bool, bool), Corner(bool, bool), } fn get_mouse_flags(win: &Window, x: i32, y: i32) -> MouseFlags { let dx0 = (win.x() - x).abs(); let dy0 = (win.y() - y).abs(); let dx1 = (win.x() + win.w() - x).abs(); let dy1 = (win.y() + win.h() - y).abs(); let dx = min(dx0, dx1); let dy = min(dy0, dy1); let d = min(dx, dy); if d <= BORDER { if dx <= BORDER && dy <= BORDER { MouseFlags::Corner(dx0 <= dx1, dy0 <= dy1) } else { MouseFlags::Edge(dx <= dy, if dx <= dy { dx0 <= dx1 } else { dy0 <= dy1 }) } } else { MouseFlags::All } } fn get_screen_coords_from_event_coords(win: &Window, (x, y): (i32, i32)) -> (i32, i32) { (x + win.x(), y + win.y()) } fn set_cursor( win: &mut Window, current_cursor: &mut fltk::enums::Cursor, flags: Option, ) { let cursor = match flags { Some(MouseFlags::All) => fltk::enums::Cursor::Move, Some(MouseFlags::Edge(bx, by)) => match (bx, by) { (true, true) => fltk::enums::Cursor::W, (true, false) => fltk::enums::Cursor::E, (false, true) => fltk::enums::Cursor::N, (false, false) => fltk::enums::Cursor::S, }, Some(MouseFlags::Corner(bx, by)) => match (bx, by) { (true, true) => fltk::enums::Cursor::NWSE, (true, false) => fltk::enums::Cursor::NESW, (false, true) => fltk::enums::Cursor::NESW, (false, false) => fltk::enums::Cursor::NWSE, }, None => fltk::enums::Cursor::Default, }; if *current_cursor != cursor { *current_cursor = cursor; win.set_cursor(cursor); } } let mut drag_flags = MouseFlags::All; let mut current_cursor = fltk::enums::Cursor::Default; let mut x = 0; let mut y = 0; let mut win_x_drag_start = 0; let mut win_y_drag_start = 0; let mut win_w_drag_start = 0; let mut win_h_drag_start = 0; win.handle(move |win, event| { match event { fltk::enums::Event::Move => { let (x, y) = get_screen_coords_from_event_coords(&win, app::event_coords()); let flags = get_mouse_flags(&win, x, y); set_cursor(win, &mut current_cursor, Some(flags)); true } fltk::enums::Event::Leave => { win.set_cursor(fltk::enums::Cursor::Default); true } fltk::enums::Event::Push => { (x, y) = get_screen_coords_from_event_coords(&win, app::event_coords()); win_x_drag_start = win.x(); win_y_drag_start = win.y(); win_w_drag_start = win.w(); win_h_drag_start = win.h(); drag_flags = get_mouse_flags(&win, x, y); true } fltk::enums::Event::Drag => { if win.opacity() == 1.0 { win.set_opacity(0.5); } let (x_new, y_new) = get_screen_coords_from_event_coords(&win, app::event_coords()); let dx = x_new - x; let dy = y_new - y; match drag_flags { MouseFlags::All => win.set_pos(win_x_drag_start + dx, win_y_drag_start + dy), MouseFlags::Edge(bx, by) => match (bx, by) { (true, true) => win.resize( win_x_drag_start + dx, win_y_drag_start, win_w_drag_start - dx, win_h_drag_start, ), (true, false) => win.resize( win_x_drag_start, win_y_drag_start, win_w_drag_start + dx, win_h_drag_start, ), (false, true) => win.resize( win_x_drag_start, win_y_drag_start + dy, win_w_drag_start, win_h_drag_start - dy, ), (false, false) => win.resize( win_x_drag_start, win_y_drag_start, win_w_drag_start, win_h_drag_start + dy, ), }, MouseFlags::Corner(bx, by) => match (bx, by) { (true, true) => win.resize( win_x_drag_start + dx, win_y_drag_start + dy, win_w_drag_start - dx, win_h_drag_start - dy, ), (true, false) => win.resize( win_x_drag_start + dx, win_y_drag_start, win_w_drag_start - dx, win_h_drag_start + dy, ), (false, true) => win.resize( win_x_drag_start, win_y_drag_start + dy, win_w_drag_start + dx, win_h_drag_start - dy, ), (false, false) => win.resize( win_x_drag_start, win_y_drag_start, win_w_drag_start + dx, win_h_drag_start + dy, ), }, } true } fltk::enums::Event::Released => { if win.opacity() != 1.0 { win.set_opacity(1.0); } true } fltk::enums::Event::KeyDown => match app::event_key() { fltk::enums::Key::Enter => { fn relative_rect(win: &Window, workspace: &Rect) -> Rect { // clamp rect to workspace and ensure it has non-zero area let mut rect = crate::protocol::Rect { x: (win.x() as f64 - workspace.x).min(workspace.w) / workspace.w, y: (win.y() as f64 - workspace.y).min(workspace.h) / workspace.h, w: win.w().max(1) as f64 / workspace.w, h: win.h().max(1) as f64 / workspace.h, }; rect.w = rect.w.min(1.0 - rect.x); rect.h = rect.h.min(1.0 - rect.y); rect } win.set_cursor(fltk::enums::Cursor::Default); win.hide(); let mut areas = CustomInputAreas::default(); let workspaces = WINCTX.lock().unwrap().as_ref().unwrap().workspaces.clone(); for (name, area) in [ ("choice_mouse", &mut areas.mouse), ("choice_touch", &mut areas.touch), ("choice_pen", &mut areas.pen), ] { let c: Choice = fltk::app::widget_from_id(name).unwrap(); match c.value() { 0 => (), v @ 1.. if (v as usize) <= workspaces.len() => { let workspace = workspaces[v as usize - 1]; *area = Some(relative_rect(win, &workspace)) } v => warn!("Unexpected value in {name}: {v}!"), } } sender.send(areas).unwrap(); true } fltk::enums::Key::Escape => { win.set_cursor(fltk::enums::Cursor::Default); win.hide(); true } _ => false, }, _ => false, } }); } fn show_overlay_window(winctx: &mut InputAreaWindowContext) { let win = &mut winctx.win; if win.shown() { return; } let screens = fltk::app::Screen::all_screens(); winctx.workspaces.clear(); winctx.workspaces.push(get_full_workspace_rect()); for screen in &screens { let fltk::draw::Rect { x, y, w, h } = screen.work_area(); winctx.workspaces.push(Rect { x: x as f64, y: y as f64, w: w as f64, h: h as f64, }); } for c in [ &mut winctx.choice_mouse, &mut winctx.choice_touch, &mut winctx.choice_pen, ] { let v = c.value(); c.clear(); c.add_choice("None"); c.add_choice("Full Workspace"); for screen in &screens { c.add_choice(&format!( "Screen {n} at {w}x{h}+{x}+{y}", n = screen.n, w = screen.w(), h = screen.h(), x = screen.x(), y = screen.y() )); } if v >= 0 && (v as usize) < 2 + screens.len() { c.set_value(v); } else { c.set_value(0); } } if win.fullscreen_active() { win.set_size(600, 600); let n = win.screen_num(); let screen = app::Screen::new(n).unwrap(); win.set_pos( screen.x() + (screen.w() - 600) / 2, screen.y() + (screen.h() - 600) / 2, ); } win.show(); win.set_on_top(); win.set_visible_focus(); } pub fn get_full_workspace_rect() -> Rect { let mut rect = Rect::default(); for screen in fltk::app::Screen::all_screens() { let fltk::draw::Rect { x, y, w, h } = screen.work_area(); rect.x = (x as f64).min(rect.x); rect.y = (y as f64).min(rect.y); rect.w = ((x + w) as f64).max(rect.w); rect.h = ((y + h) as f64).max(rect.h); } rect } ================================================ FILE: src/input/autopilot_device.rs ================================================ use autopilot::geometry::Size; use autopilot::mouse; use autopilot::mouse::ScrollDirection; use autopilot::screen::size as screen_size; use tracing::warn; use crate::input::device::{InputDevice, InputDeviceType}; use crate::protocol::{Button, KeyboardEvent, KeyboardEventType, PointerEvent, WheelEvent}; use crate::capturable::{Capturable, Geometry}; pub struct AutoPilotDevice { capturable: Box, } impl AutoPilotDevice { pub fn new(capturable: Box) -> Self { Self { capturable } } } impl InputDevice for AutoPilotDevice { fn send_wheel_event(&mut self, event: &WheelEvent) { match event.dy { 1..=i32::MAX => mouse::scroll(ScrollDirection::Up, 1), i32::MIN..=-1 => mouse::scroll(ScrollDirection::Down, 1), 0 => {} } } fn send_pointer_event(&mut self, event: &PointerEvent) { if !event.is_primary { return; } if let Err(err) = self.capturable.before_input() { warn!("Failed to activate window, sending no input ({})", err); return; } let (x_rel, y_rel, width_rel, height_rel) = match self.capturable.geometry().unwrap() { Geometry::Relative(x, y, width, height) => (x, y, width, height), #[cfg(target_os = "windows")] _ => { warn!("Failed to get window geometry, sending no input"); return; } }; #[cfg(not(target_os = "macos"))] let Size { width, height } = screen_size(); #[cfg(target_os = "macos")] let (_, _, width, height) = match crate::capturable::core_graphics::screen_coordsys() { Ok(bounds) => bounds, Err(err) => { warn!("Could not determine global coordinate system: {}", err); return; } }; if let Err(err) = mouse::move_to(autopilot::geometry::Point::new( (event.x * width_rel + x_rel) * width, (event.y * height_rel + y_rel) * height, )) { warn!("Could not move mouse: {}", err); } match event.button { Button::PRIMARY => { mouse::toggle(mouse::Button::Left, event.buttons.contains(event.button)) } Button::AUXILARY => { mouse::toggle(mouse::Button::Middle, event.buttons.contains(event.button)) } Button::SECONDARY => { mouse::toggle(mouse::Button::Right, event.buttons.contains(event.button)) } _ => (), } } fn send_keyboard_event(&mut self, event: &KeyboardEvent) { use autopilot::key::{Character, Code, KeyCode}; let state = match event.event_type { KeyboardEventType::UP => false, KeyboardEventType::DOWN => true, // autopilot doesn't handle this, so just do nothing KeyboardEventType::REPEAT => return, }; fn map_key(code: &str) -> Option { match code { "Escape" => Some(KeyCode::Escape), "Enter" => Some(KeyCode::Return), "Backspace" => Some(KeyCode::Backspace), "Tab" => Some(KeyCode::Tab), "Space" => Some(KeyCode::Space), "CapsLock" => Some(KeyCode::CapsLock), "F1" => Some(KeyCode::F1), "F2" => Some(KeyCode::F2), "F3" => Some(KeyCode::F3), "F4" => Some(KeyCode::F4), "F5" => Some(KeyCode::F5), "F6" => Some(KeyCode::F6), "F7" => Some(KeyCode::F7), "F8" => Some(KeyCode::F8), "F9" => Some(KeyCode::F9), "F10" => Some(KeyCode::F10), "F11" => Some(KeyCode::F11), "F12" => Some(KeyCode::F12), "F13" => Some(KeyCode::F13), "F14" => Some(KeyCode::F14), "F15" => Some(KeyCode::F15), "F16" => Some(KeyCode::F16), "F17" => Some(KeyCode::F17), "F18" => Some(KeyCode::F18), "F19" => Some(KeyCode::F19), "F20" => Some(KeyCode::F20), "F21" => Some(KeyCode::F21), "F22" => Some(KeyCode::F22), "F23" => Some(KeyCode::F23), "F24" => Some(KeyCode::F24), "Home" => Some(KeyCode::Home), "ArrowUp" => Some(KeyCode::UpArrow), "PageUp" => Some(KeyCode::PageUp), "ArrowLeft" => Some(KeyCode::LeftArrow), "ArrowRight" => Some(KeyCode::RightArrow), "End" => Some(KeyCode::End), "ArrowDown" => Some(KeyCode::DownArrow), "PageDown" => Some(KeyCode::PageDown), "Delete" => Some(KeyCode::Delete), "ControlLeft" | "ControlRight" => Some(KeyCode::Control), "AltLeft" | "AltRight" => Some(KeyCode::Alt), "MetaLeft" | "MetaRight" => Some(KeyCode::Meta), "ShiftLeft" | "ShiftRight" => Some(KeyCode::Shift), _ => None, } } let key = map_key(&event.code); let mut flags = Vec::new(); if event.ctrl { flags.push(autopilot::key::Flag::Control); } if event.alt { flags.push(autopilot::key::Flag::Alt); } if event.meta { flags.push(autopilot::key::Flag::Meta); } if event.shift { flags.push(autopilot::key::Flag::Shift); } match key { Some(key) => autopilot::key::toggle(&Code(key), state, &flags, 0), None => { for c in event.key.chars() { autopilot::key::toggle(&Character(c), state, &flags, 0); } } } } fn set_capturable(&mut self, capturable: Box) { self.capturable = capturable; } fn device_type(&self) -> InputDeviceType { InputDeviceType::AutoPilotDevice } } ================================================ FILE: src/input/autopilot_device_win.rs ================================================ use winapi::shared::minwindef::DWORD; use winapi::shared::windef::{HWND, POINT}; use winapi::um::winuser::*; use tracing::warn; use crate::input::autopilot_device::AutoPilotDevice; use crate::input::device::{InputDevice, InputDeviceType}; use crate::protocol::{ Button, KeyboardEvent, PointerEvent, PointerEventType, PointerType, WheelEvent, }; use crate::capturable::{Capturable, Geometry}; pub struct WindowsInput { capturable: Box, autopilot_device: AutoPilotDevice, pointer_device_handle: *mut HSYNTHETICPOINTERDEVICE__, touch_device_handle: *mut HSYNTHETICPOINTERDEVICE__, multitouch_map: std::collections::HashMap, } impl WindowsInput { pub fn new(capturable: Box) -> Self { unsafe { InitializeTouchInjection(5, TOUCH_FEEDBACK_DEFAULT); Self { capturable: capturable.clone(), autopilot_device: AutoPilotDevice::new(capturable), pointer_device_handle: CreateSyntheticPointerDevice(PT_PEN, 1, 1), touch_device_handle: CreateSyntheticPointerDevice(PT_TOUCH, 5, 1), multitouch_map: std::collections::HashMap::new(), } } } } impl InputDevice for WindowsInput { fn send_wheel_event(&mut self, event: &WheelEvent) { unsafe { mouse_event(MOUSEEVENTF_WHEEL, 0, 0, event.dy as DWORD, 0) }; } fn send_pointer_event(&mut self, event: &PointerEvent) { if let Err(err) = self.capturable.before_input() { warn!("Failed to activate window, sending no input ({})", err); return; } let Geometry::VirtualScreen(offset_x, offset_y, width, height, left, top) = self.capturable.geometry().unwrap() else { unreachable!() }; let (x, y) = ( (event.x * width as f64) as i32 + offset_x, (event.y * height as f64) as i32 + offset_y, ); let mut pointer_flags = match event.event_type { PointerEventType::DOWN => { POINTER_FLAG_INRANGE | POINTER_FLAG_INCONTACT | POINTER_FLAG_DOWN } PointerEventType::MOVE | PointerEventType::OVER | PointerEventType::ENTER => { POINTER_FLAG_INRANGE | POINTER_FLAG_UPDATE } PointerEventType::UP => POINTER_FLAG_UP, PointerEventType::CANCEL | PointerEventType::LEAVE | PointerEventType::OUT => { POINTER_FLAG_INRANGE | POINTER_FLAG_UPDATE | POINTER_FLAG_CANCELED } }; let button_change_type = match event.buttons { Button::PRIMARY => { pointer_flags |= POINTER_FLAG_INCONTACT; POINTER_CHANGE_FIRSTBUTTON_DOWN } Button::SECONDARY => POINTER_CHANGE_SECONDBUTTON_DOWN, Button::AUXILARY => POINTER_CHANGE_THIRDBUTTON_DOWN, Button::NONE => POINTER_CHANGE_NONE, _ => POINTER_CHANGE_NONE, }; if event.is_primary { pointer_flags |= POINTER_FLAG_PRIMARY; } match event.pointer_type { PointerType::Pen => { unsafe { let mut pointer_type_info = POINTER_TYPE_INFO { type_: PT_PEN, u: std::mem::zeroed(), }; *pointer_type_info.u.penInfo_mut() = POINTER_PEN_INFO { pointerInfo: POINTER_INFO { pointerType: PT_PEN, pointerId: event.pointer_id as u32, frameId: 0, pointerFlags: pointer_flags, sourceDevice: 0 as *mut winapi::ctypes::c_void, //maybe use syntheticPointerDeviceHandle here but works with 0 hwndTarget: 0 as HWND, ptPixelLocation: POINT { x: x, y: y }, ptHimetricLocation: POINT { x: 0, y: 0 }, ptPixelLocationRaw: POINT { x: x, y: y }, ptHimetricLocationRaw: POINT { x: 0, y: 0 }, dwTime: 0, historyCount: 1, InputData: 0, dwKeyStates: 0, PerformanceCount: 0, ButtonChangeType: button_change_type, }, penFlags: PEN_FLAG_NONE, penMask: PEN_MASK_PRESSURE | PEN_MASK_ROTATION | PEN_MASK_TILT_X | PEN_MASK_TILT_Y, pressure: (event.pressure * 1024f64) as u32, rotation: event.twist as u32, tiltX: event.tilt_x, tiltY: event.tilt_y, }; InjectSyntheticPointerInput(self.pointer_device_handle, &pointer_type_info, 1); } } PointerType::Touch => { unsafe { let mut pointer_type_info = POINTER_TYPE_INFO { type_: PT_TOUCH, u: std::mem::zeroed(), }; let mut pointer_touch_info: POINTER_TOUCH_INFO = std::mem::zeroed(); pointer_touch_info.pointerInfo = std::mem::zeroed(); pointer_touch_info.pointerInfo.pointerType = PT_TOUCH; pointer_touch_info.pointerInfo.pointerFlags = pointer_flags; pointer_touch_info.pointerInfo.pointerId = event.pointer_id as u32; //event.pointer_id as u32; Using the actual pointer id causes errors in the touch injection pointer_touch_info.pointerInfo.ptPixelLocation = POINT { x, y }; pointer_touch_info.touchFlags = TOUCH_FLAG_NONE; pointer_touch_info.touchMask = TOUCH_MASK_PRESSURE; pointer_touch_info.pressure = (event.pressure * 1024f64) as u32; pointer_touch_info.pointerInfo.ButtonChangeType = button_change_type; *pointer_type_info.u.touchInfo_mut() = pointer_touch_info; self.multitouch_map .insert(event.pointer_id, pointer_type_info); let len = self.multitouch_map.len(); let mut pointer_type_info_vec: Vec = Vec::new(); for (_i, info) in self.multitouch_map.iter().enumerate() { pointer_type_info_vec.push(*info.1); } let b: Box<[POINTER_TYPE_INFO]> = pointer_type_info_vec.into_boxed_slice(); let m: *mut POINTER_TYPE_INFO = Box::into_raw(b) as _; InjectSyntheticPointerInput(self.touch_device_handle, m, len as u32); match event.event_type { PointerEventType::DOWN | PointerEventType::MOVE | PointerEventType::OVER | PointerEventType::ENTER => {} PointerEventType::UP | PointerEventType::CANCEL | PointerEventType::LEAVE | PointerEventType::OUT => { self.multitouch_map.remove(&event.pointer_id); } } } } PointerType::Mouse => { let mut dw_flags = 0; let (screen_x, screen_y) = ( (event.x * width as f64) as i32 + left, (event.y * height as f64) as i32 + top, ); match event.event_type { PointerEventType::DOWN => match event.buttons { Button::PRIMARY => { dw_flags |= MOUSEEVENTF_LEFTDOWN; } Button::SECONDARY => { dw_flags |= MOUSEEVENTF_RIGHTDOWN; } Button::AUXILARY => { dw_flags |= MOUSEEVENTF_MIDDLEDOWN; } _ => {} }, PointerEventType::MOVE | PointerEventType::OVER | PointerEventType::ENTER => { unsafe { SetCursorPos(screen_x, screen_y) }; } PointerEventType::UP => match event.button { Button::PRIMARY => { dw_flags |= MOUSEEVENTF_LEFTUP; } Button::SECONDARY => { dw_flags |= MOUSEEVENTF_RIGHTUP; } Button::AUXILARY => { dw_flags |= MOUSEEVENTF_MIDDLEUP; } _ => {} }, PointerEventType::CANCEL | PointerEventType::LEAVE | PointerEventType::OUT => { dw_flags |= MOUSEEVENTF_LEFTUP; } } unsafe { mouse_event(dw_flags, 0 as u32, 0 as u32, 0, 0) }; } PointerType::Unknown => todo!(), } } fn send_keyboard_event(&mut self, event: &KeyboardEvent) { self.autopilot_device.send_keyboard_event(event); } fn set_capturable(&mut self, capturable: Box) { self.capturable = capturable; } fn device_type(&self) -> InputDeviceType { InputDeviceType::WindowsInput } } ================================================ FILE: src/input/device.rs ================================================ use crate::capturable::Capturable; use crate::protocol::{KeyboardEvent, PointerEvent, WheelEvent}; #[derive(PartialEq, Eq)] pub enum InputDeviceType { AutoPilotDevice, UInputDevice, #[cfg(target_os = "windows")] WindowsInput, } pub trait InputDevice { fn send_wheel_event(&mut self, event: &WheelEvent); fn send_pointer_event(&mut self, event: &PointerEvent); fn send_keyboard_event(&mut self, event: &KeyboardEvent); fn set_capturable(&mut self, capturable: Box); fn device_type(&self) -> InputDeviceType; } ================================================ FILE: src/input/mod.rs ================================================ pub mod autopilot_device; pub mod device; #[cfg(target_os = "windows")] pub mod autopilot_device_win; #[cfg(target_os = "linux")] pub mod uinput_device; #[cfg(target_os = "linux")] #[allow(dead_code)] pub mod uinput_keys; ================================================ FILE: src/input/uinput_device.rs ================================================ use std::cmp::Ordering; use std::ffi::CString; use std::os::raw::{c_char, c_int}; use std::time::{Duration, Instant}; use crate::capturable::x11::X11Context; use crate::capturable::{Capturable, Geometry}; use crate::input::device::{InputDevice, InputDeviceType}; use crate::protocol::{ Button, KeyboardEvent, KeyboardEventType, KeyboardLocation, PointerEvent, PointerEventType, PointerType, Rect, WheelEvent, }; use crate::cerror::CError; use tracing::{debug, warn}; extern "C" { fn init_uinput_keyboard(name: *const c_char, err: *mut CError) -> c_int; fn init_uinput_stylus(name: *const c_char, err: *mut CError) -> c_int; fn init_uinput_mouse(name: *const c_char, err: *mut CError) -> c_int; fn init_uinput_touch(name: *const c_char, err: *mut CError) -> c_int; fn destroy_uinput_device(fd: c_int); fn send_uinput_event(device: c_int, typ: c_int, code: c_int, value: c_int, err: *mut CError); } struct MultiTouch { id: i64, } pub struct UInputDevice { keyboard_fd: c_int, stylus_fd: c_int, mouse_fd: c_int, touch_fd: c_int, touches: [Option; 5], tool_pen_active: bool, pen_touching: bool, last_pen_event: Instant, capturable: Box, geometry: Rect, name_mouse_device: String, name_stylus_device: String, name_touch_device: String, num_mouse_mapping_tries: usize, num_stylus_mapping_tries: usize, num_touch_mapping_tries: usize, x11ctx: Option, } impl UInputDevice { pub fn new(capturable: Box, id: &Option) -> Result { let mut suffix = String::new(); if let Some(id) = id { suffix = format!(" - {}", id); } let mut err = CError::new(); let name_stylus = format!("Weylus Stylus{}", suffix); let name_stylus_c_str = CString::new(name_stylus.as_bytes()).unwrap(); let stylus_fd = unsafe { init_uinput_stylus(name_stylus_c_str.as_ptr(), &mut err) }; if err.is_err() { return Err(err); } let name_mouse = format!("Weylus Mouse{}", suffix); let name_mouse_c_str = CString::new(name_mouse.as_bytes()).unwrap(); let mouse_fd = unsafe { init_uinput_mouse(name_mouse_c_str.as_ptr(), &mut err) }; if err.is_err() { unsafe { destroy_uinput_device(stylus_fd) }; return Err(err); } let name_touch = format!("Weylus Touch{}", suffix); let name_touch_c_str = CString::new(name_touch.as_bytes()).unwrap(); let touch_fd = unsafe { init_uinput_touch(name_touch_c_str.as_ptr(), &mut err) }; if err.is_err() { unsafe { destroy_uinput_device(stylus_fd); destroy_uinput_device(mouse_fd); } return Err(err); } let name_keyboard = format!("Weylus Keyboard{}", suffix); let name_keyboard_c_str = CString::new(name_keyboard.as_bytes()).unwrap(); let keyboard_fd = unsafe { init_uinput_keyboard(name_keyboard_c_str.as_ptr(), &mut err) }; if err.is_err() { unsafe { destroy_uinput_device(stylus_fd); destroy_uinput_device(mouse_fd); destroy_uinput_device(touch_fd); } return Err(err); } Ok(Self { keyboard_fd, stylus_fd, mouse_fd, touch_fd, touches: Default::default(), tool_pen_active: false, pen_touching: false, last_pen_event: Instant::now(), capturable, geometry: Rect::default(), name_mouse_device: name_mouse, name_touch_device: name_touch, name_stylus_device: name_stylus, num_mouse_mapping_tries: 0, num_stylus_mapping_tries: 0, num_touch_mapping_tries: 0, x11ctx: X11Context::new(), }) } fn transform_x(&self, x: f64) -> i32 { let x = (x * self.geometry.w + self.geometry.x) * ABS_MAX; x as i32 } fn transform_y(&self, y: f64) -> i32 { let y = (y * self.geometry.h + self.geometry.y) * ABS_MAX; y as i32 } fn transform_pressure(&self, p: f64) -> i32 { (p * ABS_MAX) as i32 } fn transform_touch_size(&self, s: f64) -> i32 { (s * ABS_MAX) as i32 } fn find_slot(&self, id: i64) -> Option { self.touches .iter() .enumerate() .find_map(|(slot, mt)| match mt { Some(mt) => { if mt.id == id { Some(slot) } else { None } } _ => None, }) } fn send(&self, fd: c_int, typ: c_int, code: c_int, value: c_int) { let mut err = CError::new(); unsafe { send_uinput_event(fd, typ, code, value, &mut err); } if err.is_err() { warn!("{}", err); } } } impl Drop for UInputDevice { fn drop(&mut self) { unsafe { destroy_uinput_device(self.keyboard_fd); destroy_uinput_device(self.stylus_fd); destroy_uinput_device(self.mouse_fd); destroy_uinput_device(self.touch_fd); }; } } // Event Types const ET_SYNC: c_int = 0x00; const ET_KEY: c_int = 0x01; const ET_RELATIVE: c_int = 0x02; const ET_ABSOLUTE: c_int = 0x03; const ET_MSC: c_int = 0x04; // Event Codes const EC_SYNC_REPORT: c_int = 0; const EC_KEY_MOUSE_LEFT: c_int = 0x110; const EC_KEY_MOUSE_RIGHT: c_int = 0x111; const EC_KEY_MOUSE_MIDDLE: c_int = 0x112; const EC_KEY_TOOL_PEN: c_int = 0x140; const EC_KEY_TOOL_RUBBER: c_int = 0x141; const EC_KEY_TOUCH: c_int = 0x14a; const EC_KEY_TOOL_FINGER: c_int = 0x145; const EC_KEY_TOOL_DOUBLETAP: c_int = 0x14d; const EC_KEY_TOOL_TRIPLETAP: c_int = 0x14e; const EC_KEY_TOOL_QUADTAP: c_int = 0x14f; /* Four fingers on trackpad */ const EC_KEY_TOOL_QUINTTAP: c_int = 0x148; /* Five fingers on trackpad */ //const EC_RELATIVE_X: c_int = 0x00; //const EC_RELATIVE_Y: c_int = 0x01; const EC_REL_HWHEEL: c_int = 0x06; const EC_REL_WHEEL: c_int = 0x08; const EC_REL_WHEEL_HI_RES: c_int = 0x0b; const EC_REL_HWHEEL_HI_RES: c_int = 0x0c; const EC_ABSOLUTE_X: c_int = 0x00; const EC_ABSOLUTE_Y: c_int = 0x01; const EC_ABSOLUTE_PRESSURE: c_int = 0x18; const EC_ABSOLUTE_TILT_X: c_int = 0x1a; const EC_ABSOLUTE_TILT_Y: c_int = 0x1b; const EC_ABS_MT_SLOT: c_int = 0x2f; /* MT slot being modified */ const EC_ABS_MT_TOUCH_MAJOR: c_int = 0x30; /* Major axis of touching ellipse */ const EC_ABS_MT_TOUCH_MINOR: c_int = 0x31; /* Minor axis (omit if circular) */ const EC_ABS_MT_ORIENTATION: c_int = 0x34; /* Ellipse orientation */ const EC_ABS_MT_POSITION_X: c_int = 0x35; /* Center X touch position */ const EC_ABS_MT_POSITION_Y: c_int = 0x36; /* Center Y touch position */ const EC_ABS_MT_TRACKING_ID: c_int = 0x39; /* Unique ID of initiated contact */ const EC_ABS_MT_PRESSURE: c_int = 0x3a; /* Pressure on contact area */ const EC_MSC_TIMESTAMP: c_int = 0x05; // This is choosen somewhat arbitrarily // describes maximum value for ABS_X, ABS_Y, ABS_... // This corresponds to PointerEvent values of 1.0 const ABS_MAX: f64 = 65535.0; // This specifies how many times it should be attempted to map the input devices created via uinput // to the entire screen and not only a single monitor. Actually this is a workaround because // apparently it is impossible to set the correct mapping in a sane way. The reason is that X needs // some time to register new input devices, which makes it impossible to configure them right after // creation as the devices won't be available for configuration at that time. This means one has to // wait an unspecified amount of time until the devices show up. But just sleeping for example 3 // seconds does not solve the issue either because the input device for the stylus does not show up // if there has not been any input. As a matter of fact things are even more compilcated as for // some reason the stylus device created via uinput creates two devices for X. One can not be // mapped to the screen (this is the device that shows up with out the need to send actual inputs // via uinput) and another one that can be mapped to the screen. But this is the device that // requires sending inputs via uinput first other wise it does not show up. This is why this crude // method of just setting the mapping forcefully on the first MAX_SCREEN_MAPPING_TRIES input events // has been choosen. If anyone knows a better solution: PLEASE FIX THIS! const MAX_SCREEN_MAPPING_TRIES: usize = 100; impl InputDevice for UInputDevice { fn send_wheel_event(&mut self, event: &WheelEvent) { if let Err(err) = self.capturable.before_input() { warn!("Failed to activate window, sending no input ({})", err); return; } fn direction(d: i32) -> i32 { match d.cmp(&0) { Ordering::Equal => 0, Ordering::Less => -1, Ordering::Greater => 1, } } self.send( self.mouse_fd, ET_RELATIVE, EC_REL_WHEEL, direction(event.dy), ); self.send( self.mouse_fd, ET_RELATIVE, EC_REL_HWHEEL, direction(event.dx), ); self.send(self.mouse_fd, ET_RELATIVE, EC_REL_WHEEL_HI_RES, event.dy); self.send(self.mouse_fd, ET_RELATIVE, EC_REL_HWHEEL_HI_RES, event.dx); self.send( self.mouse_fd, ET_MSC, EC_MSC_TIMESTAMP, (event.timestamp % (i32::MAX as u64 + 1)) as i32, ); self.send(self.mouse_fd, ET_SYNC, EC_SYNC_REPORT, 0); } fn send_pointer_event(&mut self, event: &PointerEvent) { if let Err(err) = self.capturable.before_input() { warn!("Failed to activate window, sending no input ({})", err); return; } let (x, y, width, height) = match self.capturable.geometry().unwrap() { Geometry::Relative(x, y, width, height) => (x, y, width, height), }; self.geometry.x = x; self.geometry.y = y; self.geometry.w = width; self.geometry.h = height; match event.pointer_type { PointerType::Touch => { if self.num_touch_mapping_tries < MAX_SCREEN_MAPPING_TRIES { if let Some(x11ctx) = &mut self.x11ctx { // Mapping input does not work on XWayland as xinput list does not expose // device names and thus we can not identify the devices created by uinput if let Ok(session_type) = std::env::var("XDG_SESSION_TYPE") { if session_type != "wayland" { x11ctx.map_input_device_to_entire_screen( &self.name_touch_device, false, ); } } else { x11ctx .map_input_device_to_entire_screen(&self.name_touch_device, false); } } self.num_touch_mapping_tries += 1; } // This is a workaround for browsers that send events when the pen is hovering but // do not send an event when the pen leaves the hovering range. If the pen is left // in this state touch rejection may stay active and touch won't work. // Therefore, we manually remove the pen after a short delay. if self.tool_pen_active && !self.pen_touching && (Instant::now() - self.last_pen_event) > Duration::from_millis(50) { self.tool_pen_active = false; self.send(self.stylus_fd, ET_KEY, EC_KEY_TOUCH, 0); self.send(self.stylus_fd, ET_KEY, EC_KEY_TOOL_PEN, 0); self.send(self.stylus_fd, ET_KEY, EC_KEY_TOOL_RUBBER, 0); self.send(self.stylus_fd, ET_ABSOLUTE, EC_ABSOLUTE_PRESSURE, 0); self.send( self.stylus_fd, ET_MSC, EC_MSC_TIMESTAMP, (event.timestamp % (i32::MAX as u64 + 1)) as i32, ); self.send(self.stylus_fd, ET_SYNC, EC_SYNC_REPORT, 0); } match event.event_type { PointerEventType::DOWN | PointerEventType::MOVE | PointerEventType::OVER | PointerEventType::ENTER => { let slot: usize; // check if this event is already assigned to one of our 10 multitouch slots if let Some(s) = self.find_slot(event.pointer_id) { slot = s; } else { // this event is not assigned to a slot, lets try to do so now // find the first unused slot if let Some(s) = self.touches .iter() .enumerate() .find_map(|(slot, mt)| match mt { None => Some(slot), Some(_) => None, }) { slot = s; self.touches[slot] = Some(MultiTouch { id: event.pointer_id, }) } else { // out of slots, do nothing return; } }; self.send(self.touch_fd, ET_ABSOLUTE, EC_ABS_MT_SLOT, slot as i32); self.send( self.touch_fd, ET_ABSOLUTE, EC_ABS_MT_TRACKING_ID, slot as i32, ); if let PointerEventType::DOWN = event.event_type { self.send(self.touch_fd, ET_KEY, EC_KEY_TOUCH, 1); match slot { 1 => self.send(self.touch_fd, ET_KEY, EC_KEY_TOOL_FINGER, 0), 2 => self.send(self.touch_fd, ET_KEY, EC_KEY_TOOL_DOUBLETAP, 0), 3 => self.send(self.touch_fd, ET_KEY, EC_KEY_TOOL_TRIPLETAP, 0), 4 => self.send(self.touch_fd, ET_KEY, EC_KEY_TOOL_QUADTAP, 0), _ => self.send(self.touch_fd, ET_KEY, EC_KEY_TOOL_QUINTTAP, 0), } match slot { 1 => self.send(self.touch_fd, ET_KEY, EC_KEY_TOOL_DOUBLETAP, 1), 2 => self.send(self.touch_fd, ET_KEY, EC_KEY_TOOL_TRIPLETAP, 1), 3 => self.send(self.touch_fd, ET_KEY, EC_KEY_TOOL_QUADTAP, 1), 4 => self.send(self.touch_fd, ET_KEY, EC_KEY_TOOL_QUINTTAP, 1), _ => self.send(self.touch_fd, ET_KEY, EC_KEY_TOOL_FINGER, 1), } } self.send( self.touch_fd, ET_ABSOLUTE, EC_ABS_MT_PRESSURE, self.transform_pressure(event.pressure), ); let major: i32; let minor: i32; let orientation = if event.height >= event.width { major = self.transform_touch_size(event.height); minor = self.transform_touch_size(event.width); 0 } else { major = self.transform_touch_size(event.width); minor = self.transform_touch_size(event.height); 1 }; self.send(self.touch_fd, ET_ABSOLUTE, EC_ABS_MT_TOUCH_MAJOR, major); self.send(self.touch_fd, ET_ABSOLUTE, EC_ABS_MT_TOUCH_MINOR, minor); self.send( self.touch_fd, ET_ABSOLUTE, EC_ABS_MT_ORIENTATION, orientation, ); self.send( self.touch_fd, ET_ABSOLUTE, EC_ABS_MT_POSITION_X, self.transform_x(event.x), ); self.send( self.touch_fd, ET_ABSOLUTE, EC_ABS_MT_POSITION_Y, self.transform_y(event.y), ); self.send( self.touch_fd, ET_ABSOLUTE, EC_ABSOLUTE_X, self.transform_x(event.x), ); self.send( self.touch_fd, ET_ABSOLUTE, EC_ABSOLUTE_Y, self.transform_y(event.y), ); self.send( self.touch_fd, ET_MSC, EC_MSC_TIMESTAMP, (event.timestamp % (i32::MAX as u64 + 1)) as i32, ); self.send(self.touch_fd, ET_SYNC, EC_SYNC_REPORT, 0); } PointerEventType::CANCEL | PointerEventType::UP | PointerEventType::LEAVE | PointerEventType::OUT => { // remove from slot if let Some(slot) = self.find_slot(event.pointer_id) { self.send(self.touch_fd, ET_ABSOLUTE, EC_ABS_MT_SLOT, slot as i32); self.send(self.touch_fd, ET_ABSOLUTE, EC_ABS_MT_TRACKING_ID, -1); self.send(self.touch_fd, ET_KEY, EC_KEY_TOUCH, 0); self.send(self.touch_fd, ET_KEY, EC_KEY_TOOL_FINGER, 0); match slot { 1 => self.send(self.touch_fd, ET_KEY, EC_KEY_TOOL_DOUBLETAP, 0), 2 => self.send(self.touch_fd, ET_KEY, EC_KEY_TOOL_TRIPLETAP, 0), 3 => self.send(self.touch_fd, ET_KEY, EC_KEY_TOOL_QUADTAP, 0), 4 => self.send(self.touch_fd, ET_KEY, EC_KEY_TOOL_QUINTTAP, 0), _ => (), } self.send( self.touch_fd, ET_MSC, EC_MSC_TIMESTAMP, (event.timestamp % (i32::MAX as u64 + 1)) as i32, ); self.send(self.touch_fd, ET_SYNC, EC_SYNC_REPORT, 0); self.touches[slot] = None; } } }; } PointerType::Pen => { self.last_pen_event = Instant::now(); if self.num_stylus_mapping_tries < MAX_SCREEN_MAPPING_TRIES { if let Some(x11ctx) = &mut self.x11ctx { // Mapping input does not work on XWayland as xinput list does not expose // device names and thus we can not identify the devices created by uinput if let Ok(session_type) = std::env::var("XDG_SESSION_TYPE") { if session_type != "wayland" { x11ctx.map_input_device_to_entire_screen( &self.name_stylus_device, true, ); } } else { x11ctx .map_input_device_to_entire_screen(&self.name_stylus_device, true); } } self.num_touch_mapping_tries += 1; } match event.event_type { PointerEventType::DOWN | PointerEventType::MOVE | PointerEventType::OVER | PointerEventType::ENTER => { if let PointerEventType::DOWN = event.event_type { self.pen_touching = true; self.send(self.stylus_fd, ET_KEY, EC_KEY_TOUCH, 1); } if !self.tool_pen_active && !event.buttons.contains(Button::ERASER) { self.send(self.stylus_fd, ET_KEY, EC_KEY_TOOL_PEN, 1); self.send(self.stylus_fd, ET_KEY, EC_KEY_TOOL_RUBBER, 0); self.tool_pen_active = true; } if let Button::ERASER = event.button { self.send(self.stylus_fd, ET_KEY, EC_KEY_TOOL_PEN, 0); self.send(self.stylus_fd, ET_KEY, EC_KEY_TOOL_RUBBER, 1); self.tool_pen_active = false; } self.send( self.stylus_fd, ET_ABSOLUTE, EC_ABSOLUTE_X, self.transform_x(event.x), ); self.send( self.stylus_fd, ET_ABSOLUTE, EC_ABSOLUTE_Y, self.transform_y(event.y), ); self.send( self.stylus_fd, ET_ABSOLUTE, EC_ABSOLUTE_PRESSURE, if self.pen_touching { self.transform_pressure(event.pressure) } else { 0 }, ); self.send( self.stylus_fd, ET_ABSOLUTE, EC_ABSOLUTE_TILT_X, event.tilt_x, ); self.send( self.stylus_fd, ET_ABSOLUTE, EC_ABSOLUTE_TILT_Y, event.tilt_y, ); } PointerEventType::UP | PointerEventType::CANCEL | PointerEventType::LEAVE | PointerEventType::OUT => { self.send(self.stylus_fd, ET_KEY, EC_KEY_TOUCH, 0); self.send(self.stylus_fd, ET_KEY, EC_KEY_TOOL_PEN, 0); self.send(self.stylus_fd, ET_KEY, EC_KEY_TOOL_RUBBER, 0); self.send(self.stylus_fd, ET_ABSOLUTE, EC_ABSOLUTE_PRESSURE, 0); self.tool_pen_active = false; self.pen_touching = false; } } self.send( self.stylus_fd, ET_MSC, EC_MSC_TIMESTAMP, (event.timestamp % (i32::MAX as u64 + 1)) as i32, ); self.send(self.stylus_fd, ET_SYNC, EC_SYNC_REPORT, 0); } PointerType::Mouse | PointerType::Unknown => { if self.num_mouse_mapping_tries < MAX_SCREEN_MAPPING_TRIES { if let Some(x11ctx) = &mut self.x11ctx { // Mapping input does not work on XWayland as xinput list does not expose // device names and thus we can not identify the devices created by uinput if let Ok(session_type) = std::env::var("XDG_SESSION_TYPE") { if session_type != "wayland" { x11ctx.map_input_device_to_entire_screen( &self.name_mouse_device, false, ); } } else { x11ctx .map_input_device_to_entire_screen(&self.name_mouse_device, false); } } self.num_touch_mapping_tries += 1; } match event.event_type { PointerEventType::DOWN | PointerEventType::MOVE | PointerEventType::OVER | PointerEventType::ENTER => { if let PointerEventType::DOWN = event.event_type { match event.button { Button::PRIMARY => { self.send(self.mouse_fd, ET_KEY, EC_KEY_MOUSE_LEFT, 1) } Button::SECONDARY => { self.send(self.mouse_fd, ET_KEY, EC_KEY_MOUSE_RIGHT, 1) } Button::AUXILARY => { self.send(self.mouse_fd, ET_KEY, EC_KEY_MOUSE_MIDDLE, 1) } _ => (), } } self.send( self.mouse_fd, ET_ABSOLUTE, EC_ABSOLUTE_X, self.transform_x(event.x), ); self.send( self.mouse_fd, ET_ABSOLUTE, EC_ABSOLUTE_Y, self.transform_y(event.y), ); } PointerEventType::UP | PointerEventType::CANCEL | PointerEventType::LEAVE | PointerEventType::OUT => match event.button { Button::PRIMARY => self.send(self.mouse_fd, ET_KEY, EC_KEY_MOUSE_LEFT, 0), Button::SECONDARY => { self.send(self.mouse_fd, ET_KEY, EC_KEY_MOUSE_RIGHT, 0) } Button::AUXILARY => { self.send(self.mouse_fd, ET_KEY, EC_KEY_MOUSE_MIDDLE, 0) } _ => (), }, } self.send( self.mouse_fd, ET_MSC, EC_MSC_TIMESTAMP, (event.timestamp % (i32::MAX as u64 + 1)) as i32, ); self.send(self.mouse_fd, ET_SYNC, EC_SYNC_REPORT, 0); } } } fn send_keyboard_event(&mut self, event: &KeyboardEvent) { use crate::input::uinput_keys::*; if let Err(err) = self.capturable.before_input() { warn!("Failed to activate window, sending no input ({})", err); return; } fn map_key(code: &str, location: &KeyboardLocation) -> c_int { match (code, location) { ("Escape", _) => KEY_ESC, ("Digit0", KeyboardLocation::NUMPAD) => KEY_KP0, ("Digit1", KeyboardLocation::NUMPAD) => KEY_KP1, ("Digit2", KeyboardLocation::NUMPAD) => KEY_KP2, ("Digit3", KeyboardLocation::NUMPAD) => KEY_KP3, ("Digit4", KeyboardLocation::NUMPAD) => KEY_KP4, ("Digit5", KeyboardLocation::NUMPAD) => KEY_KP5, ("Digit6", KeyboardLocation::NUMPAD) => KEY_KP6, ("Digit7", KeyboardLocation::NUMPAD) => KEY_KP7, ("Digit8", KeyboardLocation::NUMPAD) => KEY_KP8, ("Digit9", KeyboardLocation::NUMPAD) => KEY_KP9, ("Minus", KeyboardLocation::NUMPAD) => KEY_KPMINUS, ("Equal", KeyboardLocation::NUMPAD) => KEY_KPEQUAL, ("Enter", KeyboardLocation::NUMPAD) => KEY_KPENTER, ("Digit0", _) => KEY_0, ("Digit1", _) => KEY_1, ("Digit2", _) => KEY_2, ("Digit3", _) => KEY_3, ("Digit4", _) => KEY_4, ("Digit5", _) => KEY_5, ("Digit6", _) => KEY_6, ("Digit7", _) => KEY_7, ("Digit8", _) => KEY_8, ("Digit9", _) => KEY_9, ("Minus", _) => KEY_MINUS, ("Equal", _) => KEY_EQUAL, ("Enter", _) => KEY_ENTER, ("Backspace", _) => KEY_BACKSPACE, ("Tab", _) => KEY_TAB, ("KeyA", _) => KEY_A, ("KeyB", _) => KEY_B, ("KeyC", _) => KEY_C, ("KeyD", _) => KEY_D, ("KeyE", _) => KEY_E, ("KeyF", _) => KEY_F, ("KeyG", _) => KEY_G, ("KeyH", _) => KEY_H, ("KeyI", _) => KEY_I, ("KeyJ", _) => KEY_J, ("KeyK", _) => KEY_K, ("KeyL", _) => KEY_L, ("KeyM", _) => KEY_M, ("KeyN", _) => KEY_N, ("KeyO", _) => KEY_O, ("KeyP", _) => KEY_P, ("KeyQ", _) => KEY_Q, ("KeyR", _) => KEY_R, ("KeyS", _) => KEY_S, ("KeyT", _) => KEY_T, ("KeyU", _) => KEY_U, ("KeyV", _) => KEY_V, ("KeyW", _) => KEY_W, ("KeyX", _) => KEY_X, ("KeyY", _) => KEY_Y, ("KeyZ", _) => KEY_Z, ("BracketLeft", _) => KEY_LEFTBRACE, ("BracketRight", _) => KEY_RIGHTBRACE, ("Semicolon", _) => KEY_SEMICOLON, ("Quote", _) => KEY_APOSTROPHE, ("Backquote", _) => KEY_GRAVE, ("Backslash", _) => KEY_BACKSLASH, ("Comma", _) => KEY_COMMA, ("Period", _) => KEY_DOT, ("Slash", _) => KEY_SLASH, ("Space", _) => KEY_SPACE, ("CapsLock", _) => KEY_CAPSLOCK, ("NumpadMultiply", _) => KEY_KPASTERISK, ("F1", _) => KEY_F1, ("F2", _) => KEY_F2, ("F3", _) => KEY_F3, ("F4", _) => KEY_F4, ("F5", _) => KEY_F5, ("F6", _) => KEY_F6, ("F7", _) => KEY_F7, ("F8", _) => KEY_F8, ("F9", _) => KEY_F9, ("F10", _) => KEY_F10, ("F11", _) => KEY_F11, ("F12", _) => KEY_F12, ("F13", _) => KEY_F13, ("F14", _) => KEY_F14, ("F15", _) => KEY_F15, ("F16", _) => KEY_F16, ("F17", _) => KEY_F17, ("F18", _) => KEY_F18, ("F19", _) => KEY_F19, ("F20", _) => KEY_F20, ("F21", _) => KEY_F21, ("F22", _) => KEY_F22, ("F23", _) => KEY_F23, ("F24", _) => KEY_F24, ("NumLock", _) => KEY_NUMLOCK, ("ScrollLock", _) => KEY_SCROLLLOCK, ("Numpad0", _) => KEY_KP0, ("Numpad1", _) => KEY_KP1, ("Numpad2", _) => KEY_KP2, ("Numpad3", _) => KEY_KP3, ("Numpad4", _) => KEY_KP4, ("Numpad5", _) => KEY_KP5, ("Numpad6", _) => KEY_KP6, ("Numpad7", _) => KEY_KP7, ("Numpad8", _) => KEY_KP8, ("Numpad9", _) => KEY_KP9, ("NumpadSubtract", _) => KEY_KPMINUS, ("NumpadAdd", _) => KEY_KPPLUS, // ("NumpadDecimal", _) => ?, ("IntlBackslash", _) => KEY_102ND, ("IntlRo", _) => KEY_RO, ("NumpadEnter", _) => KEY_KPENTER, ("NumpadDivide", _) => KEY_KPSLASH, ("NumpadEqual", _) => KEY_KPEQUAL, ("NumpadComma", _) => KEY_KPCOMMA, ("NumpadParenLeft", _) => KEY_KPLEFTPAREN, ("NumpadParenRight", _) => KEY_KPRIGHTPAREN, // ("NumpadChangeSign", _) => ?, // ("Convert", _) => ?, ("KanaMode", _) => KEY_KATAKANA, // ("NonConvert", _) => ?, ("PrintScreen", _) => KEY_SYSRQ, ("Home", _) => KEY_HOME, ("ArrowUp", _) => KEY_UP, ("PageUp", _) => KEY_PAGEUP, ("ArrowLeft", _) => KEY_LEFT, ("ArrowRight", _) => KEY_RIGHT, ("End", _) => KEY_END, ("ArrowDown", _) => KEY_DOWN, ("PageDown", _) => KEY_PAGEDOWN, ("Insert", _) => KEY_INSERT, ("Delete", _) => KEY_DELETE, ("VolumeMute", _) | ("AudioVolumeMute", _) => KEY_MUTE, ("VolumeDown", _) | ("AudioVolumeDown", _) => KEY_VOLUMEDOWN, ("VolumeUp", _) | ("AudioVolumeUp", _) => KEY_VOLUMEUP, ("Pause", _) => KEY_PAUSE, ("Lang1", _) => KEY_HANGUEL, ("Lang2", _) => KEY_HANJA, ("IntlYen", _) => KEY_YEN, ("OSLeft", _) => KEY_LEFTMETA, ("OSRight", _) => KEY_RIGHTMETA, ("ContextMenu", _) => KEY_MENU, // ("BrowserStop", _) => ?, ("Cancel", _) => KEY_CANCEL, ("Again", _) => KEY_AGAIN, ("Props", _) => KEY_PROPS, ("Undo", _) => KEY_UNDO, // ("Select", _) => ?, ("Copy", _) => KEY_COPY, ("Open", _) => KEY_OPEN, ("Paste", _) => KEY_PASTE, ("Find", _) => KEY_FIND, ("Cut", _) => KEY_CUT, ("Help", _) => KEY_HELP, // ("LaunchApp2", _) => ?, // ("LaunchApp1", _) => , ("LaunchMail", _) => KEY_MAIL, // ("BrowserFavorites", _) => ?, // ("BrowserBack", _) => ?, // ("BrowserForward", _) => ?, ("Eject", _) => KEY_EJECTCD, ("MediaTrackNext", _) => KEY_NEXTSONG, ("MediaPlayPause", _) => KEY_PLAYPAUSE, ("MediaTrackPrevious", _) => KEY_PREVIOUSSONG, ("MediaStop", _) => KEY_STOPCD, ("MediaSelect", _) | ("LaunchMediaPlayer", _) => KEY_MEDIA, // ("BrowserHome", _) => ?, // ("BrowserRefresh", _) => ?, // ("BrowserSearch", _) => ?, ("Power", _) => KEY_POWER, ("Sleep", _) => KEY_SLEEP, ("WakeUp", _) => KEY_WAKEUP, ("ControlLeft", _) => KEY_LEFTCTRL, ("ControlRight", _) => KEY_RIGHTCTRL, ("AltLeft", _) => KEY_LEFTALT, ("AltRight", _) => KEY_RIGHTALT, ("MetaLeft", _) => KEY_LEFTMETA, ("MetaRight", _) => KEY_RIGHTMETA, ("ShiftLeft", _) => KEY_LEFTSHIFT, ("ShiftRight", _) => KEY_RIGHTSHIFT, _ => KEY_UNKNOWN, } } let key_code: c_int = map_key(&event.code, &event.location); let state: c_int = match event.event_type { KeyboardEventType::UP => 0, KeyboardEventType::DOWN => 1, KeyboardEventType::REPEAT => 2, }; if key_code == KEY_UNKNOWN { if let KeyboardEventType::DOWN = event.event_type { if !event.key.is_empty() { // If the key is unknow try inserting the unicode character directly // to do so use CTRL + SHIFT + U + UTF16 HEX of the unicode point. let unicode_keys = event .key .encode_utf16() .map(|b| format!("{:X}", b)) .collect::>() .concat(); debug!( "Got unknown key: {} code: {}, trying to insert unicode using ctrl + \ shift + u + {}!", event.code, event.key, unicode_keys ); self.send(self.keyboard_fd, ET_KEY, KEY_LEFTCTRL, 1); self.send(self.keyboard_fd, ET_KEY, KEY_LEFTSHIFT, 1); self.send(self.keyboard_fd, ET_KEY, KEY_U, 1); self.send(self.keyboard_fd, ET_SYNC, EC_SYNC_REPORT, 0); for c in unicode_keys.chars() { let key_code = if c.is_alphabetic() { map_key(&format!("Key{}", c), &KeyboardLocation::STANDARD) } else { map_key(&format!("Digit{}", c), &KeyboardLocation::STANDARD) }; self.send(self.keyboard_fd, ET_KEY, key_code, 1); self.send(self.keyboard_fd, ET_SYNC, EC_SYNC_REPORT, 0); self.send(self.keyboard_fd, ET_KEY, key_code, 0); self.send(self.keyboard_fd, ET_SYNC, EC_SYNC_REPORT, 0); } self.send(self.keyboard_fd, ET_KEY, KEY_LEFTCTRL, 0); self.send(self.keyboard_fd, ET_KEY, KEY_LEFTSHIFT, 0); self.send(self.keyboard_fd, ET_KEY, KEY_U, 0); self.send(self.keyboard_fd, ET_SYNC, EC_SYNC_REPORT, 0); } } else { debug!( "Got unknow key: code: {} key: {}, ignoring event.", event.code, event.key ); } return; } if event.ctrl { self.send(self.keyboard_fd, ET_KEY, KEY_LEFTCTRL, state); self.send(self.keyboard_fd, ET_SYNC, EC_SYNC_REPORT, 0); } if event.alt { self.send(self.keyboard_fd, ET_KEY, KEY_LEFTALT, state); self.send(self.keyboard_fd, ET_SYNC, EC_SYNC_REPORT, 0); } if event.meta { self.send(self.keyboard_fd, ET_KEY, KEY_LEFTMETA, state); self.send(self.keyboard_fd, ET_SYNC, EC_SYNC_REPORT, 0); } if event.shift { self.send(self.keyboard_fd, ET_KEY, KEY_LEFTSHIFT, state); self.send(self.keyboard_fd, ET_SYNC, EC_SYNC_REPORT, 0); } self.send(self.keyboard_fd, ET_KEY, key_code, state); self.send(self.keyboard_fd, ET_SYNC, EC_SYNC_REPORT, 0); } fn set_capturable(&mut self, capturable: Box) { self.capturable = capturable; } fn device_type(&self) -> InputDeviceType { InputDeviceType::UInputDevice } } ================================================ FILE: src/input/uinput_keys.rs ================================================ use std::os::raw::c_int; pub const KEY_ESC: c_int = 1; pub const KEY_1: c_int = 2; pub const KEY_2: c_int = 3; pub const KEY_3: c_int = 4; pub const KEY_4: c_int = 5; pub const KEY_5: c_int = 6; pub const KEY_6: c_int = 7; pub const KEY_7: c_int = 8; pub const KEY_8: c_int = 9; pub const KEY_9: c_int = 10; pub const KEY_0: c_int = 11; pub const KEY_MINUS: c_int = 12; pub const KEY_EQUAL: c_int = 13; pub const KEY_BACKSPACE: c_int = 14; pub const KEY_TAB: c_int = 15; pub const KEY_Q: c_int = 16; pub const KEY_W: c_int = 17; pub const KEY_E: c_int = 18; pub const KEY_R: c_int = 19; pub const KEY_T: c_int = 20; pub const KEY_Y: c_int = 21; pub const KEY_U: c_int = 22; pub const KEY_I: c_int = 23; pub const KEY_O: c_int = 24; pub const KEY_P: c_int = 25; pub const KEY_LEFTBRACE: c_int = 26; pub const KEY_RIGHTBRACE: c_int = 27; pub const KEY_ENTER: c_int = 28; pub const KEY_LEFTCTRL: c_int = 29; pub const KEY_A: c_int = 30; pub const KEY_S: c_int = 31; pub const KEY_D: c_int = 32; pub const KEY_F: c_int = 33; pub const KEY_G: c_int = 34; pub const KEY_H: c_int = 35; pub const KEY_J: c_int = 36; pub const KEY_K: c_int = 37; pub const KEY_L: c_int = 38; pub const KEY_SEMICOLON: c_int = 39; pub const KEY_APOSTROPHE: c_int = 40; pub const KEY_GRAVE: c_int = 41; pub const KEY_LEFTSHIFT: c_int = 42; pub const KEY_BACKSLASH: c_int = 43; pub const KEY_Z: c_int = 44; pub const KEY_X: c_int = 45; pub const KEY_C: c_int = 46; pub const KEY_V: c_int = 47; pub const KEY_B: c_int = 48; pub const KEY_N: c_int = 49; pub const KEY_M: c_int = 50; pub const KEY_COMMA: c_int = 51; pub const KEY_DOT: c_int = 52; pub const KEY_SLASH: c_int = 53; pub const KEY_RIGHTSHIFT: c_int = 54; pub const KEY_KPASTERISK: c_int = 55; pub const KEY_LEFTALT: c_int = 56; pub const KEY_SPACE: c_int = 57; pub const KEY_CAPSLOCK: c_int = 58; pub const KEY_F1: c_int = 59; pub const KEY_F2: c_int = 60; pub const KEY_F3: c_int = 61; pub const KEY_F4: c_int = 62; pub const KEY_F5: c_int = 63; pub const KEY_F6: c_int = 64; pub const KEY_F7: c_int = 65; pub const KEY_F8: c_int = 66; pub const KEY_F9: c_int = 67; pub const KEY_F10: c_int = 68; pub const KEY_NUMLOCK: c_int = 69; pub const KEY_SCROLLLOCK: c_int = 70; pub const KEY_KP7: c_int = 71; pub const KEY_KP8: c_int = 72; pub const KEY_KP9: c_int = 73; pub const KEY_KPMINUS: c_int = 74; pub const KEY_KP4: c_int = 75; pub const KEY_KP5: c_int = 76; pub const KEY_KP6: c_int = 77; pub const KEY_KPPLUS: c_int = 78; pub const KEY_KP1: c_int = 79; pub const KEY_KP2: c_int = 80; pub const KEY_KP3: c_int = 81; pub const KEY_KP0: c_int = 82; pub const KEY_KPDOT: c_int = 83; pub const KEY_ZENKAKUHANKAKU: c_int = 85; pub const KEY_102ND: c_int = 86; pub const KEY_F11: c_int = 87; pub const KEY_F12: c_int = 88; pub const KEY_RO: c_int = 89; pub const KEY_KATAKANA: c_int = 90; pub const KEY_HIRAGANA: c_int = 91; pub const KEY_HENKAN: c_int = 92; pub const KEY_KATAKANAHIRAGANA: c_int = 93; pub const KEY_MUHENKAN: c_int = 94; pub const KEY_KPJPCOMMA: c_int = 95; pub const KEY_KPENTER: c_int = 96; pub const KEY_RIGHTCTRL: c_int = 97; pub const KEY_KPSLASH: c_int = 98; pub const KEY_SYSRQ: c_int = 99; pub const KEY_RIGHTALT: c_int = 100; pub const KEY_LINEFEED: c_int = 101; pub const KEY_HOME: c_int = 102; pub const KEY_UP: c_int = 103; pub const KEY_PAGEUP: c_int = 104; pub const KEY_LEFT: c_int = 105; pub const KEY_RIGHT: c_int = 106; pub const KEY_END: c_int = 107; pub const KEY_DOWN: c_int = 108; pub const KEY_PAGEDOWN: c_int = 109; pub const KEY_INSERT: c_int = 110; pub const KEY_DELETE: c_int = 111; pub const KEY_MACRO: c_int = 112; pub const KEY_MUTE: c_int = 113; pub const KEY_VOLUMEDOWN: c_int = 114; pub const KEY_VOLUMEUP: c_int = 115; pub const KEY_POWER: c_int = 116; /* SC System Power Down */ pub const KEY_KPEQUAL: c_int = 117; pub const KEY_KPPLUSMINUS: c_int = 118; pub const KEY_PAUSE: c_int = 119; pub const KEY_SCALE: c_int = 120; /* AL Compiz Scale (Expose) */ pub const KEY_KPCOMMA: c_int = 121; pub const KEY_HANGEUL: c_int = 122; pub const KEY_HANGUEL: c_int = KEY_HANGEUL; pub const KEY_HANJA: c_int = 123; pub const KEY_YEN: c_int = 124; pub const KEY_LEFTMETA: c_int = 125; pub const KEY_RIGHTMETA: c_int = 126; pub const KEY_COMPOSE: c_int = 127; pub const KEY_STOP: c_int = 128; /* AC Stop */ pub const KEY_AGAIN: c_int = 129; pub const KEY_PROPS: c_int = 130; /* AC Properties */ pub const KEY_UNDO: c_int = 131; /* AC Undo */ pub const KEY_FRONT: c_int = 132; pub const KEY_COPY: c_int = 133; /* AC Copy */ pub const KEY_OPEN: c_int = 134; /* AC Open */ pub const KEY_PASTE: c_int = 135; /* AC Paste */ pub const KEY_FIND: c_int = 136; /* AC Search */ pub const KEY_CUT: c_int = 137; /* AC Cut */ pub const KEY_HELP: c_int = 138; /* AL Integrated Help Center */ pub const KEY_MENU: c_int = 139; /* Menu (show menu) */ pub const KEY_CALC: c_int = 140; /* AL Calculator */ pub const KEY_SETUP: c_int = 141; pub const KEY_SLEEP: c_int = 142; /* SC System Sleep */ pub const KEY_WAKEUP: c_int = 143; /* System Wake Up */ pub const KEY_FILE: c_int = 144; /* AL Local Machine Browser */ pub const KEY_SENDFILE: c_int = 145; pub const KEY_DELETEFILE: c_int = 146; pub const KEY_XFER: c_int = 147; pub const KEY_PROG1: c_int = 148; pub const KEY_PROG2: c_int = 149; pub const KEY_WWW: c_int = 150; /* AL Internet Browser */ pub const KEY_MSDOS: c_int = 151; pub const KEY_COFFEE: c_int = 152; /* AL Terminal Lock/Screensaver */ pub const KEY_SCREENLOCK: c_int = KEY_COFFEE; pub const KEY_ROTATE_DISPLAY: c_int = 153; /* Display orientation for e.g. tablets */ pub const KEY_DIRECTION: c_int = KEY_ROTATE_DISPLAY; pub const KEY_CYCLEWINDOWS: c_int = 154; pub const KEY_MAIL: c_int = 155; pub const KEY_BOOKMARKS: c_int = 156; /* AC Bookmarks */ pub const KEY_COMPUTER: c_int = 157; pub const KEY_BACK: c_int = 158; /* AC Back */ pub const KEY_FORWARD: c_int = 159; /* AC Forward */ pub const KEY_CLOSECD: c_int = 160; pub const KEY_EJECTCD: c_int = 161; pub const KEY_EJECTCLOSECD: c_int = 162; pub const KEY_NEXTSONG: c_int = 163; pub const KEY_PLAYPAUSE: c_int = 164; pub const KEY_PREVIOUSSONG: c_int = 165; pub const KEY_STOPCD: c_int = 166; pub const KEY_RECORD: c_int = 167; pub const KEY_REWIND: c_int = 168; pub const KEY_PHONE: c_int = 169; /* Media Select Telephone */ pub const KEY_ISO: c_int = 170; pub const KEY_CONFIG: c_int = 171; /* AL Consumer Control Configuration */ pub const KEY_HOMEPAGE: c_int = 172; /* AC Home */ pub const KEY_REFRESH: c_int = 173; /* AC Refresh */ pub const KEY_EXIT: c_int = 174; /* AC Exit */ pub const KEY_MOVE: c_int = 175; pub const KEY_EDIT: c_int = 176; pub const KEY_SCROLLUP: c_int = 177; pub const KEY_SCROLLDOWN: c_int = 178; pub const KEY_KPLEFTPAREN: c_int = 179; pub const KEY_KPRIGHTPAREN: c_int = 180; pub const KEY_NEW: c_int = 181; /* AC New */ pub const KEY_REDO: c_int = 182; /* AC Redo/Repeat */ pub const KEY_F13: c_int = 183; pub const KEY_F14: c_int = 184; pub const KEY_F15: c_int = 185; pub const KEY_F16: c_int = 186; pub const KEY_F17: c_int = 187; pub const KEY_F18: c_int = 188; pub const KEY_F19: c_int = 189; pub const KEY_F20: c_int = 190; pub const KEY_F21: c_int = 191; pub const KEY_F22: c_int = 192; pub const KEY_F23: c_int = 193; pub const KEY_F24: c_int = 194; pub const KEY_PLAYCD: c_int = 200; pub const KEY_PAUSECD: c_int = 201; pub const KEY_PROG3: c_int = 202; pub const KEY_PROG4: c_int = 203; pub const KEY_DASHBOARD: c_int = 204; /* AL Dashboard */ pub const KEY_SUSPEND: c_int = 205; pub const KEY_CLOSE: c_int = 206; /* AC Close */ pub const KEY_PLAY: c_int = 207; pub const KEY_FASTFORWARD: c_int = 208; pub const KEY_BASSBOOST: c_int = 209; pub const KEY_PRINT: c_int = 210; /* AC Print */ pub const KEY_HP: c_int = 211; pub const KEY_CAMERA: c_int = 212; pub const KEY_SOUND: c_int = 213; pub const KEY_QUESTION: c_int = 214; pub const KEY_EMAIL: c_int = 215; pub const KEY_CHAT: c_int = 216; pub const KEY_SEARCH: c_int = 217; pub const KEY_CONNECT: c_int = 218; pub const KEY_FINANCE: c_int = 219; /* AL Checkbook/Finance */ pub const KEY_SPORT: c_int = 220; pub const KEY_SHOP: c_int = 221; pub const KEY_ALTERASE: c_int = 222; pub const KEY_CANCEL: c_int = 223; /* AC Cancel */ pub const KEY_BRIGHTNESSDOWN: c_int = 224; pub const KEY_BRIGHTNESSUP: c_int = 225; pub const KEY_MEDIA: c_int = 226; pub const KEY_SWITCHVIDEOMODE: c_int = 227; /* Cycle between available video outputs (Monitor/LCD/TV-out/etc) */ pub const KEY_KBDILLUMTOGGLE: c_int = 228; pub const KEY_KBDILLUMDOWN: c_int = 229; pub const KEY_KBDILLUMUP: c_int = 230; pub const KEY_SEND: c_int = 231; /* AC Send */ pub const KEY_REPLY: c_int = 232; /* AC Reply */ pub const KEY_FORWARDMAIL: c_int = 233; /* AC Forward Msg */ pub const KEY_SAVE: c_int = 234; /* AC Save */ pub const KEY_DOCUMENTS: c_int = 235; pub const KEY_BATTERY: c_int = 236; pub const KEY_BLUETOOTH: c_int = 237; pub const KEY_WLAN: c_int = 238; pub const KEY_UWB: c_int = 239; pub const KEY_UNKNOWN: c_int = 240; pub const KEY_VIDEO_NEXT: c_int = 241; /* drive next video source */ pub const KEY_VIDEO_PREV: c_int = 242; /* drive previous video source */ pub const KEY_BRIGHTNESS_CYCLE: c_int = 243; /* brightness up, after max is min */ pub const KEY_BRIGHTNESS_AUTO: c_int = 244; /* Set Auto Brightness: manual brightness control is off, rely on ambient */ pub const KEY_BRIGHTNESS_ZERO: c_int = KEY_BRIGHTNESS_AUTO; pub const KEY_DISPLAY_OFF: c_int = 245; /* display device to off state */ pub const KEY_WWAN: c_int = 246; /* Wireless WAN (LTE, UMTS, GSM, etc.) */ pub const KEY_WIMAX: c_int = KEY_WWAN; pub const KEY_RFKILL: c_int = 247; /* Key that controls all radios */ pub const KEY_MICMUTE: c_int = 248; /* Mute / unmute the microphone */ ================================================ FILE: src/log.rs ================================================ use std::ffi::CStr; use std::io::Write; use std::os::raw::c_char; use std::sync::mpsc; use tracing::{debug, error, info, trace, warn}; use tracing_subscriber::layer::SubscriberExt; extern "C" { fn init_ffmpeg_logger(); } struct GuiTracingWriter { gui_sender: mpsc::SyncSender, } impl Write for GuiTracingWriter { fn write(&mut self, buf: &[u8]) -> std::io::Result { self.gui_sender .try_send(String::from_utf8_lossy(buf).trim_start().into()) .ok(); Ok(buf.len()) } fn flush(&mut self) -> std::io::Result<()> { Ok(()) } } struct GuiTracingWriterFactory { sender: mpsc::SyncSender, } impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for GuiTracingWriterFactory { type Writer = GuiTracingWriter; fn make_writer(&'a self) -> Self::Writer { Self::Writer { gui_sender: self.sender.clone(), } } } pub fn get_log_level() -> tracing::Level { #[cfg(debug_assertions)] let mut level = tracing::Level::DEBUG; #[cfg(not(debug_assertions))] let mut level = tracing::Level::INFO; if let Ok(var) = std::env::var("WEYLUS_LOG_LEVEL") { let l: Result = var.parse(); if let Ok(l) = l { level = l; } } level } pub fn setup_logging(sender: mpsc::SyncSender) { if std::env::var("WEYLUS_LOG_JSON").is_ok() { let logger = tracing_subscriber::fmt() .json() .with_max_level(get_log_level()) .with_writer(std::io::stdout) .finish() .with( tracing_subscriber::fmt::Layer::default() .with_ansi(false) .without_time() .with_target(false) .compact() .with_writer(GuiTracingWriterFactory { sender }), ); tracing::subscriber::set_global_default(logger).expect("Failed to setup logger!"); } else { let logger = tracing_subscriber::fmt() .with_max_level(get_log_level()) .with_writer(std::io::stderr) .finish() .with( tracing_subscriber::fmt::Layer::default() .with_ansi(false) .without_time() .with_target(false) .compact() .with_writer(GuiTracingWriterFactory { sender }), ); tracing::subscriber::set_global_default(logger).expect("Failed to setup logger!"); } unsafe { init_ffmpeg_logger(); } } #[no_mangle] fn log_error_rust(msg: *const c_char) { let msg = unsafe { CStr::from_ptr(msg) }.to_string_lossy(); error!("{}", msg); } #[no_mangle] fn log_debug_rust(msg: *const c_char) { let msg = unsafe { CStr::from_ptr(msg) }.to_string_lossy(); debug!("{}", msg); } #[no_mangle] fn log_info_rust(msg: *const c_char) { let msg = unsafe { CStr::from_ptr(msg) }.to_string_lossy(); info!("{}", msg); } #[no_mangle] fn log_trace_rust(msg: *const c_char) { let msg = unsafe { CStr::from_ptr(msg) }.to_string_lossy(); trace!("{}", msg); } #[no_mangle] fn log_warn_rust(msg: *const c_char) { let msg = unsafe { CStr::from_ptr(msg) }.to_string_lossy(); warn!("{}", msg); } ================================================ FILE: src/main.rs ================================================ #![cfg_attr(feature = "bench", feature(test))] #[cfg(feature = "bench")] extern crate test; #[macro_use] extern crate bitflags; use clap::CommandFactory; use clap_complete::generate; #[cfg(unix)] use signal_hook::iterator::Signals; #[cfg(unix)] use signal_hook::{consts::TERM_SIGNALS, low_level::signal_name}; use tracing::{error, info, warn}; use std::sync::mpsc; use config::{get_config, Config}; mod capturable; mod cerror; mod config; mod gui; mod input; mod log; mod protocol; mod video; mod web; mod websocket; mod weylus; fn main() { let (sender, receiver) = mpsc::sync_channel::(100); log::setup_logging(sender); let conf = get_config(); if let Some(shell) = conf.completions { generate( shell, &mut Config::command(), "weylus", &mut std::io::stdout(), ); return; } if conf.print_index_html { print!("{}", web::INDEX_HTML); return; } if conf.print_access_html { print!("{}", web::ACCESS_HTML); return; } if conf.print_style_css { print!("{}", web::STYLE_CSS); return; } if conf.print_lib_js { print!("{}", web::LIB_JS); return; } #[cfg(target_os = "linux")] { // make sure XInitThreads is called before any threading is done crate::capturable::x11::x11_init(); if let Err(err) = gstreamer::init() { error!( "Failed to initialize gstreamer, screen capturing will most likely not work \ on Wayland: {}", err ); } } if conf.no_gui { let mut weylus = crate::weylus::Weylus::new(); weylus.start(&conf, |msg| match msg { web::Web2UiMessage::UInputInaccessible => { warn!(std::include_str!("strings/uinput_error.txt")) } }); #[cfg(unix)] { let mut signals = Signals::new(TERM_SIGNALS).unwrap(); for sig in signals.forever() { info!( "Shutting down after receiving signal {signame} ({sig})...", signame = signal_name(sig).unwrap_or("UNKNOWN SIGNAL") ); std::thread::spawn(move || { for sig in signals.forever() { warn!( "Received second signal {signame} ({sig}) while shutting down \ gracefully, proceeding with forceful shutdown...", signame = signal_name(sig).unwrap_or("UNKNOWN SIGNAL") ); std::process::exit(1); } }); weylus.stop(); break; } } #[cfg(not(unix))] { loop { std::thread::park(); } } } else { gui::run(&conf, receiver); } } #[cfg(feature = "bench")] #[cfg(test)] mod tests { use super::*; use capturable::{Capturable, Recorder}; use test::Bencher; #[cfg(target_os = "linux")] #[bench] fn bench_capture_x11(b: &mut Bencher) { let mut x11ctx = capturable::x11::X11Context::new().unwrap(); let root = x11ctx.capturables().unwrap().remove(0); let mut r = root.recorder(false).unwrap(); b.iter(|| { r.capture().unwrap(); }); } #[cfg(target_os = "linux")] #[bench] fn bench_video_x11(b: &mut Bencher) { let mut x11ctx = capturable::x11::X11Context::new().unwrap(); let root = x11ctx.capturables().unwrap().remove(0); let mut r = root.recorder(false).unwrap(); let (width, height) = r.capture().unwrap().size(); let opts = video::EncoderOptions { try_vaapi: true, try_nvenc: true, try_videotoolbox: false, try_mediafoundation: false, }; let mut encoder = video::VideoEncoder::new(width, height, width, height, |_| {}, opts).unwrap(); b.iter(|| encoder.encode(r.capture().unwrap())); } #[cfg(target_os = "linux")] #[bench] fn bench_capture_wayland(b: &mut Bencher) { gstreamer::init().unwrap(); let root = capturable::pipewire::get_capturables(false) .unwrap() .remove(0); let mut r = root.recorder(false).unwrap(); let _ = r.capture(); b.iter(|| { r.capture().unwrap(); }); } #[cfg(target_os = "linux")] #[bench] fn bench_video_wayland(b: &mut Bencher) { gstreamer::init().unwrap(); let root = capturable::pipewire::get_capturables(false) .unwrap() .remove(0); let mut r = root.recorder(false).unwrap(); let (width, height) = r.capture().unwrap().size(); let opts = video::EncoderOptions { try_vaapi: true, try_nvenc: true, try_videotoolbox: false, try_mediafoundation: false, }; let mut encoder = video::VideoEncoder::new(width, height, width, height, |_| {}, opts).unwrap(); b.iter(|| encoder.encode(r.capture().unwrap())); } #[cfg(target_os = "linux")] #[bench] fn bench_video_vaapi(b: &mut Bencher) { const WIDTH: usize = 1920; const HEIGHT: usize = 1080; const N: usize = 60; let mut bufs = vec![vec![0u8; SIZE]; N]; for i in 0..N { for j in 0..SIZE { bufs[i][j] = ((i * SIZE + j) % 256) as u8; } } let opts = video::EncoderOptions { try_vaapi: true, try_nvenc: false, try_videotoolbox: false, try_mediafoundation: false, }; let mut encoder = video::VideoEncoder::new(WIDTH, HEIGHT, WIDTH, HEIGHT, |_| {}, opts).unwrap(); const SIZE: usize = WIDTH * HEIGHT * 4; let mut i = 0; b.iter(|| { encoder.encode(video::PixelProvider::BGR0(WIDTH, HEIGHT, &bufs[i % N])); i += 1; }); } #[cfg(target_os = "linux")] #[bench] fn bench_video_x264(b: &mut Bencher) { const WIDTH: usize = 1920; const HEIGHT: usize = 1080; const N: usize = 60; let mut bufs = vec![vec![0u8; SIZE]; N]; for i in 0..N { for j in 0..SIZE { bufs[i][j] = ((i * SIZE + j) % 256) as u8; } } let opts = video::EncoderOptions { try_vaapi: false, try_nvenc: false, try_videotoolbox: false, try_mediafoundation: false, }; let mut encoder = video::VideoEncoder::new(WIDTH, HEIGHT, WIDTH, HEIGHT, |_| {}, opts).unwrap(); const SIZE: usize = WIDTH * HEIGHT * 4; let mut i = 0; b.iter(|| { encoder.encode(video::PixelProvider::BGR0(WIDTH, HEIGHT, &bufs[i % N])); i += 1; }); } #[cfg(target_os = "linux")] #[bench] fn bench_video_nvenc(b: &mut Bencher) { const WIDTH: usize = 1920; const HEIGHT: usize = 1080; const N: usize = 60; let mut bufs = vec![vec![0u8; SIZE]; N]; for i in 0..N { for j in 0..SIZE { bufs[i][j] = ((i * SIZE + j) % 256) as u8; } } let opts = video::EncoderOptions { try_vaapi: false, try_nvenc: true, try_videotoolbox: false, try_mediafoundation: false, }; let mut encoder = video::VideoEncoder::new(WIDTH, HEIGHT, WIDTH, HEIGHT, |_| {}, opts).unwrap(); const SIZE: usize = WIDTH * HEIGHT * 4; let mut i = 0; b.iter(|| { encoder.encode(video::PixelProvider::BGR0(WIDTH, HEIGHT, &bufs[i % N])); i += 1; }); } } ================================================ FILE: src/protocol.rs ================================================ use serde::{Deserialize, Deserializer, Serialize}; #[derive(Serialize, Deserialize, Debug)] pub struct ClientConfiguration { #[cfg(target_os = "linux")] pub uinput_support: bool, pub capturable_id: usize, pub capture_cursor: bool, pub max_width: usize, pub max_height: usize, pub client_name: Option, pub frame_rate: f64, } #[derive(Serialize, Deserialize, Debug)] pub enum MessageInbound { PointerEvent(PointerEvent), WheelEvent(WheelEvent), KeyboardEvent(KeyboardEvent), GetCapturableList, Config(ClientConfiguration), PauseVideo, ResumeVideo, RestartVideo, ChooseCustomInputAreas, } #[derive(Serialize, Deserialize, Debug)] pub enum MessageOutbound { CapturableList(Vec), NewVideo, ConfigOk, CustomInputAreas(CustomInputAreas), ConfigError(String), Error(String), } #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq)] pub struct Rect { pub x: f64, pub y: f64, pub w: f64, pub h: f64, } impl Default for Rect { fn default() -> Self { Self { x: 0.0, y: 0.0, w: 1.0, h: 1.0, } } } #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Default)] pub struct CustomInputAreas { pub mouse: Option, pub touch: Option, pub pen: Option, } #[derive(Serialize, Deserialize, Debug)] pub enum PointerType { #[serde(rename = "")] Unknown, #[serde(rename = "mouse")] Mouse, #[serde(rename = "pen")] Pen, #[serde(rename = "touch")] Touch, } #[derive(Serialize, Deserialize, Debug)] pub enum PointerEventType { #[serde(rename = "pointerdown")] DOWN, #[serde(rename = "pointerup")] UP, #[serde(rename = "pointercancel")] CANCEL, #[serde(rename = "pointermove")] MOVE, #[serde(rename = "pointerover")] OVER, #[serde(rename = "pointerenter")] ENTER, #[serde(rename = "pointerleave")] LEAVE, #[serde(rename = "pointerout")] OUT, } #[derive(Serialize, Deserialize, Debug)] pub enum KeyboardEventType { #[serde(rename = "down")] DOWN, #[serde(rename = "up")] UP, #[serde(rename = "repeat")] REPEAT, } #[derive(Serialize, Deserialize, Debug)] pub enum KeyboardLocation { STANDARD, LEFT, RIGHT, NUMPAD, } fn location_from<'de, D: Deserializer<'de>>(deserializer: D) -> Result { let code: u8 = Deserialize::deserialize(deserializer)?; match code { 0 => Ok(KeyboardLocation::STANDARD), 1 => Ok(KeyboardLocation::LEFT), 2 => Ok(KeyboardLocation::RIGHT), 3 => Ok(KeyboardLocation::NUMPAD), _ => Err(serde::de::Error::custom( "Failed to parse keyboard location code.", )), } } bitflags! { #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, Copy)] pub struct Button: u8 { const NONE = 0b0000_0000; const PRIMARY = 0b0000_0001; const SECONDARY = 0b0000_0010; const AUXILARY = 0b0000_0100; const FOURTH = 0b0000_1000; const FIFTH = 0b0001_0000; const ERASER = 0b0010_0000; } } fn button_from<'de, D: Deserializer<'de>>(deserializer: D) -> Result { let bits: u8 = Deserialize::deserialize(deserializer)?; Button::from_bits(bits).map_or( Err(serde::de::Error::custom("Failed to parse button code.")), Ok, ) } #[derive(Serialize, Deserialize, Debug)] pub struct KeyboardEvent { pub event_type: KeyboardEventType, pub code: String, pub key: String, #[serde(deserialize_with = "location_from")] pub location: KeyboardLocation, pub alt: bool, pub ctrl: bool, pub shift: bool, pub meta: bool, } #[derive(Serialize, Deserialize, Debug)] pub struct PointerEvent { pub event_type: PointerEventType, pub pointer_id: i64, pub timestamp: u64, pub is_primary: bool, pub pointer_type: PointerType, #[serde(deserialize_with = "button_from")] pub button: Button, #[serde(deserialize_with = "button_from")] pub buttons: Button, pub x: f64, pub y: f64, // pub movement_x: f64, // pub movement_y: f64, pub pressure: f64, pub tilt_x: i32, pub tilt_y: i32, pub twist: i32, pub width: f64, pub height: f64, } #[derive(Serialize, Deserialize, Debug)] pub struct WheelEvent { pub dx: i32, pub dy: i32, pub timestamp: u64, } pub trait WeylusSender { type Error: std::error::Error; fn send_message(&mut self, message: MessageOutbound) -> Result<(), Self::Error>; fn send_video(&mut self, bytes: &[u8]) -> Result<(), Self::Error>; } pub trait WeylusReceiver: Iterator> { type Error: std::error::Error; } ================================================ FILE: src/strings/uinput_error.txt ================================================ Weylus uses the uinput interface to simulate input events on Linux. To enable stylus and multi-touch support /dev/uinput needs to be writable by Weylus. To make /dev/uinput permanently writable by your user, run the following inside a terminal: sudo groupadd -r uinput sudo usermod -aG uinput $USER echo 'KERNEL=="uinput", MODE="0660", GROUP="uinput", OPTIONS+="static_node=uinput"' | sudo tee /etc/udev/rules.d/60-weylus.rules Then, either reboot, or run sudo udevadm control --reload sudo udevadm trigger then log out and log in again. To undo this, run: sudo rm /etc/udev/rules.d/60-weylus.rules This allows your user to synthesize input events system-wide, even when another user is logged in. Therefore, untrusted users should not be added to the uinput group. If you do not want to make use of this feature uncheck "Enable uinput" in your browser. Like this Weylus will only simulate a mouse. ================================================ FILE: src/video.rs ================================================ use std::os::raw::{c_int, c_uchar, c_void}; use std::time::Instant; use tracing::warn; use crate::cerror::CError; extern "C" { fn init_video_encoder( rust_ctx: *mut c_void, width_in: c_int, height_in: c_int, width_out: c_int, height_out: c_int, try_vaapi: c_int, try_nvenc: c_int, try_videotoolbox: c_int, try_mediafoundation: c_int, ) -> *mut c_void; fn open_video(handle: *mut c_void, err: *mut CError); fn destroy_video_encoder(handle: *mut c_void); fn encode_video_frame(handle: *mut c_void, micros: c_int, err: *mut CError); fn fill_rgb(ctx: *mut c_void, data: *const u8, err: *mut CError); fn fill_rgb0(ctx: *mut c_void, data: *const u8, err: *mut CError); fn fill_bgr0(ctx: *mut c_void, data: *const u8, stride: c_int, err: *mut CError); } // this is used as callback in lib/encode_video.c via ffmpegs AVIOContext #[no_mangle] fn write_video_packet(video_encoder: *mut c_void, buf: *const c_uchar, buf_size: c_int) -> c_int { let video_encoder = unsafe { (video_encoder as *mut VideoEncoder).as_mut().unwrap() }; (video_encoder.write_data)(unsafe { std::slice::from_raw_parts(buf as *const u8, buf_size as usize) }); 0 } pub enum PixelProvider<'a> { // 8 bits per color RGB(usize, usize, &'a [u8]), RGB0(usize, usize, &'a [u8]), BGR0(usize, usize, &'a [u8]), // width, height, stride BGR0S(usize, usize, usize, &'a [u8]), } impl<'a> PixelProvider<'a> { pub fn size(&self) -> (usize, usize) { match self { PixelProvider::RGB(w, h, _) => (*w, *h), PixelProvider::RGB0(w, h, _) => (*w, *h), PixelProvider::BGR0(w, h, _) => (*w, *h), PixelProvider::BGR0S(w, h, _, _) => (*w, *h), } } } #[derive(Clone, Copy)] pub struct EncoderOptions { pub try_vaapi: bool, pub try_nvenc: bool, pub try_videotoolbox: bool, pub try_mediafoundation: bool, } pub struct VideoEncoder { handle: *mut c_void, width_in: usize, height_in: usize, width_out: usize, height_out: usize, write_data: Box, start_time: Instant, } impl VideoEncoder { pub fn new( width_in: usize, height_in: usize, width_out: usize, height_out: usize, mut write_data: impl FnMut(&[u8]) + 'static, options: EncoderOptions, ) -> Result, CError> { let mut video_encoder = Box::new(Self { handle: std::ptr::null_mut(), width_in, height_in, width_out, height_out, write_data: Box::new(move |data| write_data(data)), start_time: Instant::now(), }); let handle = unsafe { init_video_encoder( video_encoder.as_mut() as *mut _ as *mut c_void, width_in as c_int, height_in as c_int, width_out as c_int, height_out as c_int, options.try_vaapi.into(), options.try_nvenc.into(), options.try_videotoolbox.into(), options.try_mediafoundation.into(), ) }; video_encoder.handle = handle; let mut err = CError::new(); unsafe { open_video(video_encoder.handle, &mut err) }; if err.is_err() { return Err(err); } Ok(video_encoder) } pub fn encode(&mut self, pixel_provider: PixelProvider) { let mut err = CError::new(); match pixel_provider { PixelProvider::BGR0(w, _, bgr0) => unsafe { fill_bgr0(self.handle, bgr0.as_ptr(), (w * 4) as c_int, &mut err); }, PixelProvider::BGR0S(_, _, stride, bgr0) => unsafe { fill_bgr0(self.handle, bgr0.as_ptr(), stride as c_int, &mut err); }, PixelProvider::RGB(_, _, rgb) => unsafe { fill_rgb(self.handle, rgb.as_ptr(), &mut err); }, PixelProvider::RGB0(_, _, rgb) => unsafe { fill_rgb0(self.handle, rgb.as_ptr(), &mut err); }, } if err.is_err() { warn!("Failed to fill video frame: {}", err); return; } unsafe { encode_video_frame( self.handle, (Instant::now() - self.start_time).as_millis() as c_int, &mut err, ); } if err.is_err() { warn!("Failed to encode video frame: {}", err); return; } } pub fn check_size( &self, width_in: usize, height_in: usize, width_out: usize, height_out: usize, ) -> bool { (self.width_in == width_in) && (self.height_in == height_in) && (self.width_out == width_out) && (self.height_out == height_out) } } impl Drop for VideoEncoder { fn drop(&mut self) { if !self.handle.is_null() { unsafe { destroy_video_encoder(self.handle) } } } } ================================================ FILE: src/web.rs ================================================ use bytes::Bytes; use fastwebsockets::upgrade; use handlebars::Handlebars; use http_body_util::combinators::BoxBody; use http_body_util::{BodyExt, Full}; use hyper::body::Incoming; use hyper::server::conn::http1; use hyper::service::service_fn; use hyper::{Method, Request, Response, StatusCode}; use hyper_util::rt::TokioIo; use serde::Serialize; use std::collections::HashMap; use std::convert::Infallible; use std::net::SocketAddr; use std::path::PathBuf; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; use tokio::net::TcpListener; use tokio::sync::{mpsc, oneshot}; use tracing::{debug, error, info, warn}; use crate::websocket::{weylus_websocket_channel, WeylusClientConfig, WeylusClientHandler}; #[derive(Debug)] pub enum WebStartUpMessage { Start, Error, } pub enum Web2UiMessage { UInputInaccessible, } pub const INDEX_HTML: &str = std::include_str!("../www/templates/index.html"); pub const ACCESS_HTML: &str = std::include_str!("../www/static/access_code.html"); pub const STYLE_CSS: &str = std::include_str!("../www/static/style.css"); pub const LIB_JS: &str = std::include_str!("../www/static/lib.js"); #[derive(Serialize)] struct IndexTemplateContext { access_code: Option, uinput_enabled: bool, capture_cursor_enabled: bool, log_level: String, enable_custom_input_areas: bool, } fn response_from_str(s: &str, content_type: &str) -> Response> { Response::builder() .status(StatusCode::OK) .header("content-type", content_type) .body(s.to_string().into()) .unwrap() } fn response_not_found() -> Response> { Response::builder() .status(StatusCode::NOT_FOUND) .header("content-type", "text/html; charset=utf-8") .body("Not found!".into()) .unwrap() } async fn response_from_path_or_default( path: Option<&PathBuf>, default: &str, content_type: &str, ) -> Response> { match path { Some(path) => match tokio::fs::read_to_string(path).await { Ok(s) => response_from_str(&s, content_type), Err(err) => { warn!("Failed to load file: {}", err); response_from_str(default, content_type) } }, None => response_from_str(default, content_type), } } async fn serve( addr: SocketAddr, mut req: Request, context: Arc>, sender_ui: mpsc::Sender, num_clients: Arc, semaphore_websocket_shutdown: Arc, notify_disconnect: Arc, ) -> Result>, hyper::Error> { debug!("Got request: {:?}", req); let mut authed = false; if let Some(access_code) = &context.web_config.access_code { if req.method() == Method::GET && (req.uri().path() == "/" || req.uri().path() == "/ws") { use url::form_urlencoded; if let Some(query) = req.uri().query() { let params = form_urlencoded::parse(query.as_bytes()) .into_owned() .collect::>(); if let Some(code) = params.get("access_code") { if code == access_code { authed = true; debug!(address = ?addr, "Web-Client authenticated."); } } } } } else { authed = true; } if req.method() != Method::GET { return Ok(response_not_found().map(|r| r.boxed())); } match req.uri().path() { "/" => { if !authed { return Ok(response_from_path_or_default( context.web_config.custom_access_html.as_ref(), ACCESS_HTML, "text/html; charset=utf-8", ) .await .map(|r| r.boxed())); } let config = IndexTemplateContext { access_code: context.web_config.access_code.clone(), uinput_enabled: cfg!(target_os = "linux"), capture_cursor_enabled: cfg!(not(target_os = "windows")), log_level: crate::log::get_log_level().to_string(), enable_custom_input_areas: context.web_config.enable_custom_input_areas, }; let html = if let Some(path) = context.web_config.custom_index_html.as_ref() { let mut reg = Handlebars::new(); if let Err(err) = reg.register_template_file("index", path) { warn!("Failed to register template from path: {}", err); context.templates.render("index", &config) } else { reg.render("index", &config) } } else { context.templates.render("index", &config) }; match html { Ok(html) => { Ok(response_from_str(&html, "text/html; charset=utf-8").map(|r| r.boxed())) } Err(err) => { error!("Failed to render index template: {}", err); Ok(response_not_found().map(|r| r.boxed())) } } } "/ws" => { if !authed { return Ok(Response::builder() .status(StatusCode::UNAUTHORIZED) .body("unauthorized".to_string().boxed()) .unwrap()); } let (response, fut) = upgrade::upgrade(&mut req).unwrap(); num_clients.fetch_add(1, Ordering::Relaxed); let config = context.weylus_client_config.clone(); tokio::spawn(async move { match fut.await { Ok(ws) => { let (sender, receiver) = weylus_websocket_channel(ws, semaphore_websocket_shutdown); std::thread::spawn(move || { let client = WeylusClientHandler::new( sender, receiver, || { if let Err(err) = sender_ui.blocking_send(Web2UiMessage::UInputInaccessible) { warn!( "Failed to send message 'UInputInaccessible': {err}." ); } }, config, ); client.run(); num_clients.fetch_sub(1, Ordering::Relaxed); notify_disconnect.notify_waiters(); }); } Err(err) => { eprintln!("Error in websocket connection: {}", err); num_clients.fetch_sub(1, Ordering::Relaxed); notify_disconnect.notify_waiters(); } } }); Ok(response.map(|r| r.boxed())) } "/style.css" => Ok(response_from_path_or_default( context.web_config.custom_style_css.as_ref(), STYLE_CSS, "text/css; charset=utf-8", ) .await .map(|r| r.boxed())), "/lib.js" => Ok(response_from_path_or_default( context.web_config.custom_lib_js.as_ref(), LIB_JS, "text/javascript; charset=utf-8", ) .await .map(|r| r.boxed())), _ => Ok(response_not_found().map(|r| r.boxed())), } } #[derive(Clone)] pub struct WebServerConfig { pub bind_addr: SocketAddr, pub access_code: Option, pub custom_index_html: Option, pub custom_access_html: Option, pub custom_style_css: Option, pub custom_lib_js: Option, pub enable_custom_input_areas: bool, } struct Context<'a> { web_config: WebServerConfig, weylus_client_config: WeylusClientConfig, templates: Handlebars<'a>, } pub fn run( sender_ui: tokio::sync::mpsc::Sender, sender_startup: oneshot::Sender, notify_shutdown: Arc, web_server_config: WebServerConfig, weylus_client_config: WeylusClientConfig, ) -> std::thread::JoinHandle<()> { let mut templates = Handlebars::new(); templates .register_template_string("index", INDEX_HTML) .unwrap(); let context = Context { web_config: web_server_config, weylus_client_config, templates, }; std::thread::spawn(move || run_server(context, sender_ui, sender_startup, notify_shutdown)) } #[tokio::main] async fn run_server( context: Context<'static>, sender_ui: tokio::sync::mpsc::Sender, sender_startup: oneshot::Sender, notify_shutdown: Arc, ) { let addr = context.web_config.bind_addr; let listener = match TcpListener::bind(addr).await { Ok(listener) => listener, Err(err) => { error!("Failed to bind to socket: {err}."); sender_startup.send(WebStartUpMessage::Error).unwrap(); return; } }; sender_startup.send(WebStartUpMessage::Start).unwrap(); let context = Arc::new(context); let broadcast_shutdown = Arc::new(tokio::sync::Notify::new()); let num_clients = Arc::new(AtomicUsize::new(0)); let notify_disconnect = Arc::new(tokio::sync::Notify::new()); let semaphore_websocket_shutdown = Arc::new(tokio::sync::Semaphore::new(0)); loop { let (tcp, remote_address) = tokio::select! { res = listener.accept() => { match res { Ok(conn) => conn, Err(err) => { warn!("Connection failed: {err}."); continue; } } }, _ = notify_shutdown.notified() => { info!("Webserver is shutting down."); broadcast_shutdown.notify_waiters(); break; } }; debug!(address = ?remote_address, "Client connected."); let io = TokioIo::new(tcp); let sender_ui = sender_ui.clone(); let broadcast_shutdown = broadcast_shutdown.clone(); let context = context.clone(); let num_clients = num_clients.clone(); let semaphore_websocket_shutdown = semaphore_websocket_shutdown.clone(); let notify_disconnect = notify_disconnect.clone(); tokio::task::spawn(async move { let conn = http1::Builder::new().serve_connection( io, service_fn({ move |req| { let context = context.clone(); let num_clients = num_clients.clone(); let semaphore_websocket_shutdown = semaphore_websocket_shutdown.clone(); let notify_disconnect = notify_disconnect.clone(); serve( remote_address, req, context, sender_ui.clone(), num_clients, semaphore_websocket_shutdown, notify_disconnect, ) } }), ); let conn = conn.with_upgrades(); tokio::select! { conn = conn => match conn { Ok(_) => (), Err(err) => { warn!("Error polling connection ({remote_address}): {err}.") } }, _ = broadcast_shutdown.notified() => { info!("Closing connection to: {remote_address}."); } } }); } semaphore_websocket_shutdown.add_permits(num_clients.load(Ordering::Relaxed)); loop { let remaining_clients = num_clients.load(Ordering::Relaxed); if remaining_clients == 0 { break; } else { debug!("Waiting for remaining clients ({remaining_clients}) to disconnect."); } tokio::select! { _ = notify_disconnect.notified() => (), _ = tokio::time::sleep(Duration::from_secs(1)) => { semaphore_websocket_shutdown.add_permits(num_clients.load(Ordering::Relaxed)); }, } } } ================================================ FILE: src/websocket.rs ================================================ use fastwebsockets::{FragmentCollectorRead, Frame, OpCode, WebSocket, WebSocketError}; use hyper::upgrade::Upgraded; use hyper_util::rt::TokioIo; use std::convert::Infallible; use std::sync::mpsc::RecvTimeoutError; use std::sync::{mpsc, Arc}; use std::thread::{spawn, JoinHandle}; use std::time::{Duration, Instant}; use tokio::sync::mpsc::channel; use tracing::{error, trace, warn}; use crate::capturable::{get_capturables, Capturable, Recorder}; use crate::input::device::{InputDevice, InputDeviceType}; use crate::protocol::{ ClientConfiguration, KeyboardEvent, MessageInbound, MessageOutbound, PointerEvent, WeylusReceiver, WeylusSender, WheelEvent, }; use crate::cerror::CErrorCode; use crate::video::{EncoderOptions, VideoEncoder}; struct VideoConfig { capturable: Box, capture_cursor: bool, max_width: usize, max_height: usize, frame_rate: f64, } enum VideoCommands { Start(VideoConfig), Pause, Resume, Restart, } fn send_message(sender: &mut S, message: MessageOutbound) where S: WeylusSender, { if let Err(err) = sender.send_message(message) { warn!("Failed to send message to client: {err}"); } } pub struct WeylusClientHandler { sender: S, receiver: Option, video_sender: mpsc::Sender, input_device: Option>, capturables: Vec>, on_uinput_inaccessible: FnUInput, config: WeylusClientConfig, #[cfg(target_os = "linux")] capture_cursor: bool, client_name: Option, video_thread: JoinHandle<()>, } #[derive(Clone, Copy)] pub struct WeylusClientConfig { pub encoder_options: EncoderOptions, #[cfg(target_os = "linux")] pub wayland_support: bool, pub no_gui: bool, } impl WeylusClientHandler { pub fn new( sender: S, receiver: R, on_uinput_inaccessible: FnUInput, config: WeylusClientConfig, ) -> Self where R: WeylusReceiver, S: WeylusSender + Clone + Send + Sync + 'static, { let (video_sender, video_receiver) = mpsc::channel::(); let video_thread = { let sender = sender.clone(); // offload creating the videostream to another thread to avoid blocking the thread that // is receiving messages from the websocket spawn(move || handle_video(video_receiver, sender, config.encoder_options)) }; Self { sender, receiver: Some(receiver), video_sender, input_device: None, capturables: vec![], on_uinput_inaccessible, config, #[cfg(target_os = "linux")] capture_cursor: false, client_name: None, video_thread, } } pub fn run(mut self) where R: WeylusReceiver, S: WeylusSender + Clone + Send + Sync + 'static, FnUInput: Fn(), { for message in self.receiver.take().unwrap() { match message { Ok(message) => { trace!("Received message: {message:?}"); match message { MessageInbound::PointerEvent(event) => self.process_pointer_event(&event), MessageInbound::WheelEvent(event) => self.process_wheel_event(&event), MessageInbound::KeyboardEvent(event) => self.process_keyboard_event(&event), MessageInbound::GetCapturableList => self.send_capturable_list(), MessageInbound::Config(config) => self.update_config(config), MessageInbound::PauseVideo => { self.video_sender.send(VideoCommands::Pause).unwrap() } MessageInbound::ResumeVideo => { self.video_sender.send(VideoCommands::Resume).unwrap() } MessageInbound::RestartVideo => { self.video_sender.send(VideoCommands::Restart).unwrap() } MessageInbound::ChooseCustomInputAreas => { let (sender, receiver) = std::sync::mpsc::channel(); crate::gui::get_input_area(self.config.no_gui, sender); let mut sender = self.sender.clone(); spawn(move || { while let Ok(areas) = receiver.recv() { send_message( &mut sender, MessageOutbound::CustomInputAreas(areas), ); } }); } } } Err(err) => { warn!("Failed to read message {err}!"); self.send_message(MessageOutbound::Error( "Failed to read message!".to_string(), )); } } } drop(self.video_sender); if let Err(err) = self.video_thread.join() { warn!("Failed to join video thread: {err:?}"); } } fn send_message(&mut self, message: MessageOutbound) where S: WeylusSender, { send_message(&mut self.sender, message) } fn process_wheel_event(&mut self, event: &WheelEvent) { match &mut self.input_device { Some(i) => i.send_wheel_event(event), None => warn!("Input device is not initalized, can not process WheelEvent!"), } } fn process_pointer_event(&mut self, event: &PointerEvent) { if self.input_device.is_some() { self.input_device .as_mut() .unwrap() .send_pointer_event(event) } else { warn!("Input device is not initalized, can not process PointerEvent!"); } } fn process_keyboard_event(&mut self, event: &KeyboardEvent) { if self.input_device.is_some() { self.input_device .as_mut() .unwrap() .send_keyboard_event(event) } else { warn!("Input device is not initalized, can not process KeyboardEvent!"); } } fn send_capturable_list(&mut self) where S: WeylusSender, { let mut windows = Vec::::new(); self.capturables = get_capturables( #[cfg(target_os = "linux")] self.config.wayland_support, #[cfg(target_os = "linux")] self.capture_cursor, ); self.capturables.iter().for_each(|c| { windows.push(c.name()); }); self.send_message(MessageOutbound::CapturableList(windows)); } fn update_config(&mut self, config: ClientConfiguration) where S: WeylusSender, FnUInput: Fn(), { let client_name_changed = if self.client_name != config.client_name { self.client_name = config.client_name; true } else { false }; if config.capturable_id < self.capturables.len() { let capturable = self.capturables[config.capturable_id].clone(); #[cfg(target_os = "linux")] { self.capture_cursor = config.capture_cursor; } #[cfg(target_os = "linux")] if config.uinput_support { if self.input_device.as_ref().map_or(true, |d| { client_name_changed || d.device_type() != InputDeviceType::UInputDevice }) { let device = crate::input::uinput_device::UInputDevice::new( capturable.clone(), &self.client_name, ); match device { Ok(d) => self.input_device = Some(Box::new(d)), Err(e) => { error!("Failed to create uinput device: {}", e); if let CErrorCode::UInputNotAccessible = e.to_enum() { (self.on_uinput_inaccessible)(); } self.send_message(MessageOutbound::ConfigError( "Failed to create uinput device!".to_string(), )); return; } } } else if let Some(d) = self.input_device.as_mut() { d.set_capturable(capturable.clone()); } } else if self.input_device.as_ref().map_or(true, |d| { d.device_type() != InputDeviceType::AutoPilotDevice }) { self.input_device = Some(Box::new( crate::input::autopilot_device::AutoPilotDevice::new(capturable.clone()), )); } else if let Some(d) = self.input_device.as_mut() { d.set_capturable(capturable.clone()); } #[cfg(target_os = "macos")] if self.input_device.is_none() { self.input_device = Some(Box::new( crate::input::autopilot_device::AutoPilotDevice::new(capturable.clone()), )); } else { self.input_device .as_mut() .map(|d| d.set_capturable(capturable.clone())); } #[cfg(target_os = "windows")] if self.input_device.is_none() { self.input_device = Some(Box::new( crate::input::autopilot_device_win::WindowsInput::new(capturable.clone()), )); } else { self.input_device .as_mut() .map(|d| d.set_capturable(capturable.clone())); } self.video_sender .send(VideoCommands::Start(VideoConfig { capturable, capture_cursor: config.capture_cursor, max_width: config.max_width, max_height: config.max_height, frame_rate: config.frame_rate, })) .unwrap(); } else { error!("Got invalid id for capturable: {}", config.capturable_id); self.send_message(MessageOutbound::ConfigError( "Invalid id for capturable!".to_string(), )); } } } fn handle_video( receiver: mpsc::Receiver, mut sender: S, encoder_options: EncoderOptions, ) { const EFFECTIVE_INIFINITY: Duration = Duration::from_secs(3600 * 24 * 365 * 200); let mut recorder: Option> = None; let mut video_encoder: Option> = None; let mut max_width = 1920; let mut max_height = 1080; let mut frame_duration = EFFECTIVE_INIFINITY; let mut last_frame = Instant::now(); let mut paused = false; loop { let now = Instant::now(); let elapsed = now - last_frame; let frames_passed = (elapsed.as_secs_f64() / frame_duration.as_secs_f64()) as u32; let next_frame = last_frame + (frames_passed + 1) * frame_duration; let timeout = next_frame - now; last_frame = next_frame; if frames_passed > 0 { trace!("Dropped {frames_passed} frame(s)!"); } match receiver.recv_timeout(if paused { EFFECTIVE_INIFINITY } else { timeout }) { Ok(VideoCommands::Start(config)) => { #[allow(unused_assignments)] { // gstpipewire can not handle setting a pipeline's state to Null after another // pipeline has been created and its state has been set to Play. // This line makes sure that there always is only a single recorder and thus // single pipeline in this thread by forcing rust to call the destructor of the // current pipeline here, right before creating a new pipeline. // See: https://gitlab.freedesktop.org/pipewire/pipewire/-/issues/986 // // This shouldn't affect other Recorder trait objects. recorder = None; } match config.capturable.recorder(config.capture_cursor) { Ok(r) => { recorder = Some(r); max_width = config.max_width; max_height = config.max_height; send_message(&mut sender, MessageOutbound::ConfigOk); } Err(err) => { warn!("Failed to init screen cast: {}!", err); send_message( &mut sender, MessageOutbound::Error("Failed to init screen cast!".into()), ) } } last_frame = Instant::now(); // The Duration type can not handle infinity, if the frame rate is set to 0 we just // set the duration between two frames to a very long one, which is effectively // infinity. let d = 1.0 / config.frame_rate; frame_duration = if d.is_finite() { Duration::from_secs_f64(d) } else { EFFECTIVE_INIFINITY }; frame_duration = frame_duration.min(EFFECTIVE_INIFINITY); } Ok(VideoCommands::Pause) => { paused = true; } Ok(VideoCommands::Resume) => { paused = false; } Ok(VideoCommands::Restart) => { video_encoder = None; } Err(RecvTimeoutError::Timeout) => { if recorder.is_none() { warn!("Screen capture not initalized, can not send video frame!"); continue; } let pixel_data = recorder.as_mut().unwrap().capture(); if let Err(err) = pixel_data { warn!("Error capturing screen: {}", err); continue; } let pixel_data = pixel_data.unwrap(); let (width_in, height_in) = pixel_data.size(); let scale = (max_width as f64 / width_in as f64).min(max_height as f64 / height_in as f64); // limit video to 4K let scale_max = (3840.0 / width_in as f64).min(2160.0 / height_in as f64); let scale = scale.min(scale_max); let mut width_out = width_in; let mut height_out = height_in; if scale < 1.0 { width_out = (width_out as f64 * scale) as usize; height_out = (height_out as f64 * scale) as usize; } // video encoder is not setup or setup for encoding the wrong size: restart it if video_encoder.is_none() || !video_encoder .as_ref() .unwrap() .check_size(width_in, height_in, width_out, height_out) { send_message(&mut sender, MessageOutbound::NewVideo); let mut sender = sender.clone(); let res = VideoEncoder::new( width_in, height_in, width_out, height_out, move |data| { if let Err(err) = sender.send_video(data) { warn!("Failed to send video frame: {err}!"); } }, encoder_options, ); match res { Ok(r) => video_encoder = Some(r), Err(e) => { warn!("{}", e); continue; } }; } let video_encoder = video_encoder.as_mut().unwrap(); video_encoder.encode(pixel_data); } // stop thread once the channel is closed Err(RecvTimeoutError::Disconnected) => return, }; } } pub struct WsWeylusReceiver { recv: tokio::sync::mpsc::Receiver, } impl Iterator for WsWeylusReceiver { type Item = Result; fn next(&mut self) -> Option { self.recv.blocking_recv().map(Ok) } } impl WeylusReceiver for WsWeylusReceiver { type Error = Infallible; } pub enum WsMessage { Frame(Frame<'static>), Video(Vec), MessageOutbound(MessageOutbound), } unsafe impl Send for WsMessage {} #[derive(Clone)] pub struct WsWeylusSender { sender: tokio::sync::mpsc::Sender, } impl WeylusSender for WsWeylusSender { type Error = tokio::sync::mpsc::error::SendError; fn send_message(&mut self, message: MessageOutbound) -> Result<(), Self::Error> { self.sender .blocking_send(WsMessage::MessageOutbound(message)) } fn send_video(&mut self, bytes: &[u8]) -> Result<(), Self::Error> { self.sender.blocking_send(WsMessage::Video(bytes.to_vec())) } } pub fn weylus_websocket_channel( websocket: WebSocket>, semaphore_shutdown: Arc, ) -> (WsWeylusSender, WsWeylusReceiver) { let (rx, mut tx) = websocket.split(|ws| tokio::io::split(ws)); let mut rx = FragmentCollectorRead::new(rx); let (sender_inbound, receiver_inbound) = channel::(32); let (sender_outbound, mut receiver_outbound) = channel::(32); { let sender_outbound = sender_outbound.clone(); tokio::spawn(async move { let mut send_fn = |frame| async { if let Err(err) = sender_outbound.send(WsMessage::Frame(frame)).await { warn!("Failed to send websocket frame while receiving fragmented frame: {err}.") }; Ok(()) }; loop { let fut = rx.read_frame::<_, WebSocketError>(&mut send_fn); let frame = tokio::select! { _ = semaphore_shutdown.acquire() => break, frame = fut => match frame { Ok(frame) => frame, Err(err) => { warn!("Invalid websocket frame: {err}."); break; }, }, }; match frame.opcode { OpCode::Close => break, OpCode::Text => match serde_json::from_slice(&frame.payload) { Ok(msg) => { if let Err(err) = sender_inbound.send(msg).await { warn!("Failed to forward inbound message to WeylusClientHandler: {err}."); } } Err(err) => warn!("Failed to parse message: {err}"), }, _ => {} } } }); } tokio::spawn(async move { loop { let msg = if let Some(msg) = receiver_outbound.recv().await { msg } else { break; }; match msg { WsMessage::Frame(frame) => { if let Err(err) = tx.write_frame(frame).await { if let WebSocketError::ConnectionClosed = err { break; } warn!("Failed to send frame: {err}"); } } WsMessage::Video(data) => { if let Err(err) = tx.write_frame(Frame::binary(data.into())).await { if let WebSocketError::ConnectionClosed = err { break; } warn!("Failed to send video frame: {err}"); } } WsMessage::MessageOutbound(msg) => { let json_string = serde_json::to_string(&msg).unwrap(); let data = json_string.as_bytes(); if let Err(err) = tx.write_frame(Frame::text(data.into())).await { if let WebSocketError::ConnectionClosed = err { break; } warn!("Failed to send outbound message: {err}"); } } } } }); ( WsWeylusSender { sender: sender_outbound, }, WsWeylusReceiver { recv: receiver_inbound, }, ) } ================================================ FILE: src/weylus.rs ================================================ use std::net::SocketAddr; use std::sync::Arc; use tracing::error; use crate::config::Config; use crate::video::EncoderOptions; use crate::web::{Web2UiMessage, WebServerConfig, WebStartUpMessage}; use crate::websocket::WeylusClientConfig; pub struct Weylus { notify_shutdown: Arc, web_thread: Option>, } impl Weylus { pub fn new() -> Self { Self { notify_shutdown: Arc::new(tokio::sync::Notify::new()), web_thread: None, } } pub fn start( &mut self, config: &Config, mut on_web_message: impl FnMut(Web2UiMessage) + Send + 'static, ) -> bool { let encoder_options = EncoderOptions { #[cfg(target_os = "linux")] try_vaapi: config.try_vaapi, #[cfg(not(target_os = "linux"))] try_vaapi: false, #[cfg(any(target_os = "linux", target_os = "windows"))] try_nvenc: config.try_nvenc, #[cfg(not(any(target_os = "linux", target_os = "windows")))] try_nvenc: false, #[cfg(target_os = "macos")] try_videotoolbox: config.try_videotoolbox, #[cfg(not(target_os = "macos"))] try_videotoolbox: false, #[cfg(target_os = "windows")] try_mediafoundation: config.try_mediafoundation, #[cfg(not(target_os = "windows"))] try_mediafoundation: false, }; let (sender_ui, mut receiver_ui) = tokio::sync::mpsc::channel(100); let (sender_startup, receiver_startup) = tokio::sync::oneshot::channel(); let web_thread = crate::web::run( sender_ui, sender_startup, self.notify_shutdown.clone(), WebServerConfig { bind_addr: SocketAddr::new(config.bind_address, config.web_port), access_code: config.access_code.clone(), custom_index_html: config.custom_index_html.clone(), custom_access_html: config.custom_access_html.clone(), custom_style_css: config.custom_style_css.clone(), custom_lib_js: config.custom_lib_js.clone(), #[cfg(target_os = "linux")] enable_custom_input_areas: config.wayland_support, #[cfg(not(target_os = "linux"))] enable_custom_input_areas: false, }, WeylusClientConfig { encoder_options, #[cfg(target_os = "linux")] wayland_support: config.wayland_support, no_gui: config.no_gui, }, ); match receiver_startup.blocking_recv() { Ok(WebStartUpMessage::Start) => (), Ok(WebStartUpMessage::Error) => { if web_thread.join().is_err() { error!("Webserver thread panicked."); } return false; } Err(err) => { error!("Error communicating with webserver thread: {}", err); if web_thread.join().is_err() { error!("Webserver thread panicked."); } return false; } } self.web_thread = Some(web_thread); std::thread::spawn(move || { while let Some(msg) = receiver_ui.blocking_recv() { on_web_message(msg); } }); true } pub fn stop(&mut self) { self.notify_shutdown.notify_one(); self.wait(); } fn wait(&mut self) { if let Some(t) = self.web_thread.take() { if t.join().is_err() { error!("Web thread panicked."); } } } } impl Drop for Weylus { fn drop(&mut self) { self.stop(); } } ================================================ FILE: ts/lib.ts ================================================ interface Window { ManagedMediaSource: any; } enum LogLevel { ERROR = 0, WARN, INFO, DEBUG, TRACE, } let log_pre: HTMLPreElement; let log_level: LogLevel = LogLevel.ERROR; let no_log_messages: boolean = true; let fps_out: HTMLOutputElement; let frame_count = 0; let last_fps_calc: number = performance.now(); let check_video: HTMLInputElement; function run(level: string) { window.onload = () => { log_pre = document.getElementById("log") as HTMLPreElement; log_pre.textContent = ""; log_level = LogLevel[level]; fps_out = document.getElementById("fps") as HTMLOutputElement; check_video = document.getElementById("enable_video") as HTMLInputElement; window.addEventListener("error", (e: ErrorEvent | Event | UIEvent) => { if ((e as ErrorEvent).error) { let err = e as ErrorEvent; log(LogLevel.ERROR, err.filename + ":L" + err.lineno + ":" + err.colno + ": " + err.message + " Error object: " + JSON.stringify(err.error)); } else if ((e as Event | UIEvent).target) { let ev = e as Event; let src = (e.target as any).src; if (ev.target instanceof HTMLVideoElement) log(LogLevel.ERROR, "Failed to decode video, try reducing resolution or disabling hardware acceleration and reload the page. Error src: " + src); else log(LogLevel.ERROR, "Failed to obtain resource, target: " + ev.target + " type: " + ev.type + " src: " + src + " Error object: " + JSON.stringify(ev)); } else { log(LogLevel.WARN, "Got unknown event: " + JSON.stringify(e)); } return false; }, true) init(); }; } function log(level: LogLevel, msg: string) { if (level > log_level) return; if (level == LogLevel.TRACE) console.trace(msg); else if (level == LogLevel.DEBUG) console.debug(msg); else if (level == LogLevel.INFO) console.info(msg); else if (level == LogLevel.WARN) console.warn(msg); else if (level == LogLevel.ERROR) console.error(msg); if (no_log_messages) { no_log_messages = false; document.getElementById("log_section").classList.remove("hide"); } log_pre.textContent += LogLevel[level] + ": " + msg + "\n"; } function frame_rate_scale(x: number) { return Math.pow(x / 100, 1.5); } function frame_rate_scale_inv(x: number) { return 100 * Math.pow(x, 2 / 3); } function calc_max_video_resolution(scale: number) { return [ Math.round(scale * window.innerWidth * window.devicePixelRatio), Math.round(scale * window.innerHeight * window.devicePixelRatio) ]; } function fresh_canvas() { let canvas_old = document.getElementById("canvas"); let canvas = document.createElement("canvas"); canvas.id = canvas_old.id; canvas_old.classList.forEach((cls) => canvas.classList.add(cls)); canvas_old.replaceWith(canvas); return canvas; } class Rect { x: number; y: number; w: number; h: number; } class CustomInputAreas { mouse: Rect; touch: Rect; pen: Rect; } class Settings { webSocket: WebSocket; checks: Map; capturable_select: HTMLSelectElement; frame_rate_input: HTMLInputElement; frame_rate_output: HTMLOutputElement; scale_video_input: HTMLInputElement; scale_video_output: HTMLOutputElement; range_min_pressure: HTMLInputElement; check_aggressive_seek: HTMLInputElement; client_name_input: HTMLInputElement; visible: boolean; custom_input_areas: CustomInputAreas; settings: HTMLElement; constructor(webSocket: WebSocket) { this.webSocket = webSocket; this.checks = new Map(); this.capturable_select = document.getElementById("window") as HTMLSelectElement; this.frame_rate_input = document.getElementById("frame_rate") as HTMLInputElement; this.frame_rate_input.min = frame_rate_scale_inv(0).toString(); this.frame_rate_input.max = frame_rate_scale_inv(120).toString(); this.frame_rate_output = this.frame_rate_input.nextElementSibling as HTMLOutputElement; this.scale_video_input = document.getElementById("scale_video") as HTMLInputElement; this.scale_video_output = this.scale_video_input.nextElementSibling as HTMLOutputElement; this.range_min_pressure = document.getElementById("min_pressure") as HTMLInputElement; this.client_name_input = document.getElementById("client_name") as HTMLInputElement; this.frame_rate_input.oninput = () => { this.frame_rate_output.value = Math.round(frame_rate_scale(this.frame_rate_input.valueAsNumber)).toString(); } this.scale_video_input.oninput = () => { let [w, h] = calc_max_video_resolution(this.scale_video_input.valueAsNumber) this.scale_video_output.value = w + "x" + h } this.visible = true; // Settings UI this.settings = document.getElementById("settings"); this.settings.onclick = (e) => e.stopPropagation(); let handle = document.getElementById("handle"); // Settings elements this.settings.querySelectorAll("input[type=checkbox]").forEach( (elem, _key, _parent) => this.checks.set(elem.id, elem as HTMLInputElement) ); this.load_settings(); // event handling // client only handle.onclick = () => { this.toggle() }; this.checks.get("lefty").onchange = (e) => { if ((e.target as HTMLInputElement).checked) this.settings.classList.add("lefty"); else this.settings.classList.remove("lefty"); this.save_settings(); } document.getElementById("vanish").onclick = () => { this.settings.classList.add("vanish"); } this.checks.get("stretch").onchange = () => { stretch_video(); this.save_settings(); }; this.checks.get("enable_debug_overlay").onchange = (e) => { let enabled = (e.target as HTMLInputElement).checked; if (enabled) { debug_overlay.classList.remove("hide"); } else { debug_overlay.classList.add("hide"); } this.save_settings(); }; this.check_aggressive_seek = this.checks.get("aggressive_seeking"); this.check_aggressive_seek.onchange = () => { this.save_settings(); }; this.checks.get("enable_video").onchange = (e) => { let enabled = (e.target as HTMLInputElement).checked; document.getElementById("video").classList.toggle("vanish", !enabled); document.getElementById("canvas").classList.toggle("vanish", enabled); this.save_settings(); if (enabled) { this.webSocket.send('"ResumeVideo"'); } else { this.webSocket.send('"PauseVideo"'); } } let upd_pointer = () => { this.save_settings(); new PointerHandler(this.webSocket); } this.checks.get("enable_mouse").onchange = upd_pointer; this.checks.get("enable_stylus").onchange = upd_pointer; this.checks.get("enable_touch").onchange = upd_pointer; this.checks.get("energysaving").onchange = (e) => { this.save_settings(); this.toggle_energysaving((e.target as HTMLInputElement).checked); }; this.checks.get("enable_custom_input_areas").onchange = () => { this.save_settings(); }; this.frame_rate_input.onchange = () => this.save_settings(); this.range_min_pressure.onchange = () => this.save_settings(); // server let upd_server_config = () => { this.save_settings(); this.send_server_config() }; this.checks.get("uinput_support").onchange = upd_server_config; this.checks.get("capture_cursor").onchange = upd_server_config; this.scale_video_input.onchange = upd_server_config; this.client_name_input.onchange = upd_server_config; this.frame_rate_input.onchange = upd_server_config; document.getElementById("refresh").onclick = () => this.webSocket.send('"GetCapturableList"'); document.getElementById("custom_input_areas").onclick = () => { this.webSocket.send('"ChooseCustomInputAreas"'); }; this.capturable_select.onchange = () => this.send_server_config(); } send_server_config() { let config = new Object(null); config["capturable_id"] = Number(this.capturable_select.value); for (const key of [ "uinput_support", "capture_cursor"]) config[key] = this.checks.get(key).checked; let [w, h] = calc_max_video_resolution(this.scale_video_input.valueAsNumber); config["max_width"] = w; config["max_height"] = h; config["frame_rate"] = frame_rate_scale(this.frame_rate_input.valueAsNumber); if (this.client_name_input.value) config["client_name"] = this.client_name_input.value; this.webSocket.send(JSON.stringify({ "Config": config })); } save_settings() { let settings = Object(null); for (const [key, elem] of this.checks.entries()) settings[key] = elem.checked; settings["frame_rate"] = frame_rate_scale(this.frame_rate_input.valueAsNumber).toString(); settings["scale_video"] = this.scale_video_input.value; settings["min_pressure"] = this.range_min_pressure.value; settings["custom_input_areas"] = this.custom_input_areas; settings["client_name"] = this.client_name_input.value; localStorage.setItem("settings", JSON.stringify(settings)); } load_settings() { let settings_string = localStorage.getItem("settings"); if (settings_string === null) { this.frame_rate_input.value = frame_rate_scale_inv(30).toString(); this.frame_rate_output.value = (30).toString(); let [w, h] = calc_max_video_resolution(this.scale_video_input.valueAsNumber) this.scale_video_output.value = w + "x" + h; return; } try { let settings = JSON.parse(settings_string); for (const [key, elem] of this.checks.entries()) { if (typeof settings[key] === "boolean") elem.checked = settings[key]; } let upd_limit = settings["frame_rate"]; if (upd_limit) this.frame_rate_input.value = frame_rate_scale_inv(upd_limit).toString(); else this.frame_rate_input.value = frame_rate_scale_inv(30).toString(); this.frame_rate_output.value = Math.round(frame_rate_scale(this.frame_rate_input.valueAsNumber)).toString(); let scale_video = settings["scale_video"]; if (scale_video) this.scale_video_input.value = scale_video; let [w, h] = calc_max_video_resolution(this.scale_video_input.valueAsNumber); this.scale_video_output.value = w + "x" + h; let min_pressure = settings["min_pressure"]; if (min_pressure) this.range_min_pressure.value = min_pressure; this.custom_input_areas = settings["custom_input_areas"]; if (this.checks.get("lefty").checked) { this.settings.classList.add("lefty"); } if (!this.checks.get("enable_video").checked || this.checks.get("energysaving").checked) { this.checks.get("enable_video").checked = false; if (this.checks.get("energysaving").checked) this.checks.get("enable_video").disabled = true; document.getElementById("video").classList.add("vanish"); document.getElementById("canvas").classList.remove("vanish"); } if (this.checks.get("energysaving").checked) { this.toggle_energysaving(true); } if (this.checks.get("enable_debug_overlay").checked) { debug_overlay.classList.remove("hide"); } if (document.getElementById("custom_input_areas").classList.contains("hide")) { this.checks.get("enable_custom_input_areas").checked = false; } let client_name = settings["client_name"]; if (client_name) this.client_name_input.value = client_name; } catch { log(LogLevel.DEBUG, "Failed to load settings.") return; } } stretched_video() { return this.checks.get("stretch").checked } pointer_types() { let ptrs = []; if (this.checks.get("enable_mouse").checked) ptrs.push("mouse"); if (this.checks.get("enable_stylus").checked) ptrs.push("pen"); if (this.checks.get("enable_touch").checked) ptrs.push("touch"); return ptrs; } toggle() { this.settings.classList.toggle("hide"); this.visible = !this.visible; } onCapturableList(window_names: string[]) { let current_selection = undefined; if (this.capturable_select.selectedOptions[0]) current_selection = this.capturable_select.selectedOptions[0].textContent; let new_index: number; this.capturable_select.innerText = ""; window_names.forEach((name, i) => { let option = document.createElement("option"); option.value = String(i); option.innerText = name; this.capturable_select.appendChild(option); if (name === current_selection) new_index = i; }); if (new_index !== undefined) this.capturable_select.value = String(new_index); else if (current_selection) // Can't find the window, so don't select anything this.capturable_select.value = ""; } toggle_energysaving(energysaving: boolean) { let canvas = fresh_canvas(); if (energysaving) { let ctx = canvas.getContext("2d"); ctx.fillStyle = "#000"; ctx.fillRect(0, 0, canvas.width, canvas.height); } if (energysaving) { this.checks.get("enable_video").checked = false; this.checks.get("enable_video").disabled = true; this.checks.get("enable_video").dispatchEvent(new Event("change")); } else this.checks.get("enable_video").disabled = false; if (settings) new PointerHandler(this.webSocket); } video_enabled(): boolean { return this.checks.get("enable_video").checked; } } let settings: Settings; let debug_overlay: HTMLElement; let last_pointer_data: Object; class PEvent { event_type: string; pointer_id: number; timestamp: number; is_primary: boolean; pointer_type: string; button: number; buttons: number; x: number; y: number; // movement_x: number; // movement_y: number; pressure: number; tilt_x: number; tilt_y: number; twist: number; width: number; height: number; constructor(eventType: string, event: PointerEvent, targetRect: DOMRect) { let diag_len = Math.sqrt(targetRect.width * targetRect.width + targetRect.height * targetRect.height) this.event_type = eventType.toString(); this.pointer_id = event.pointerId; this.timestamp = Math.round(event.timeStamp * 1000); this.is_primary = event.isPrimary; this.pointer_type = event.pointerType; let btn = event.button; // for some reason the secondary and auxiliary buttons are ordered differently for // the button and buttons properties if (btn == 2) btn = 1; else if (btn == 1) btn = 2; this.button = (btn < 0 ? 0 : 1 << btn); this.buttons = event.buttons; let x_offset = 0; let y_offset = 0; let x_scale = 1; let y_scale = 1; if (settings.checks.get("enable_custom_input_areas").checked) { let custom_input_area: Rect = null; if (event.pointerType == "mouse") { custom_input_area = settings.custom_input_areas.mouse; } else if (event.pointerType == "touch") { custom_input_area = settings.custom_input_areas.touch; } else if (event.pointerType == "pen") { custom_input_area = settings.custom_input_areas.pen; } if (custom_input_area) { x_scale = custom_input_area.w; y_scale = custom_input_area.h; x_offset = custom_input_area.x; y_offset = custom_input_area.y; } } this.x = (event.clientX - targetRect.left) / targetRect.width * x_scale + x_offset; this.y = (event.clientY - targetRect.top) / targetRect.height * y_scale + y_offset; // this.movement_x = event.movementX ? event.movementX : 0; // this.movement_y = event.movementY ? event.movementY : 0; this.pressure = Math.max(event.pressure, settings.range_min_pressure.valueAsNumber); this.tilt_x = event.tiltX; this.tilt_y = event.tiltY; this.width = event.width / diag_len; this.height = event.height / diag_len; this.twist = event.twist; } } class WEvent { dx: number; dy: number; timestamp: number; constructor(event: WheelEvent) { /* The WheelEvent can have different scrolling modes that affect how much scrolling * should be done. Unfortunately there is not always a way to accurately convert the scroll * distance into pixels. Thus the following is a guesstimate and scales the WheelEvent's * deltaX/Y values accordingly. */ let scale = 1; switch (event.deltaMode) { case 0x01: // DOM_DELTA_LINE scale = 10; break; case 0x02: // DOM_DELTA_PAGE scale = 1000; break; default: // DOM_DELTA_PIXEL } this.dx = Math.round(scale * event.deltaX); this.dy = Math.round(scale * event.deltaY); this.timestamp = Math.round(event.timeStamp * 1000); } } // in milliseconds const fade_time = 5000; const vs_source = ` attribute vec3 aVertex; uniform float uTime; varying lowp vec4 vColor; void main() { float dt = uTime - aVertex[2]; gl_Position = vec4(aVertex[0], aVertex[1], 1.0, 1.0); vColor = vec4(0.0, 170.0/255.0, 1.0, 1.0) * max(1.0 - dt/${fade_time}.0, 0.0); } `; const fs_source = ` varying lowp vec4 vColor; void main() { gl_FragColor = vColor; } `; class Painter { canvas: HTMLCanvasElement; gl: WebGLRenderingContext; /* Store lines currently being drawn. * * Keys are pointerIds, values are an array of the last position (x, y), thickness and event * time and another array with vertices to be used by webgl. Each vertex is made of 3 floats, x * and y coordinates and the event time. Vertices always come in pairs of two. Two such vertices * describe the edges of the line to be drawn with regard to it's thickness. TRIANGLE_STRIP is * then used to connect them and draw an actual line with some thickness depending on the * pressure applied. */ lines_active: Map // Array of vertices that are not actively drawn anymore and do not need updates, except // removing them after they faded away. lines_old: number[][]; vertex_attr: GLint; vertex_buffer: WebGLBuffer; time_attr: WebGLUniformLocation; initialized: boolean; constructor(canvas: HTMLCanvasElement) { this.canvas = canvas; canvas.width = window.innerWidth * window.devicePixelRatio; canvas.height = window.innerHeight * window.devicePixelRatio; this.gl = canvas.getContext("webgl"); if (this.gl) { this.lines_active = new Map(); this.lines_old = []; this.setupWebGL(); } } loadShader(type, source): WebGLShader { let gl = this.gl; const shader = gl.createShader(type); gl.shaderSource(shader, source); gl.compileShader(shader); if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { log(LogLevel.WARN, "Failed to compile shaders: " + gl.getShaderInfoLog(shader)); gl.deleteShader(shader); return null; } return shader; } setupWebGL() { let gl = this.gl; gl.enable(gl.BLEND); gl.clearColor(0, 0, 0, 0); gl.clear(gl.COLOR_BUFFER_BIT); const vertex_shader = this.loadShader(gl.VERTEX_SHADER, vs_source); const fragment_shader = this.loadShader(gl.FRAGMENT_SHADER, fs_source); if (!vertex_shader || !fragment_shader) return; const shader_program = gl.createProgram(); gl.attachShader(shader_program, vertex_shader); gl.attachShader(shader_program, fragment_shader); gl.linkProgram(shader_program); if (!gl.getProgramParameter(shader_program, gl.LINK_STATUS)) { log(LogLevel.WARN, "Unable to initialize the shader program: " + gl.getProgramInfoLog(shader_program)); return; } this.vertex_attr = gl.getAttribLocation(shader_program, "aVertex"); this.time_attr = gl.getUniformLocation(shader_program, "uTime"); this.vertex_buffer = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, this.vertex_buffer); gl.vertexAttribPointer(this.vertex_attr, 3, gl.FLOAT, false, 0, 0); gl.enableVertexAttribArray(this.vertex_attr); gl.useProgram(shader_program); this.initialized = true; requestAnimationFrame(() => this.render()); } render() { // only do work if necessary if (!check_video.checked && (this.lines_active.size > 0 || this.lines_old.length > 0)) { if (this.lines_old.length > 0) { if (performance.now() - this.lines_old[0][this.lines_old[0].length - 1] > fade_time) this.lines_old.shift(); } let gl = this.gl; gl.viewport(0, 0, this.canvas.width, this.canvas.height); gl.clear(gl.COLOR_BUFFER_BIT); gl.uniform1f(this.time_attr, performance.now()); gl.bindBuffer(gl.ARRAY_BUFFER, this.vertex_buffer); for (let vertices of this.lines_old) { gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(vertices), gl.DYNAMIC_DRAW); gl.drawArrays(gl.TRIANGLE_STRIP, 0, vertices.length / 3) } for (let [_, vertices] of this.lines_active.values()) { // sometimes there are no linesegments because there has been only a single // PointerEvent if (vertices.length == 0) continue; gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(vertices), gl.DYNAMIC_DRAW); gl.drawArrays(gl.TRIANGLE_STRIP, 0, vertices.length / 3) } } requestAnimationFrame(() => this.render()); } appendEventToLine(event: PointerEvent) { let line = this.lines_active.get(event.pointerId); if (!line) { line = [null, []]; this.lines_active.set(event.pointerId, line) } let max_pixels = Math.max(this.canvas.width, this.canvas.height); let x = event.clientX * window.devicePixelRatio / this.canvas.width * 2 - 1; let y = 1 - event.clientY * window.devicePixelRatio / this.canvas.height * 2; let delta = event.pressure + 0.4; let t = performance.now(); // to draw a line segment, there has to be some previous position if (line[0]) { let [x0, y0, delta0, t0] = line[0]; // get vector perpendicular to the linesegment to calculate quadrangel around the // segment with appropriate thickness let dx = (y - y0); let dy = -(x - x0); let dd = Math.sqrt(dx ** 2 + dy ** 2); if (dd == 0) { return; } dx = dx / dd * max_pixels / this.canvas.width * 0.004; dy = dy / dd * max_pixels / this.canvas.height * 0.004; if (line[1].length == 0) line[1].push( x0 + delta0 * dx, y0 + delta0 * dy, t0, x0 - delta0 * dx, y0 - delta0 * dy, t0, ); line[1].push( x + delta * dx, y + delta * dy, t, x - delta * dx, y - delta * dy, t ) } line[0] = [x, y, delta, t]; } onstart(event: PointerEvent) { this.appendEventToLine(event); } onmove(event: PointerEvent) { if (this.lines_active.has(event.pointerId)) { const events = typeof event.getCoalescedEvents === 'function' ? event.getCoalescedEvents() : [event]; for (const e of events) { this.appendEventToLine(e); } } } onstop(event: PointerEvent) { let lines = this.lines_active.get(event.pointerId); if (lines) { if (lines[1].length > 0) this.lines_old.push(lines[1]); this.lines_active.delete(event.pointerId); } } } class PointerHandler { webSocket: WebSocket; pointerTypes: string[]; constructor(webSocket: WebSocket) { let video = document.getElementById("video"); let canvas = document.getElementById("canvas"); this.webSocket = webSocket; this.pointerTypes = settings.pointer_types(); video.onpointerdown = (e) => this.onEvent(e, "pointerdown"); video.onpointerup = (e) => this.onEvent(e, "pointerup"); video.onpointercancel = (e) => this.onEvent(e, "pointercancel"); video.onpointermove = (e) => this.onEvent(e, "pointermove"); video.onpointerout = (e) => this.onEvent(e, "pointerout"); video.onpointerleave = (e) => this.onEvent(e, "pointerleave"); video.onpointerenter = (e) => this.onEvent(e, "pointerenter"); video.onpointerover = (e) => this.onEvent(e, "pointerover"); let painter: Painter; if (!settings.checks.get("energysaving").checked) painter = new Painter(canvas as HTMLCanvasElement); if (painter && painter.initialized) { canvas.onpointerdown = (e) => { this.onEvent(e, "pointerdown"); painter.onstart(e); }; canvas.onpointerup = (e) => { this.onEvent(e, "pointerup"); painter.onstop(e); }; canvas.onpointercancel = (e) => { this.onEvent(e, "pointercancel"); painter.onstop(e); }; canvas.onpointermove = (e) => { this.onEvent(e, "pointermove"); painter.onmove(e); }; canvas.onpointerout = (e) => { this.onEvent(e, "pointerout"); painter.onstop(e); }; canvas.onpointerleave = (e) => { this.onEvent(e, "pointerleave"); painter.onstop(e); }; canvas.onpointerenter = (e) => { this.onEvent(e, "pointerenter"); painter.onmove(e); }; canvas.onpointerover = (e) => { this.onEvent(e, "pointerover"); painter.onmove(e); }; } else { canvas.onpointerdown = (e) => this.onEvent(e, "pointerdown"); canvas.onpointerup = (e) => this.onEvent(e, "pointerup"); canvas.onpointercancel = (e) => this.onEvent(e, "pointercancel"); canvas.onpointermove = (e) => this.onEvent(e, "pointermove"); canvas.onpointerout = (e) => this.onEvent(e, "pointerout"); canvas.onpointerleave = (e) => this.onEvent(e, "pointerleave"); canvas.onpointerenter = (e) => this.onEvent(e, "pointerenter"); canvas.onpointerover = (e) => this.onEvent(e, "pointerover"); } // This is a workaround for the following Safari/WebKit bug: // https://bugs.webkit.org/show_bug.cgi?id=217430 // I have no idea why this works but it does. video.ontouchmove = (e) => e.preventDefault(); canvas.ontouchmove = (e) => e.preventDefault(); for (let elem of [video, canvas]) { elem.onwheel = (e) => { this.webSocket.send(JSON.stringify({ "WheelEvent": new WEvent(e) })); } } } onEvent(event: PointerEvent, event_type: string) { if (settings.checks.get("enable_debug_overlay").checked) { let props = [ "altKey", "altitudeAngle", "azimuthAngle", "button", "buttons", "clientX", "clientY", "ctrlKey", "height", "isPrimary", "metaKey", "movementX", "movementY", "offsetX", "offsetY", "pageX", "pageY", "pointerId", "pointerType", "pressure", "screenX", "screenY", "shiftKey", "tangentialPressure", "tiltX", "tiltY", "timeStamp", "twist", "type", "width", "x", "y", ]; if (!last_pointer_data) { last_pointer_data = {}; for (let prop of props) { let span_id = `prop_${prop}_span`; let span = document.getElementById(span_id); span = document.createElement("span"); span.id = span_id; debug_overlay.appendChild(span); debug_overlay.appendChild(document.createElement("br")); } } for (let prop of props) { let span_id = `prop_${prop}_span`; let span = document.getElementById(span_id); let v = event[prop]; span.textContent = `${prop}: ${v}`; if (last_pointer_data[prop] == v) { span.classList.remove("updated"); } else { span.classList.add("updated"); last_pointer_data[prop] = v; } } } if (this.pointerTypes.includes(event.pointerType)) { let rect = (event.target as HTMLElement).getBoundingClientRect(); const events = event_type === "pointermove" && typeof event.getCoalescedEvents === 'function' ? event.getCoalescedEvents() : [event]; for (let event of events) { this.webSocket.send( JSON.stringify( { "PointerEvent": new PEvent( event_type, event, rect ) } ) ); } if (settings.visible) { settings.toggle(); } } } } class KEvent { event_type: string; code: string; key: string; location: number; alt: boolean; ctrl: boolean; shift: boolean; meta: boolean; constructor(event_type: string, event: KeyboardEvent) { this.event_type = event_type; this.code = event.code; this.key = event.key; this.location = event.location; this.alt = event.altKey; this.ctrl = event.ctrlKey; this.shift = event.shiftKey; this.meta = event.metaKey; } } class KeyboardHandler { webSocket: WebSocket; constructor(webSocket: WebSocket) { this.webSocket = webSocket; let d = document; let s = document.getElementById("settings") // Consume all KeyboardEvents, except the settings menu is open. // this avoids making the main/video/canvas element focusable by using // a tabindex which interferes with PointerEvent than can be considered // hovering. function settings_hidden() { return s.classList.contains("hide") || s.classList.contains("vanish"); } d.onkeydown = (e) => { if (!settings_hidden()) return true; if (e.repeat) return this.onEvent(e, "repeat"); return this.onEvent(e, "down"); }; d.onkeyup = (e) => { if (!settings_hidden()) return true; return this.onEvent(e, "up"); }; d.onkeypress = (e) => { if (!settings_hidden()) return true; e.preventDefault(); e.stopPropagation(); return false; }; } onEvent(event: KeyboardEvent, event_type: string) { this.webSocket.send(JSON.stringify({ "KeyboardEvent": new KEvent(event_type, event) })); event.preventDefault(); event.stopPropagation(); return false; } } function frame_rate_stats() { let t = performance.now(); let fps = Math.round(frame_count / (t - last_fps_calc) * 10000) / 10; fps_out.value = fps.toString(); frame_count = 0; last_fps_calc = t; setTimeout(() => frame_rate_stats(), 1500); } function handle_messages( webSocket: WebSocket, video: HTMLVideoElement, onConfigOk: Function, onConfigError: Function, onCapturableList: Function, ) { let mediaSource: MediaSource = null; let sourceBuffer: SourceBuffer = null; let queue = []; const MAX_BUFFER_LENGTH = 20; // In seconds function upd_buf() { if (sourceBuffer == null) return; if (!sourceBuffer.updating && queue.length > 0 && mediaSource.readyState == "open") { let buffer_length = 0; if (sourceBuffer.buffered.length) { // Assume only one time range... buffer_length = sourceBuffer.buffered.end(0) - sourceBuffer.buffered.start(0); } if (buffer_length > MAX_BUFFER_LENGTH) { sourceBuffer.remove(0, sourceBuffer.buffered.end(0) - MAX_BUFFER_LENGTH / 2); // This will trigger updateend when finished } else { try { sourceBuffer.appendBuffer(queue.shift()); } catch (err) { log(LogLevel.DEBUG, "Error appending to sourceBuffer:" + err); // Drop everything, and try to pick up the stream again if (sourceBuffer.updating) sourceBuffer.abort(); sourceBuffer.remove(0, Infinity); } } } } webSocket.onmessage = (event: MessageEvent) => { if (typeof event.data == "string") { let msg = JSON.parse(event.data); if (typeof msg == "string") { if (msg == "NewVideo") { let MS = window.ManagedMediaSource ? window.ManagedMediaSource : window.MediaSource; mediaSource = new MS(); sourceBuffer = null; video.src = URL.createObjectURL(mediaSource); mediaSource.addEventListener("sourceopen", (_) => { let mimeType = 'video/mp4; codecs="avc1.4D403D"'; if (!MS.isTypeSupported(mimeType)) mimeType = "video/mp4"; sourceBuffer = mediaSource.addSourceBuffer(mimeType); sourceBuffer.addEventListener("updateend", upd_buf); // try to recover from errors by restarting the video if (sourceBuffer.onerror) sourceBuffer.onerror = () => settings.send_server_config(); }) } else if (msg == "ConfigOk") { onConfigOk(); } } else if (typeof msg == "object") { if ("CapturableList" in msg) onCapturableList(msg["CapturableList"]); else if ("Error" in msg) alert(msg["Error"]); else if ("ConfigError" in msg) { onConfigError(msg["ConfigError"]); } else if ("CustomInputAreas" in msg) { settings.custom_input_areas = msg["CustomInputAreas"]; settings.checks.get("enable_custom_input_areas").checked = true; settings.save_settings(); } } return; } // not a string -> got a video frame queue.push(event.data); upd_buf(); frame_count += 1; // only seek if there is data available, some browsers choke otherwise if (video.seekable.length > 0) { let seek_time = video.seekable.end(video.seekable.length - 1); if (video.readyState >= (settings.check_aggressive_seek.checked ? 3 : 4) // but make sure to catch up if the video is more than 3 seconds behind || seek_time - video.currentTime > 3) { if (isFinite(seek_time)) video.currentTime = seek_time; else log(LogLevel.WARN, "Failed to seek to end of video.") } } } } function check_apis() { let apis = [ { attrs: ["MediaSource", "ManagedMediaSource"], msg: "This browser doesn't support MSE/MMS required to playback video stream, try upgrading!" }, { attrs: ["PointerEvent"], msg: "This browser doesn't support PointerEvents, input will not work, try upgrading!" }, ]; outer: for (let d of apis) { for (let attr of d.attrs) { if (attr in window) { continue outer; } } log(LogLevel.ERROR, d.msg); } } function init() { check_apis(); let protocol = document.location.protocol == "https:" ? "wss://" : "ws://"; let webSocket = new WebSocket( protocol + window.location.hostname + ":" + window.location.port + "/ws" + window.location.search ); webSocket.binaryType = "arraybuffer"; debug_overlay = document.getElementById("debug_overlay"); settings = new Settings(webSocket); let video = document.getElementById("video") as HTMLVideoElement; let canvas = document.getElementById("canvas") as HTMLCanvasElement; video.oncontextmenu = function(event) { event.preventDefault(); event.stopPropagation(); return false; }; canvas.oncontextmenu = function(event) { event.preventDefault(); event.stopPropagation(); return false; }; let toggle_fullscreen_btn = document.getElementById("fullscreen") as HTMLButtonElement; if (document.exitFullscreen) { toggle_fullscreen_btn.onclick = () => { if (!document.fullscreenElement) document.body.requestFullscreen({ navigationUI: "hide" }); else document.exitFullscreen(); } } else { // if document.exitFullscreen is not present we are probably running on iOS/iPadOS. // As input is broken in fullscreen mode on these, do not offer fullscreen in the first // place. toggle_fullscreen_btn.parentElement.removeChild(toggle_fullscreen_btn); } let handle_disconnect = (msg: string) => { document.body.onclick = video.onclick = (e) => { e.stopPropagation(); if (window.confirm(msg + " Reload page?")) location.reload(); } } webSocket.onerror = () => handle_disconnect("Lost connection."); webSocket.onclose = () => handle_disconnect("Connection closed."); window.onresize = () => { stretch_video(); canvas.width = window.innerWidth * window.devicePixelRatio; canvas.height = window.innerHeight * window.devicePixelRatio; let [w, h] = calc_max_video_resolution(settings.scale_video_input.valueAsNumber); settings.scale_video_output.value = w + "x" + h; settings.send_server_config(); } video.controls = false; video.disableRemotePlayback = true; video.onloadeddata = () => stretch_video(); let is_connected = false; handle_messages(webSocket, video, () => { if (!is_connected) { new KeyboardHandler(webSocket); new PointerHandler(webSocket); is_connected = true; } }, (err) => alert(err), (window_names) => settings.onCapturableList(window_names) ); window.onunload = () => { webSocket.close(); } webSocket.onopen = function(event) { webSocket.send('"GetCapturableList"'); if (!settings.video_enabled()) webSocket.send('"PauseVideo"'); settings.send_server_config(); document.onvisibilitychange = () => { if (document.hidden) { webSocket.send('"PauseVideo"'); } else if (settings.video_enabled()) { webSocket.send('"ResumeVideo"'); } }; } frame_rate_stats(); } // object-fit: fill; <-- this is unfortunately not supported on iOS, so we use the following // workaround function stretch_video() { let video = document.getElementById("video") as HTMLVideoElement; if (settings.stretched_video()) { video.style.transform = "scaleX(" + document.body.clientWidth / video.clientWidth + ") scaleY(" + document.body.clientHeight / video.clientHeight + ")"; } else { let scale = Math.min(document.body.clientWidth / video.clientWidth, document.body.clientHeight / video.clientHeight); video.style.transform = "scale(" + scale + ")"; } } ================================================ FILE: tsconfig.json ================================================ { "compilerOptions": { "target": "es2020", "lib": ["es2020", "dom", "es6"], "module": "commonjs", "outDir": "www/static", "sourceMap": false }, "include": [ "ts/**/*.ts" ] } ================================================ FILE: weylus.desktop ================================================ [Desktop Entry] Type=Application Name=Weylus Comment=Use your tablet as graphic tablet/touch screen on your computer. TryExec=weylus Exec=weylus Terminal=false Icon=input-tablet Categories=Office;Graphics;Education;Presentation; StartupWMClass=weylus ================================================ FILE: weylus_tls.sh ================================================ #!/usr/bin/env sh function die { # cleanup to ensure restarting this script doesn't fail because # of ports that are still in use kill $(jobs -p) > /dev/null 2>&1 exit $1 } # generate certificate if it doesn't exist yet if [ ! -e weylus.pem ] then openssl req -batch -newkey rsa:4096 -sha256 -keyout weylus.key -nodes -x509 -days 365 \ -subj="/CN=Weylus" -out weylus.crt # combine into a pem file as this is everything hitch needs cat weylus.key weylus.crt > weylus.pem rm weylus.key weylus.crt fi # WEYLUS can be used to determine which version of Weylus to run # If unset, try ./weylus and then weylus from path. If both fail, # read the path to Weylus from stdin. if [ -z "$WEYLUS" ] then if [ -e weylus ] then WEYLUS=./weylus else if which weylus > /dev/null 2>&1 then WEYLUS=weylus else echo "Please specify path to weylus." echo -n "> " read -r WEYLUS fi fi fi if [ -z "$ACCESS_CODE" ] then # generate access code if none is given ACCESS_CODE="$(openssl rand -base64 12)" echo "Autogenerated access code: $ACCESS_CODE" fi # cleanup on CTRL+C trap die SIGINT # The TLS proxy will be set up as follows: Proxy all incoming traffic from # port 1701 to 1702 on which the actual instance of Weylus is running. # start Weylus listening only on the local interface $WEYLUS --bind-address "127.0.0.1" \ --web-port "1702" \ --access-code "$ACCESS_CODE" \ --no-gui & # start the proxy hitch --frontend="[0.0.0.0]:1701" --backend="[127.0.0.1]:1702" \ --daemon=off --tls-protos="TLSv1.2 TLSv1.3" "weylus.pem" & wait ================================================ FILE: www/static/access_code.html ================================================ Weylus


================================================ FILE: www/static/style.css ================================================ @media (prefers-color-scheme: dark) { :root { --color: #ddd; --background-color-1: #303030; --background-color-2: #202020; } } @media (prefers-color-scheme: light) { :root { --color: #111; --background-color-1: #eee; --background-color-2: #fff; } } main { touch-action: none; user-select: none; -webkit-touch-callout: none; -webkit-user-select: none; -khtml-user-select: none; -moz-user-select: none; -ms-user-select: none; } body, html, main { width: 100%; height: 100%; margin: 0; padding: 0; overflow: hidden; display: flex; color: var(--color); background: var(--background-color-2); } video { max-width: 100%; max-height: 100%; width: auto; height: auto; display: block; margin: auto auto; } canvas { position: absolute; width: 100%; height: 100%; } input[type='text'] { touch-action: auto !important; user-select: text; -webkit-touch-callout: default; -webkit-user-select: text; -khtml-user-select: text; -moz-user-select: text; -ms-user-select: text; } input[type='range']::-webkit-slider-runnable-track, input[type="range"]::-moz-range-track { background-color: #00aaff; } .container { width: 100%; height: 100%; display: flex; justify-content: center; align-items: center; } #settings, #handle, #debug_overlay { color: var(--color); background: var(--background-color-1); } #settings_scroll { overflow: auto; width: 100%; height: 100%; } #settings { width: 16em; display: block; position: absolute; right: 0; top: 0; bottom: 0; transform: none; padding: 1em; font-family: sans-serif; transition: transform 0.5s, opacity 0.5s; } #settings.lefty { left: 0; right: auto; } #settings.hide, #settings.vanish { transform: translateX(100%); opacity: 40%; } #settings.hide.lefty, #settings.vanish.lefty { transform: translateX(-100%); opacity: 40%; } #handle { position: absolute; top: 0; left: -1em; width: 1em; border-radius: 0 0 0 0.25em; padding: 0.2em 0 0.1em 0; font-size: 150%; text-align:center; } .lefty #handle { left: auto; right: -1em; border-radius: 0 0 0.25em 0; } .vanish #handle, video.vanish, canvas.vanish { display: none !important; } #settings h2 { font-size: 1.2em; margin: 0; text-align: center; } #settings h3 { font-size: 1em; margin-top: 0.2; text-align: center; } #settings section { margin-bottom: 1em; } #settings section label, section button { display: block; margin-top: 0.5em; } #settings section.hide, section label.hide, section button.hide, #debug_overlay.hide { display: none !important; } select { width: 15em; } #displayoptions { display: flex; flex-direction: row; } .lefty #displayoptions { flex-direction: row-reverse; } #displayoptions label { flex-grow: 1; } label input:disabled + span { color: #888; } #frame_update_limit { width: auto; } #lefty { display: none; } #leftylabel::before { content: "❮ "; } .lefty #leftylabel { text-align: right; } .lefty #leftylabel::before { content: ""; } .lefty #leftylabel::after { content: " ❯"; } #vanish { text-align: right; } .lefty #vanish { text-align: left; } #log { max-width: 15em; overflow-x: auto; touch-action: pan-x pan-y; user-select: text; -webkit-touch-callout: default; -webkit-user-select: text; -khtml-user-select: text; -moz-user-select: text; -ms-user-select: text; } #debug_overlay { --padding: 3px; position: absolute; top:0; bottom: 0; left: 0; right: 0; margin: auto; width: fit-content; height: fit-content; z-index: 1; pointer-events: none; opacity: 75%; padding: var(--padding); border: 2px solid var(--color); white-space: pre; font-size: small; text-align: center; } #debug_overlay span.updated { color: darkgreen; } ================================================ FILE: www/templates/index.html ================================================ Weylus
⠿

Settings

Video

Input