Repository: tycho-kirchner/shournal Branch: master Commit: f7c511243968 Files: 332 Total size: 1.4 MB Directory structure: gitextract_hf3qq4ys/ ├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── README-compile.md ├── README-shell-integration.md ├── README.md ├── cmake/ │ └── FindShournalUtil.cmake ├── extern/ │ ├── folly/ │ │ ├── LICENSE │ │ ├── README.md │ │ └── UninitializedMemoryHacks.h │ ├── tsl-ordered-map/ │ │ ├── CMakeLists.txt │ │ ├── LICENSE │ │ ├── ordered_hash.h │ │ ├── ordered_map.h │ │ └── ordered_set.h │ └── xxHash/ │ ├── .gitattributes │ ├── .gitignore │ ├── .travis.yml │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── appveyor.yml │ ├── cmake_unofficial/ │ │ ├── .gitignore │ │ ├── CMakeLists.txt │ │ └── README.md │ ├── doc/ │ │ └── xxhash_spec.md │ ├── xxhash.c │ ├── xxhash.h │ ├── xxhsum.1 │ ├── xxhsum.1.md │ └── xxhsum.c ├── html-export/ │ ├── dist/ │ │ ├── htmlexportres.qrc │ │ ├── index.html │ │ ├── main.js │ │ └── main.licenses.txt │ ├── package.json │ ├── src/ │ │ ├── annotation_line_render.js │ │ ├── command_list.js │ │ ├── command_manipulation.js │ │ ├── command_timeline.js │ │ ├── conversions.js │ │ ├── d3js_util.js │ │ ├── generic_text_dialog.js │ │ ├── globals.js │ │ ├── html_util.js │ │ ├── index.js │ │ ├── limited_queue.js │ │ ├── map_extended.js │ │ ├── plot_cmdcount_per_cwd.js │ │ ├── plot_cmdcount_per_session.js │ │ ├── plot_io_per_dir.js │ │ ├── plot_most_written_files.js │ │ ├── plot_simple_bar.js │ │ ├── session_timeline.js │ │ ├── stats.js │ │ ├── timeline_group_find.js │ │ ├── tooltip.js │ │ ├── util.js │ │ └── zoom_buttons.js │ └── webpack.config.js ├── install/ │ ├── 90-shournaladd.rules.in │ ├── CMakeLists.txt │ ├── postinst-dkms.in │ ├── postinst.in │ ├── prerm-dkms.in │ ├── prerm.in │ └── shournalk-load.conf ├── kernel/ │ ├── CMakeLists.txt │ ├── Kbuild │ ├── LICENSE │ ├── cmake/ │ │ └── FindKernelHeaders.cmake │ ├── dkms.conf.in │ ├── event_consumer.c │ ├── event_consumer.h │ ├── event_consumer_cache.c │ ├── event_consumer_cache.h │ ├── event_handler.c │ ├── event_handler.h │ ├── event_queue.c │ ├── event_queue.h │ ├── event_target.c │ ├── event_target.h │ ├── hash_table_str.c │ ├── hash_table_str.h │ ├── kfileextensions.c │ ├── kfileextensions.h │ ├── kpathtree.c │ ├── kpathtree.h │ ├── kutil.c │ ├── kutil.h │ ├── shournal_kio.c │ ├── shournal_kio.h │ ├── shournalk_global.c │ ├── shournalk_global.h │ ├── shournalk_main.c │ ├── shournalk_sysfs.c │ ├── shournalk_sysfs.h │ ├── shournalk_test.c │ ├── shournalk_test.h │ ├── shournalk_user.h │ ├── tracepoint_helper.c │ ├── tracepoint_helper.h │ ├── xxhash_shournalk.c │ └── xxhash_shournalk.h ├── shell-integration-scripts/ │ ├── CMakeLists.txt │ ├── _source_me_generic.sh │ ├── integration_fan.sh │ ├── integration_ko.sh │ ├── integration_main.sh.in │ └── util.sh ├── src/ │ ├── CMakeLists.txt │ ├── common/ │ │ ├── CMakeLists.txt │ │ ├── app.cpp │ │ ├── app.h │ │ ├── cefd.cpp │ │ ├── cefd.h │ │ ├── console_dialog.cpp │ │ ├── console_dialog.h │ │ ├── cxxhash.cpp │ │ ├── cxxhash.h │ │ ├── database/ │ │ │ ├── command_query_iterator.cpp │ │ │ ├── command_query_iterator.h │ │ │ ├── commandinfo.cpp │ │ │ ├── commandinfo.h │ │ │ ├── db_connection.cpp │ │ │ ├── db_connection.h │ │ │ ├── db_controller.cpp │ │ │ ├── db_controller.h │ │ │ ├── db_conversions.cpp │ │ │ ├── db_conversions.h │ │ │ ├── db_globals.cpp │ │ │ ├── db_globals.h │ │ │ ├── file_query_helper.cpp │ │ │ ├── file_query_helper.h │ │ │ ├── fileinfos.cpp │ │ │ ├── fileinfos.h │ │ │ ├── insertifnotexist.cpp │ │ │ ├── insertifnotexist.h │ │ │ ├── qexcdatabase.cpp │ │ │ ├── qexcdatabase.h │ │ │ ├── qsqlquerythrow.cpp │ │ │ ├── qsqlquerythrow.h │ │ │ ├── query_columns.h │ │ │ ├── sessioninfo.cpp │ │ │ ├── sessioninfo.h │ │ │ ├── sqlite_database_scheme.cpp │ │ │ ├── sqlite_database_scheme.h │ │ │ ├── sqlite_database_scheme_updates.cpp │ │ │ ├── sqlite_database_scheme_updates.h │ │ │ ├── sqlquery.cpp │ │ │ ├── sqlquery.h │ │ │ ├── storedfiles.cpp │ │ │ └── storedfiles.h │ │ ├── fdcommunication.cpp │ │ ├── fdcommunication.h │ │ ├── fileeventhandler.cpp │ │ ├── fileeventhandler.h │ │ ├── fileevents.cpp │ │ ├── fileevents.h │ │ ├── generic_container.h │ │ ├── groupcontrol.cpp │ │ ├── groupcontrol.h │ │ ├── hashcontrol.cpp │ │ ├── hashcontrol.h │ │ ├── hashmeta.cpp │ │ ├── hashmeta.h │ │ ├── idmapentry.h │ │ ├── interrupt_handler.cpp │ │ ├── interrupt_handler.h │ │ ├── limited_priority_queue.h │ │ ├── logger.cpp │ │ ├── logger.h │ │ ├── oscpp/ │ │ │ ├── CMakeLists.txt │ │ │ ├── cflock.cpp │ │ │ ├── cflock.h │ │ │ ├── excos.cpp │ │ │ ├── excos.h │ │ │ ├── fdentries.cpp │ │ │ ├── fdentries.h │ │ │ ├── os.cpp │ │ │ ├── os.h │ │ │ ├── oscaps.cpp │ │ │ ├── oscaps.h │ │ │ ├── osutil.cpp │ │ │ └── osutil.h │ │ ├── pathtree.cpp │ │ ├── pathtree.h │ │ ├── pidcontrol.cpp │ │ ├── pidcontrol.h │ │ ├── qfddummydevice.cpp │ │ ├── qfddummydevice.h │ │ ├── qfilethrow.cpp │ │ ├── qfilethrow.h │ │ ├── qoptargparse/ │ │ │ ├── CMakeLists.txt │ │ │ ├── excoptargparse.cpp │ │ │ ├── excoptargparse.h │ │ │ ├── qoptarg.cpp │ │ │ ├── qoptarg.h │ │ │ ├── qoptargparse.cpp │ │ │ ├── qoptargparse.h │ │ │ ├── qoptargtrigger.cpp │ │ │ ├── qoptargtrigger.h │ │ │ ├── qoptsqlarg.cpp │ │ │ ├── qoptsqlarg.h │ │ │ ├── qoptvarlenarg.cpp │ │ │ └── qoptvarlenarg.h │ │ ├── qresource_helper.cpp │ │ ├── qresource_helper.h │ │ ├── qsimplecfg/ │ │ │ ├── CMakeLists.txt │ │ │ ├── cfg.cpp │ │ │ ├── cfg.h │ │ │ ├── exccfg.cpp │ │ │ ├── exccfg.h │ │ │ ├── section.cpp │ │ │ └── section.h │ │ ├── safe_file_update.h │ │ ├── settings.cpp │ │ ├── settings.h │ │ ├── shournal_run_common.cpp │ │ ├── shournal_run_common.h │ │ ├── socket_message.cpp │ │ ├── socket_message.h │ │ ├── stdiocpp.cpp │ │ ├── stdiocpp.h │ │ ├── stupidinject.cpp │ │ ├── stupidinject.h │ │ ├── subprocess.cpp │ │ ├── subprocess.h │ │ ├── user_kernerl.h │ │ ├── util/ │ │ │ ├── CMakeLists.txt │ │ │ ├── cleanupresource.h │ │ │ ├── compareoperator.cpp │ │ │ ├── compareoperator.h │ │ │ ├── compat.h │ │ │ ├── conversions.cpp │ │ │ ├── conversions.h │ │ │ ├── cpp_exit.cpp │ │ │ ├── cpp_exit.h │ │ │ ├── exccommon.cpp │ │ │ ├── exccommon.h │ │ │ ├── nullable_value.h │ │ │ ├── qformattedstream.cpp │ │ │ ├── qformattedstream.h │ │ │ ├── qoutstream.cpp │ │ │ ├── qoutstream.h │ │ │ ├── staticinitializer.h │ │ │ ├── strlight.cpp │ │ │ ├── strlight.h │ │ │ ├── strlight_util.cpp │ │ │ ├── strlight_util.h │ │ │ ├── sys_ioprio.h │ │ │ ├── translation.cpp │ │ │ ├── translation.h │ │ │ ├── util.cpp │ │ │ ├── util.h │ │ │ ├── util_performance.cpp │ │ │ └── util_performance.h │ │ ├── xxhash_common.c │ │ └── xxhash_common.h │ ├── shell-integration-fanotify/ │ │ ├── CMakeLists.txt │ │ ├── attached_bash.cpp │ │ ├── attached_bash.h │ │ ├── attached_shell.cpp │ │ ├── attached_shell.h │ │ ├── event_open.cpp │ │ ├── event_open.h │ │ ├── event_process.cpp │ │ ├── event_process.h │ │ ├── libshellwatch.version │ │ ├── libshournal-shellwatch.cpp │ │ ├── shell_globals.cpp │ │ ├── shell_globals.h │ │ ├── shell_logger.cpp │ │ ├── shell_logger.h │ │ ├── shell_request_handler.cpp │ │ └── shell_request_handler.h │ ├── shournal/ │ │ ├── CMakeLists.txt │ │ ├── argcontrol_dbdelete.cpp │ │ ├── argcontrol_dbdelete.h │ │ ├── argcontrol_dbquery.cpp │ │ ├── argcontrol_dbquery.h │ │ ├── cmd_stats.cpp │ │ ├── cmd_stats.h │ │ ├── command_printer.cpp │ │ ├── command_printer.h │ │ ├── command_printer_html.cpp │ │ ├── command_printer_html.h │ │ ├── command_printer_human.cpp │ │ ├── command_printer_human.h │ │ ├── command_printer_json.cpp │ │ ├── command_printer_json.h │ │ └── shournal.cpp │ ├── shournal-run/ │ │ ├── CMakeLists.txt │ │ ├── fifocom.cpp │ │ ├── fifocom.h │ │ ├── filewatcher_shournalk.cpp │ │ ├── filewatcher_shournalk.h │ │ ├── mark_helper.cpp │ │ ├── mark_helper.h │ │ ├── shournal-run.cpp │ │ ├── shournalk_ctrl.c │ │ └── shournalk_ctrl.h │ └── shournal-run-fanotify/ │ ├── CMakeLists.txt │ ├── fanotify_controller.cpp │ ├── fanotify_controller.h │ ├── filewatcher_fan.cpp │ ├── filewatcher_fan.h │ ├── mount_controller.cpp │ ├── mount_controller.h │ ├── msenter.cpp │ ├── msenter.h │ ├── orig_mountspace_process.cpp │ ├── orig_mountspace_process.h │ └── shournal-run-fanotify.cpp └── test/ ├── CMakeLists.txt ├── autotest.h ├── helper_for_test.cpp ├── helper_for_test.h ├── integration_test_shell.cpp ├── main.cpp ├── sqlite_sample_db_v2_2/ │ └── readFiles/ │ ├── 3 │ └── 4 ├── test_cfg.cpp ├── test_cxxhash.cpp ├── test_db_controller.cpp ├── test_fdcommunication.cpp ├── test_fileeventhandler.cpp ├── test_osutil.cpp ├── test_pathtree.cpp ├── test_qformattedstream.cpp ├── test_qoptargparse.cpp └── test_util.cpp ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ CMakeLists.txt.user* *.geany *.autosave package-lock.json html-export/node_modules html-export/dist/SAMPLE_DATA.js html-export/.vscode .eslintrc.js ================================================ FILE: CMakeLists.txt ================================================ cmake_minimum_required(VERSION 3.6) if (NOT (CMAKE_VERSION VERSION_LESS "3.20")) cmake_policy( SET CMP0115 NEW ) endif() # version applies to all released files: shournal, shournal-run, libshournal-shellwatch.so # and shell-integration-scripts (e.g. integration_ko.bash) set(shournal_version "3.3") cmake_policy( SET CMP0048 NEW ) project(shournal VERSION ${shournal_version} LANGUAGES CXX C) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") find_package(ShournalUtil REQUIRED) if (NOT DEFINED MSENTER_GROUPNAME) set(MSENTER_GROUPNAME "shournalmsenter") endif() add_definitions( -DSHOURNAL_MSENTERGROUP="${MSENTER_GROUPNAME}") # No need to make configurable - user can override in /etc/shournal.d/kgroup # (or, not recommended, use a custom rule in /etc/udev/rules.d). # DO NOT CHANGE, it is documented in the README. set(GROUPNAME_SHOURNALK "shournalk") # Inside docker no kernel module may be installed, # but we default to using the host's kernel-module. # When only shournal-run-fanotify is desired, no need to # compile shournal-run. # -DSHOURNAL_EDITION={full, docker, ko, fanotify} if(NOT DEFINED SHOURNAL_EDITION) set(SHOURNAL_EDITION "full") endif() if(NOT ${SHOURNAL_EDITION} MATCHES "full|docker|ko|fanotify") message( FATAL_ERROR "invalid SHOURNAL_EDITION passed: ${SHOURNAL_EDITION}" ) endif() set (CMAKE_CXX_STANDARD 11) if(CMAKE_COMPILER_IS_GNUCXX) if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0) message(FATAL_ERROR "GCC version must be at least 5.0!") endif() else() message(WARNING "You are using an unsupported compiler. Compilation was only tested with GCC.") endif() if ( CMAKE_COMPILER_IS_GNUCXX ) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wunused-result -Werror=return-type") endif() add_definitions( -DSHOURNAL_VERSION="${shournal_version}" ) if (NOT EXISTS ${CMAKE_BINARY_DIR}/CMakeCache.txt) if (NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "Release" CACHE STRING "" FORCE) endif() endif() IF(CMAKE_BUILD_TYPE MATCHES Release) ADD_DEFINITIONS( -DQT_NO_DEBUG_OUTPUT=1) SET(CMAKE_AR "gcc-ar") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -s") ENDIF() # Profile purposes IF(CMAKE_BUILD_TYPE MATCHES RelWithDebInfo) ADD_DEFINITIONS( -DQT_NO_DEBUG_OUTPUT=1) set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g -DNDEBUG") ENDIF() # Meaningful stacktraces: SET (CMAKE_ENABLE_EXPORTS TRUE) # cmake policy: allow for hidden symbols in static libs cmake_policy( SET CMP0063 NEW ) set(CMAKE_POLICY_DEFAULT_CMP0063 NEW) function(hide_static_lib_symbols staticLib) set_target_properties(${staticLib} PROPERTIES CXX_VISIBILITY_PRESET hidden) set_target_properties(${staticLib} PROPERTIES CMAKE_VISIBILITY_INLINES_HIDDEN 1) endfunction(hide_static_lib_symbols) # append the content of f2 to f1 function(append_to_file f1 f2) file(READ ${f2} CONTENTS) file(APPEND ${f1} "${CONTENTS}") endfunction() # Below code could be used to strip *all* symbols, however, we do it only # for the shared lib to allow for meaningful stacktraces in shournal and shournal-run # CMP0063 NEW allows hiding symbols also in static libraries # If cmake is too old, try to use compiler optinons directly or # print warning, if that also fails. # if (CMAKE_VERSION VERSION_LESS "3.3") # IF (CMAKE_COMPILER_IS_GNUCXX OR # "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") # SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") # set(CMAKE_VISIBILITY_INLINES_HIDDEN 1) # else() # message("Warning - cannot hide all symbols of libshournal.so.") # message("Please upgrade cmake or use clang/gcc") # ENDIF() # else() # cmake_policy( SET CMP0063 NEW ) # set(CMAKE_POLICY_DEFAULT_CMP0063 NEW) # set(CMAKE_CXX_VISIBILITY_PRESET hidden) # set(CMAKE_VISIBILITY_INLINES_HIDDEN 1) # endif() include(GNUInstallDirs) set(shournal_install_dir_script ${CMAKE_INSTALL_FULL_DATAROOTDIR}/${PROJECT_NAME}) set(shournal_install_dir_lib ${CMAKE_INSTALL_FULL_LIBDIR}/${PROJECT_NAME}) set(shournal_install_dir_shournalk_src /usr/src/shournalk-${shournal_version}) set(libshournal_fullname "libshournal-shellwatch.so") set(full_path_libshournal ${shournal_install_dir_lib}/${libshournal_fullname}) set(CMAKE_INCLUDE_CURRENT_DIR ON) set(CMAKE_AUTOMOC ON) # qt resource files (.qrc): set(CMAKE_AUTORCC ON) find_package(Qt5 COMPONENTS Core Sql Network REQUIRED) include_directories( extern/tsl-ordered-map extern/folly extern/xxHash ) add_subdirectory("extern/tsl-ordered-map") set(XXHASH_BUNDLED_MODE ON) add_subdirectory(extern/xxHash/cmake_unofficial EXCLUDE_FROM_ALL) add_subdirectory("src") add_subdirectory("shell-integration-scripts") add_subdirectory("install") # Kernel module if(${SHOURNAL_EDITION} MATCHES "full|ko") add_subdirectory("kernel") endif() # Turn on tests with 'cmake -Dtest=ON'. # To run the tests enter directory "test" within the build directory # and enter "ctest". option(test "Build all tests." OFF) if (test) add_subdirectory("test") endif() # install license install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE" RENAME copyright # following Lintian DESTINATION ${CMAKE_INSTALL_FULL_DOCDIR} ) ############## Package creation using 'cpack' ############## # generic set(CPACK_GENERATOR "DEB") set(CPACK_PACKAGE_VERSION ${shournal_version}) set(CPACK_PACKAGE_CONTACT "Tycho Kirchner ") SET(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE") # If CPACK_INSTALL_PREFIX is not set, let it default to CMAKE_INSTALL_PREFIX # see also: https://stackoverflow.com/a/7363073/7015849 # set(CPACK_SET_DESTDIR true) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "File-journal for your shell" ) set(SHOURNAL_CONFLICTS_LIST "shournal" "shournal-docker" "shournal-ko" "shournal-fanotify" ) if(${SHOURNAL_EDITION} STREQUAL "full") set(CPACK_PACKAGE_NAME "shournal") set(edition_description "full suite (all backends)") elseif(${SHOURNAL_EDITION} STREQUAL "ko") set(CPACK_PACKAGE_NAME "shournal-ko") set(edition_description "kernel backend only (fanotify backend not included)") elseif(${SHOURNAL_EDITION} STREQUAL "fanotify") set(CPACK_PACKAGE_NAME "shournal-fanotify") set(edition_description "fanotify backend only (no kernel module included)") elseif(${SHOURNAL_EDITION} STREQUAL "docker") set(CPACK_PACKAGE_NAME "shournal-docker") set(edition_description "docker-version to be installed inside containers") else() message( FATAL_ERROR "invalid cpack COMPONENT: ${SHOURNAL_EDITION}" ) endif() list(REMOVE_ITEM SHOURNAL_CONFLICTS_LIST "${CPACK_PACKAGE_NAME}" ) JOIN("${SHOURNAL_CONFLICTS_LIST}" ", " SHOURNAL_CONFLICTS) # CPACK_DEBIAN_PACKAGE_DESCRIPTION requires newlines # be indented by one space. For the sake of simplicity: # No new lines here: set(CPACK_PACKAGE_DESCRIPTION "Integrated tool to increase the reproducibility \ of your work on the shell: what did you do when and \ where and what files were modified/read. This package \ provides the ${edition_description}.") # deb specific # set(CPACK_GENERATOR "DEB") set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION}") execute_process(COMMAND dpkg --print-architecture OUTPUT_VARIABLE DEB_ARCH OUTPUT_STRIP_TRAILING_WHITESPACE) set(CPACK_DEBIAN_FILE_NAME ${CPACK_PACKAGE_NAME}_${shournal_version}_${DEB_ARCH}.deb) set(CPACK_DEBIAN_PACKAGE_CONFLICTS "${SHOURNAL_CONFLICTS}") set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/tycho-kirchner/shournal") set(CPACK_DEBIAN_PACKAGE_DEPENDS "libc6 (>= 2.2), libstdc++6 (>= 5.0), libgcc1, \ libqt5core5a (>= 5.6), libqt5network5, libqt5sql5-sqlite, libcap2, uuid-runtime" ) set(CPACK_DEBIAN_PACKAGE_SECTION "utils") # generate the postinst based on the groupname set(debPostinstPath "${CMAKE_BINARY_DIR}/debian/postinst") set(debPrermPath "${CMAKE_BINARY_DIR}/debian/prerm") set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${debPostinstPath}" "${debPrermPath}" ) set(CPACK_DEBIAN_PACKAGE_CONTROL_STRICT_PERMISSION TRUE) if(${SHOURNAL_EDITION} MATCHES "full|ko") set(CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEBIAN_PACKAGE_DEPENDS}, dkms") append_to_file( "${debPostinstPath}" ${CMAKE_BINARY_DIR}/install/postinst-dkms ) append_to_file( "${debPrermPath}" ${CMAKE_BINARY_DIR}/install/prerm-dkms ) endif() # call it *after* setting above variables, otherwise # generic .gz's are generated. include(CPack) ================================================ FILE: LICENSE ================================================ GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU General Public License is a free, copyleft license for software and other kinds of works. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others. For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it. For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions. Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users. Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Use with the GNU Affero General Public License. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode: Copyright (C) This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an "about box". You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see . The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read . ================================================ FILE: README-compile.md ================================================ # Compile and install from source * Install gcc >= 5.0. Other compilers might work but are untested. * Install cmake >= 3.6 and make * For safe generation of uuids it is recommend to install uuidd (uuid-runtime) * Install qt-dev, uuid-dev, qt-sqlite-driver, Qt version >= 5.6. *With a little effort, shournal could be modified to support Qt version >= 5.3. Please open an issue, if that would be helpful to you.* * To build the kernel-module the headers are also required (linux-headers-$(uname -r)) *Packages lists*: Debian: ~~~ apt-get install g++ cmake make qtbase5-dev libqt5sql5-sqlite \ uuid-dev libcap-dev uuid-runtime linux-headers-$(dpkg --print-architecture) dkms ~~~ Ubuntu: ~~~ apt-get install g++ cmake make qtbase5-dev libqt5sql5-sqlite \ uuid-dev libcap-dev uuid-runtime dkms \ linux-headers-generic # or linux-headers-generic-hwe-$(lsb_release -rs) on HWE ~~~ Opensuse: ~~~ zypper install gcc-c++ cmake make libqt5-qtbase-devel \ libQt5Sql5-sqlite libuuid-devel libcap-devel uuidd \ kernel-default-devel dkms ~~~ Arch Linux: ~~~ yay -S gcc cmake make qt5-base uuid libcap linux-headers dkms ~~~ CentOS (note: CentOS 7 as of July 2019 only ships with gcc 4.8 -> compile gcc >= 5.0 yourself. cmake3 and cmake are seperate packages where cmake in version 2 is the default. Please ensure to compile with cmake3. The kernel 3.10 is too old for *shournal*'s kernel-module. Either install a newer one or stick with the fanotify-edition): ~~~ yum install gcc-c++ cmake3 make qt5-qtbase-devel libuuid-devel \ libcap-devel uuidd kernel-devel dkms ~~~ * In the source-tree-directory, enter the following commands to compile and install. By default `SHOURNAL_EDITION` `full` is built (see below). Supported options include `full, docker, ko, fanotify`. The `ko` (kernel module) edition does not install the fanotify backend which may be desirable for security reasons as the setuid-binary `shournal-run-fanotify` is omitted. For a description of the other editions refer to [Binary releases](./README.md#binary-releases). ~~~ mkdir -p build cd build # If you later want to generate a deb-package, it is recommended # to use /usr as prefix: -DCMAKE_INSTALL_PREFIX=/usr cmake -DSHOURNAL_EDITION=full .. make # as root: make install # or if using a Debian-based distribution, generate a .deb-package: cpack -G DEB ~~~ * To also compile unit- and integration-tests, run cmake with `-Dtest=ON` (debugging symbols via `-DCMAKE_BUILD_TYPE=Debug`). This generates a `test/runTests` binary in the build dir. Without any arguments, it runs unit-tests. Integrations-tests are, for instance, executed using: ~~~ # SHOURNAL_BACKEND=fanotify|ko SHOURNAL_BACKEND=fanotify test/runTests --integration --shell 'bash -i' ~~~ **After compile and install**:
If you created a .deb-package, please see [Binary releases](./README.md#binary-releases). **Otherwise:** **Kernel-module backend**
For a quick test, the module can be loaded right from the build-tree: `$ insmod kernel/shournalk.ko`.
To install the kernel-module (not built in SHOURNAL_EDITION's *docker* and *fanotify*) it is recommended to install it using dkms, e.g.: ~~~ dkms build shournalk/2.4 # adjust version as needed. dkms install shournalk/2.4 # and load it with modprobe shournalk ~~~ Depending on your distribution the dkms service may be disabled, thus after a kernel-update shournal stops working. At least on Opensuse Tumbleweed it can be enabled with ~~~ systemctl enable dkms ~~~ **fanotify backend**
Add a group to your system, which is primarily needed for the shell-integration: ```groupadd shournalmsenter``` However, *do not add any users to that group*. It is part of a permission check, where root adopts that gid (within shournal). If you don't like the default group name, you can specify your own: at build time pass the following to cmake: ```-DMSENTER_GROUPNAME=$your_group_name``` For **further post-install steps** please see [Binary releases](./README.md#binary-releases). Please note that file-paths may need to be adjusted, e.g. the location of the `SOURCE_ME.$shell_name` scripts after `make install` is typically `/usr/local/share/shournal/`, not `/usr/share/shournal/`. To **uninstall**, after having installed with `make install`, you can execute
`xargs rm < install_manifest.txt`, but see [here](https://stackoverflow.com/a/44649542/7015849) for the limitations.
To uninstall the kernel-module backend:
`sudo dkms remove shournalk/2.4` (adjust version as needed). ================================================ FILE: README-shell-integration.md ================================================ # Shell integration for shournal ## Basic setup (interactive) After installation, to start observing your *interactive* shell-sessions append the following to your shell's rc:
**~/.bashrc**
~~~ HISTCONTROL=ignoredups:erasedups # NOT ALLOWED: ignorespace,ignoreboth source /usr/share/shournal/SOURCE_ME.bash SHOURNAL_ENABLE ~~~ **~/.zshrc**
~~~ source /usr/share/shournal/SOURCE_ME.zsh SHOURNAL_ENABLE ~~~ Launch a new shell afterwards and check whether it's working: ~~~ $ echo foo > bar $ shournal --query --wfile bar cmd-id 66075 $?=0 2021-11-02 14:23 $ echo foo > bar Working directory: /home/tycho session-uuid 3hIZtDwhEey5WPDVv9W/Cw== 1 written file: /home/tycho/bar (4 bytes) Hash: 8087352826690557229 $ # or just look into the history: $ shournal --query --history 3 # ... ~~~ The shell-integration injects code into `PROMPT_COMMAND`, `PS0` and `PS1` (bash) or the `preexec/precmd_functions` (zsh), so please do not overwrite those after having enabled shournal. Further basic history functionality must be available, e.g. in bash HISTCONTROL must not ignore commands with leading spaces (see above). shournal's shell integration checks the typical variables and gives hints, if there is need for action. Other commands include
`SHOURNAL_DISABLE` to disable the observation
`SHOURNAL_PRINT_VERSIONS` to print the version of each component
`SHOURNAL_SET_VERBOSITY` to change the default verbosity ("dbg, info, warning, critical"). For dbg, shournal must have been compiled with debugging symbols. A verbosity higher than *warning* is not recommended. ## Advanced setup (non-interactive) To also observe non-interactive commands executed via ssh ~~~ ssh localhost echo foo ~~~ or the *Sun Grid Engine* (SGE) the following setup is recommended:

**bash**
Put the following near the **beginning** of your bashrc: ~~~ if [[ -n ${SGE_O_WORKDIR+x} || ( -n ${BASH_EXECUTION_STRING+x} && ( -n ${SSH_CLIENT+x} || -n ${SSH_TTY+x} ) ) ]]; then source /usr/share/shournal/SOURCE_ME.bash SHOURNAL_ENABLE fi ~~~ In particular that code has to run before the sourcing of ~/.bashrc stops due to a negative interactive-check. For example, some distributions place the following near the top of the bashrc: ~~~ case $- in *i*) ;; *) return;; esac ~~~ **zsh**
Put the following into ~/.zprofile ~~~ if [[ -n ${SGE_O_WORKDIR+x} || ( -n ${ZSH_EXECUTION_STRING+x} && ( -n ${SSH_CLIENT+x} || -n ${SSH_TTY+x} ) ) ]]; then source /usr/share/shournal/SOURCE_ME.zsh SHOURNAL_ENABLE fi ~~~ Note that depending on your server environment, this requires zsh to be executed as login shell, e.g.
`ssh HOST zsh -l -c command`. Alternatively you may use ~/.zshenv but beware that this file is always sourced, also during `zsh -c ':'` invocations on the interactive command-line, so at least an additional check for
`[ $SHLVL -eq 1 ]` is recommended. For cluster software systems other than SGE, you may `export SHOURNAL_IS_CLUSTERJOB=true`, before `SHOURNAL_ENABLE`, if and only if the shell is about to execute a cluster job. Note that in this case, shournal performs a re-execution of the current command and only returns control flow after flushing the database, because cluster software systems tend to kill background processes, once the main job script finished. To totally disable cluster job detection, set `SHOURNAL_NO_CLUSTER_JOB_DETECT=true` before `SHOURNAL_ENABLE`. ## Prerequisites of the fanotify backend If the *fanotify* backend is used, please ensure the following: * The shell must be linked dynamically against (g)libc (default case, can be tested e.g. with
`file $(which bash) | grep "dynamically linked"` ). * Sourcing of SOURCE_ME.$shell must be within the shell's rc-file. * `SHOURNAL_ENABLE` should be within the shell's rc, because on the very first enable the shell is re-executed, purging all non-exported variables. * For non-interactive commands `SHOURNAL_ENABLE` must be called before the actual execution begins. Note that the kernel module backend does not have those prerequisites and should be preferred in most cases. ## Updates If the shell-integration is running while shournal is updated, it is recommended, to restart your shell. A more elegant way than logout-login might be to `exec` your $shell. ## FAQ * **How to obtain the value of variables?**.
If shell-variables are used within a command, shournal's reports might not seem to be very helpful. However, the shell-integration assigns each shell-session a unique identifier (uuid). In the likely case that the variable was assigned *during that session*, you might be able to obtain its value. This of course only works, if SHOURNAL_ENABLE was called, *before* a variable was assigned. Example:
`shournal --query --shell-session-id 'L/932KZTEemRB/dOGB9LOA==' | grep var_name` * **What about new, nested shell-sessions**?
By *new shell-sessions* it is meant to call e.g. `bash` within an already running bash-process. What happens next really depends on whether the shell is itself **observed** by shournal or not (e.g. whether `SHOURNAL_ENABLE` is within the .bashrc or not). On calling `SHOURNAL_ENABLE` file-events are then considered to belong to the new shell-session and are no longer reported to the original observation-process of the caller. If a **non-observed** shell is a called, shournal's later report will not be very helpful: all file-modifications caused by that process will yield the plain shell-command (and not individual commands possibly entered within the new shell session). ## Limitations * File-operations (redirections) which spread over **multiple** command-sequences within the **interactive shell** might lead to surprising (*kernel module backend*) or incorrect (*fanotify backend*) results.
Example: ~~~ $ exec 3> /tmp/foo # open fd 3 $ echo "test" >&3 $ exec 3>&- # close fd 3. ~~~ In case of the *kernel module backend* as usual the close event is tracked, however `shournal -q -wf /tmp/foo` prints only the command `exec 3>&-`. By using the shell-session uuid it should be possible to reconstruct those cases.
In case of the *fanotify backend* the close-event is lost. * **Additional limitations of the fanotify-backend**:
Filesystem-events of asynchronously launched processes, which close the inherited shournal-socket, might be lost, because an external shournal-run process waits until all instances of that socket are closed. Steps to reproduce: In an *observed* shell-session enter
`bash -c 'eval "exec $_SHOURNAL_SOCKET_NB>&-"; sleep 1; echo foo > bar' &`
Note that e.g. in *Python* processes launched via its *subprocess*-module do not inherit file descriptors by default. There seems to be no general solution to this problem, but in most cases there should be some mechanism to wait for the processes to finish, within the interactive shell-session or a script. * For further limitations please also read the general [README](/../../). ## Motivation For a general introduction about the data and meta-data *shournal* stores please visit the general [README](/../../). Having to type *shournal* before every single command one wants to observe can be tiresome. Another typing-overhead would be introduced by using pipes or redirections. Consider the following **broken** example: shournal --exec echo hi > foo # Don't do this. As many shell users know the redirection applies to the whole command, while shournal itself only observes "echo hi". The file modification event ('hi' written to 'foo') is hence **not** tracked by shournal. To actually observe such a command one must rather type shournal --exec sh -c 'echo hi > foo' That's annoying, right? Therefore before observing one or multiple commands, `source` the respective integration-file within your shell's rc (e.g. .bashrc) and type SHOURNAL_ENABLE That's (almost) all. Forget about *shournal* until needed ( e.g. you want to know how a certain file was created). ================================================ FILE: README.md ================================================ ![shournal logo](images/shournal.png) ## A (file-) journal for your shell **Log shell-commands and used files. Snapshot executed scripts. Fully automatic.**
*There are two kinds of people: those who backup, and those who have never lost their data.*
~~~ $ SHOURNAL_ENABLE $ cat demo.sh #!/usr/bin/env bash echo hi | tee out.log $ ./demo.sh hi $ shournal -q --wfile out.log cmd-id 2 $?=0 2022-11-08 08:46 $ ./demo.sh Working directory: /home/user 1 written file: /home/user/out.log (3 bytes) Hash: 15349503233279147316 1 read file: /home/user/demo.sh (42 bytes) Hash: 13559791986335963073 id 1 #!/usr/bin/env bash echo hi | tee out.log ~~~ ***shournal* records that `out.log` was written by the command `./demo.sh` and created a backup of the script `demo.sh` because it was read by the bash interpreter.** *shournal* does not guess the files - it asks the Linux kernel. It's fast enough, see [Overhead](#overhead). After installation and easy setup of the [shell-integration](./README-shell-integration.md) the following questions may be answered within seconds: * What files were modified, read or executed by a command? Or reverse: What shell-commands were used to create/modify or read from a certain file? * You executed a script. What was the script-content by the time it was called? * The command read a config-file - which one, and what was in it? * The command ran for a long time - can a re-execution be avoided (s. `--stat`)? * What other commands were executed during the same shell-session? * What about working directory, command start- and end-time or the exit status ($?) ? Besides output on the command-line in a human-readable format (or JSON) you can export (parts of) your command-history into a standalone html-file where it is displayed in an interactive time-line-plot. Further miscellaneous statistics are displayed in bar-plots, e.g. the commands with most file-modifications. Using the external software [shournal-to-snakemake]( https://github.com/snakemake/shournal-to-snakemake) an observed shell-command-series can be directly transformed into rules for the [*Snakemake workflow engine*](https://github.com/snakemake/snakemake), a tool to *create reproducible and scalable data analyses*. *shournal* runs on GNU/Linux or Microsoft Windows via the Windows Subsystem for Linux (WSL) using its *fanotify* edition. For a more formal description please also check out our paper
Kirchner, T., Riege, K. & Hoffmann, S. *Bashing irreproducibility with shournal*. Sci Rep 14, 4872 (2024). https://doi.org/10.1038/s41598-024-53811-9
![Example session animation](images/shournal-example-session.gif) ## Examples Please note: below examples make use of the [shell-integration](./README-shell-integration.md).
* Create a file and ask shournal, how it came to be: ~~~ $ SHOURNAL_ENABLE # monitor all commands using the shell-integration $ echo hi > foo $ shournal --query --wfile foo cmd-id 1 $?=0 2019-05-14 10:19 $ echo hi > foo 1 written file: /home/user/foo (3 bytes) Hash: 15349503233279147316 ~~~ * shournal can be configured, to store *specific* read files, like shell-scripts, within it's database. Sometimes old script versions are of interest. Query by **read filename** and optionally restore the files with `--restore-rfiles`: ~~~ $ shournal -q --rname demo.sh cmd-id 34 $?=0 2022-04-21 15:15 $ ./demo.sh 1 read file: /home/user/demo.sh (34 bytes) Hash: 16696055267278105544 id 3 #!/usr/bin/env bash echo version1 cmd-id 35 $?=0 2022-04-21 15:15 $ ./demo.sh 1 read file: /home/user/demo.sh (34 bytes) Hash: 17683376525180966954 id 4 #!/usr/bin/env bash echo version2 $ shournal -q --rname demo.sh --restore-rfiles # restore read files ... 2 file(s) restored at /tmp/shournal-restore-user ~~~ * List all commands which contained the string `demo` (% is wildcard): ~~~ $ shournal -q -cmdtxt %demo% cmd-id 1 $?=0 2022-04-20 15:46 $ cat demo.sh ... cmd-id 2 $?=0 2022-04-20 15:46 $ ./demo.sh ... ~~~ * Are input files up to date, such that re-execution of the command can be avoided? Add `--stat` to the query, reporting current file statuses as U (up to date), M (modified), N (not exist) ERROR (in case of an error) or NA (not queried, only using json). ~~~ $ cat foo > bar $ shournal -q -wf bar --stat cmd-id 1 $?=0 2025-02-10 11:38-11:38 $ cat foo > bar ... 1 written file: /home/tycho/bar (3 bytes) Hash: 15349503233279147316 U 1 read file: /home/tycho/foo (3 bytes) Hash: 15349503233279147316 id 404002 U ~~~ To query only for changed input files, execute ~~~ shournal -q -wf bar --stat --output-format json | grep -F 'COMMAND:' | \ sed -n 's/COMMAND://p' | \ jq -r '.fileReadEvents | .[] | .status + " " + .path' | grep -v ^U ~~~ * What commands were executed at the current working directory? ~~~ shournal --query -cwd "$PWD" ~~~ * What commands were executed within a specific shell-session? The uuid can be taken from the command output of a previous query. ~~~ shournal --query --shell-session-id $uuid ~~~ * Find out the value of a variable. For instance, the command `echo $foo > bar` was executed in the shell-session with id `puLvkEizEe6CgvXjQlmnIQ==`. If `foo` was set within that shell session, its value can often be retrieved by ~~~ shournal -q -sid puLvkEizEe6CgvXjQlmnIQ== | fgrep 'foo=' ~~~ * For the full list of query-options, please enter ~~~ shournal --query --help ~~~ Instead of printing the `--query`-results to terminal, you can also create fancy html-plots, by appending `--output-format html -o out.html`. Use an ordinary web-browser for display. ## Installation ### Binary releases For **Debian/Ubuntu-based** distributions .deb-packages are available on the [release-page](https://github.com/tycho-kirchner/shournal/releases/latest). Three different editions are provided for different use-cases: most users will want to install *shournal* on a real host (or virtual machine) and *shournal-docker* [inside Docker](#running-inside-docker) (or another container platform). *shournal-fanotify* does not contain the kernel backend and is targeted at institutions where the usage of *out-of-tree kernel-modules* is discouraged.
Only LTS-releases are officially supported, the packages are known to work from Debian 10 (Buster) and Ubuntu 18.04 (Bionic) onwards. Before installing *shournal* including its kernel backend, make sure, the kernel headers are installed:
**Ubuntu**: `apt install linux-headers-generic`
**Ubuntu** with [HWE](https://askubuntu.com/questions/248914/what-is-hardware-enablement-hwe): `apt install linux-headers-generic-hwe-$(lsb_release -rs)`
**Debian**: `apt install linux-headers-$(dpkg --print-architecture)`
Install deb-packages as usual, e.g.
`sudo apt install ./shournal_2.2_amd64.deb`
To enable the shell-integration: * for *bash*: put the following to the end of your ~/.bashrc
`source /usr/share/shournal/SOURCE_ME.bash`
* for *zsh*: put the following to the end of your ~/.zshrc
`source /usr/share/shournal/SOURCE_ME.zsh`
and run `SHOURNAL_ENABLE` afterwards. For **any Linux**, a flat binary is available on the [release-page](https://github.com/tycho-kirchner/shournal/releases/latest) to be used without installation: ~~~ tar -xf shournal-fanotify*.tar.xz cd shournal-fanotify/ sudo groupadd shournalmsenter sudo chown root shournal-run-fanotify && sudo chmod u+s shournal-run-fanotify ./shournal-run-fanotify -e echo Hello World # Source shournal's shell integration from bashrc/zshrc, e.g. # echo "source '$PWD/SOURCE_ME.bash'" >> ~/.bashrc # echo "source '$PWD/SOURCE_ME.zsh'" >> ~/.zshrc # Enable with: SHOURNAL_ENABLE. ~~~ An **update** of *shournal* should be performed after all users have logged out, because the shell integrations need to be resourced. Further in case of the *kernel module* backend unloading the old version stops all running observations. **After installation**: Depending on your distribution, additional steps might be necessary to enable the (recommended) uuidd-daemon. If systemd is in use, one may need to: systemctl enable uuidd systemctl start uuidd Add yourself or other users to the group *shournalk*:
`sudo adduser $USER shournalk` (relogin to take affect).
You may override this group: ~~~ mkdir -p /etc/shournal.d/ echo GROUPNAME > /etc/shournal.d/kgroup ~~~ replacing GROUPNAME with the value of your choice. This rule takes into effect the next time shournal's kernel module is loaded ( so call e.g. `modprobe -r shournalk; modprobe shournalk` or reboot). More details and advanced options (logging commands executed via ssh) can be found [here](./README-shell-integration.md). ### Compile and install from source Please refer to the instructions found within the [compile-README](./README-compile.md). ## FAQ * **Does shournal track file rename/move operations?**
No, but most often it should not be a problem. Using the `--wfile` commandline-query-option, shournal finds the stored command by content (size, hash) and mtime, not by its name. For the name, `--wname` can be used. More concrete: ~~~ shournal --exec sh -c 'echo foo > bar; mv bar bar_old' ~~~ Querying for bar_old by content (`--wfile`-option) yields exactly the given command, however, `--wname bar_old` does **not** work (`--wname bar` of course works). To use the bar_old *file name* (and not content) as basis for a successful query, in this case `--command-text -like '%bar_old%'` can be used. * **What happens to an appended file?**
How to get a "modification history"? Please read above rename/move-text first. Appending to a file is currently handled as if a new one was created - only the last command, which modified a given file can be found with good certainty (by file **content**). However, querying by path/file**name** works. If the file was appended *and* renamed, things get more complicated. * **To track files, they can be hashed. Is that slow for big files?**
No, because per default only certain small parts of the file are hashed. * **What does the following message mean and how to get rid of it?**:
`fanotify_mark: failed to add path /foobar ... Permission denied`. This message might be printed on executing a command with shournal. Most probably the administrator mounted a filesystem object for which you don't have permissions, thus you cannot *monitor* file events. In this case you cannot perform file operations at this path anyway, so it should be safe to silence this warning by adding the path within the config-file in section `[mounts]`. If you want to ignore all fanotify_mark permission errors, you can set the flag in section `[mounts]`: ~~~ [mounts] ignore_no_permission = true ~~~ ## Configuration shournal stores a self-documenting config-file typically at ~/.config/shournal which is created on first run. It can be edited either directly with a plain text editor or via `--edit-cfg`. For completeness, the most important points are listed here as well. * Write- and read events can be configured, so only events occurring at specific (include-)paths are stored. Put each path into a separate line, all paths being enclosed by triple quotes: ~~~ include_paths = ''' /home/me /media ''' ~~~ Each exclude_path should be a sub-path of an include path. * Note that by default, there is a limit on the number of logged events per command (max_event_count). Read files (e.g. scripts) can **further** be configured to be stored within shournal's database. Files are only stored, if the configured max. file-size, file extension (e.g. sh) and mimetype (e.g. application/x-shellscript) matches. To find a mimetype for a given file you should use
`shournal --print-mime test.sh`. The correspondence of mimetype and file extension is explained in more detail within the config-file. Further, at your wish, read files are only stored if *you* have write permission for them (not only read) - often system-provided scripts (owned by root) are not of particular interest. shournal will not store more read files per command, than max_count_of_files. Matching files coming first have precedence. ## Running inside Docker To use *shournal* within Docker (or another container platform), depending on the backend the following steps are necessary:
**kernel module backend**
Install *shournal* on the host and *shournal-docker* inside the container. For *unprivileged* containers *sysfs* is mounted readonly. In this case create a bindmount from /sys/kernel/shournalk_root to /tmp/shournalk-sysfs, e.g.
`docker run ... -v /sys/kernel/shournalk_root:/tmp/shournalk-sysfs`. **fanotify backend**
Install *shournal-docker* (or *shournal-fanotify*) inside docker. For *unprivileged* containers the capabilities SYS_ADMIN, SYS_PTRACE and SYS_NICE are required, e.g.
`docker run ... --cap-add SYS_ADMIN --cap-add SYS_PTRACE --cap-add SYS_NICE`.
You may need to [configure the backend](#backend-configuration). ## Running on a Amazon AWS EC2 instance In order to run *shournal* on a Amazon AWS EC2 instance it may be necessary, to enable additional software package repositories. For Ubuntu 22.04 on a t3.micro instance enter the following commands before installing *shournal* ~~~ sudo add-apt-repository universe sudo apt update ~~~ ## Backend configuration shournal provides two backends, a custom *kernel module* and *fanotify*. The *kernel module* is used by default, except the *shournal-fanotify* edition is installed, where only the *fanotify* backend is available. In general it is recommended to stick with the *kernel module* as it is faster and has less interference with the process environment - for example no new mount namespaces have to be created and no file descriptor inheritance is necessary to wait for the end of a process tree. See also: [shell-integration](./README-shell-integration.md#limitations).
If both backends are installed you may configure the default one globally by creating the file `/etc/shournal.d/backend` or for each user by creating `~/.config/shournal/backend` with content `ko` or `fanotify`. ## Disk-space - get rid of obsolete file-events Depending on the file-activity of the observed commands, shournal's database will sooner or later grow. When you feel that enough time has passed and want to get rid of old events, this can be done by e.g. `shournal --delete --older-than 1y` which deletes all commands (and file-events) older than one year. More options are available, see also `shournal --delete --help` ## Remote file-systems * *shournal* is able to monitor file events of specific processes (PID's). Therefore, remote filesystems such as NFS or sshfs can be observed as long as *shournal* runs on the same (virtual) machine as the observed process. Consequently file events *another kernel* performs are lost. * For sshfs in case of the *fanotify* backend it is necessary, to add ```-o allow_root``` to the sshfs-options, otherwise permission errors during ```fanotify_mark``` are raised. See also: https://serverfault.com/a/188896 ## Security ### kernel-module backend In the kernel module it is ensured that each user is only allowed to monitor his/her own processes. Further, the kernel thread, which processes file events, runs with effective caller credentials and checks allowed accesses on a per-file basis. Memory allocations are cgroup-aware, even for reading (in case of hashing) and writing (in case of logging) files. ### fanotify backend *shournal-run-fanotify* is a so called "setuid"-program: whenever a regular user calls it, it runs with root-permissions in the first place. As soon as possible, it runs effectively with user permissions though. It must be setuid for two reaons: * fanotify requires root for initializing, because it is in principle able, to **forbid** a process to access a file. shournal does not make use of this feature so this is not a real security concern. * unsharing the *mount namespace* requires root, because setuid-programs *could* still refer to seemingly obsolete mounts. This means that under awkward circumstances an unmount-event, which has security-relevant consequences (e.g. mounting a new file to /etc/shadow) might not propagate to processes which run in other mount namespaces. To make sure mount-propagation applies, **all mounts, which carry setuid-binaries or files they refer to, should be mounted *shared***, or no (security-relevant) mount/unmount events should occur, after the first shournal-process started. Shared mounts are the default in all recent distributions I know of. See also man 7 mount_namespaces and [shared subtrees](https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt). ## Limitations Processes can communicate via IPC (inter-process-communication). If the observed process *A* instructs the **not** observed process *B* via IPC to modify a file, the filesystem-event is not registered by *shournal*. For performance reasons, all files opened with write-permissions are reported as *written* by shournal, irrespective of whether the process actually wrote to it. By using file size and content (hash) you should be able to cover those cases. The provided timestamp is determined shortly after a file was closed. Note that it is possible that some other process has written to it in between. This however is only a problem, if that other process was itself **not** observed. Whether memory mapped (see mmap(2) ) file-events are reported correctly depends on **when** the underlying file-descriptor is closed. It is thus application dependent and does not work in general. ### Additional limitations of the fanotify backend The file observation only works, if the process does not unshare the mount-namespace itself, e.g. monitoring a program started via *flatpak* fails. For further limitations please visit the fanotify manpage. ## Known Issues * on NFS-storages: file events are lost, if the user does not have read-permissions while a file is closed. Steps to reproduce: - open a file readable for you on a NFS storage - chmod it 000 - close it --> the event is lost ## How does it work? shournal attempts to deterministically associate files and shell- commands without changing the users workflow. Under Linux file operations are performed by the kernel, tracing these operations thus requires OS-level support. During the execution of a shell-command, shournal instruments the kernel to trace files used by the shell-process and any of it’s descendant processes. More particular, to keep the tracing-overhead low, only the closing of files is traced and (meta-)data collection starts afterwards in an asynchronous manner. **shournalk** as a kernel module runs directly in *kernel space* and is based on [tracepoints](https://www.kernel.org/doc/html/latest/trace/tracepoints.html) and the [ftrace-framework](https://www.kernel.org/doc/Documentation/trace/ftrace.txt) which basically allow for custom code to be run at certain kernel execution paths without recompilation of the kernel itself. Only three events are traced: closing of files, fork and exit. (Meta-)data collection also takes place entirely in kernel space. The **fanotify backend** employs the kernel-native [fanotify filesystem API]( https://man7.org/linux/man-pages/man7/fanotify.7.html) to register for close-events of whole mount-points which are isolated against unrelated processes using unshared [mount namespaces](https://man7.org/linux/man-pages/man7/mount_namespaces.7.html). shournal thereby ensures that all file-operations during the execution of a shell- command refer to the same, unique mount namespace. While the process-filtering takes place in kernel space — so only file-events of observed processes are copied to user-space — the (meta-)data collection happens in user space. ## Overhead File tracing imposes a **runtime overhead**. For a detailed performance evaluation please refer to our [paper](https://doi.org/10.1038/s41598-024-53811-9) . In brief: We measured the following command executions with shournal v2.9: * compile elfutils-0.176 * git checkout — checkout the Linux kernel’s source code from v4.19 to v3.10. * kernel copy — cp of the 4.19 Linux source. The relative runtime-overheads are shown in below table, strace is listed for comparison with ptrace-based solutions: | Backend | compile | checkout | cp | | ------------- | ------- | -------- | ----- | | kernel module | 0.05% | 0.49% | 0.29% | | fanotify | 1.2% | 1.3% | 6.2% | | (strace) | 140% | 41% | 100% | The benchmark involves tracing, (meta-)data collection and saving to a binary temporary file. As this file can be kept indefinitely, the final storing into the SQL-database is not part of the runtime-measurement. For the `cp` benchmark, where ~120.000 file-events occurred in ~4 seconds, the runtime overhead of the fanotify backend may become noticeable. Note that many file-events in short time constitute a worst-case. Where performance is critical, the kernel module backend should be used. The **storage overhead** largely depends on configuration, e.g. the number of stored scripts and file-metadata is limited by default, to avoid e.g. a backup-script from flooding the database. For the cp-test the average disk-usage per file-event is approx. 174 bytes which already includes indexes to speed up queries. So one GiB of disk-space is sufficient for approx. 6 million events. Based on the experience of real-world users the database is typically not larger than a few hundred megabytes after months of usage. ## Credits shournal makes use of great tools and libraries, most importantly the Qt-framework, xxhash, tsl::ordered_map and cmake and also the Linux-Kernel's *fanotify*. For the html-plot d3js, jquery, popper.js, bootstrap, webpack and others are used. Thanks to the developers! The project arose in the Hoffmann Research Group: Computational Biology of Aging at the Fritz Lipmann Institute in Jena (Germany). Special thanks to Steve Hoffmann and Konstantin Riege - without you this project couldn't have been accomplished. # License The whole project is licensed under the GPL, v3 or later (see LICENSE file for details)
**except** * The kernel module within `kernel/` which is licensed under the GNU General Public License version 2 only. * The libraries within `extern/` → Please refer to the licenses within their respective directories. * The javascript-libraries in the auto-generated `html-export/dist/main.js` → the licenses are stored in `html-export/dist/main.licenses.txt`. Copyleft (C) 2021, Tycho Kirchner ================================================ FILE: cmake/FindShournalUtil.cmake ================================================ # Join a list of strings using seperator sep # and store the output in result. function(JOIN vals sep result) string (REGEX REPLACE "([^\\]|^);" "\\1${sep}" _tmp_str "${vals}") string (REGEX REPLACE "[\\](.)" "\\1" _tmp_str "${_tmp_str}") set (${result} "${_tmp_str}" PARENT_SCOPE) endfunction() ================================================ FILE: extern/folly/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS Files in folly/external/farmhash licensed as follows Copyright (c) 2014 Google, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: extern/folly/README.md ================================================ Folly: Facebook Open-source Library ----------------------------------- [![Build Status](https://travis-ci.org/facebook/folly.svg?branch=master)](https://travis-ci.org/facebook/folly) ### What is `folly`? Folly (acronymed loosely after Facebook Open Source Library) is a library of C++14 components designed with practicality and efficiency in mind. **Folly contains a variety of core library components used extensively at Facebook**. In particular, it's often a dependency of Facebook's other open source C++ efforts and place where those projects can share code. It complements (as opposed to competing against) offerings such as Boost and of course `std`. In fact, we embark on defining our own component only when something we need is either not available, or does not meet the needed performance profile. We endeavor to remove things from folly if or when `std` or Boost obsoletes them. Performance concerns permeate much of Folly, sometimes leading to designs that are more idiosyncratic than they would otherwise be (see e.g. `PackedSyncPtr.h`, `SmallLocks.h`). Good performance at large scale is a unifying theme in all of Folly. ### Logical Design Folly is a collection of relatively independent components, some as simple as a few symbols. There is no restriction on internal dependencies, meaning that a given folly module may use any other folly components. All symbols are defined in the top-level namespace `folly`, except of course macros. Macro names are ALL_UPPERCASE and should be prefixed with `FOLLY_`. Namespace `folly` defines other internal namespaces such as `internal` or `detail`. User code should not depend on symbols in those namespaces. Folly has an `experimental` directory as well. This designation connotes primarily that we feel the API may change heavily over time. This code, typically, is still in heavy use and is well tested. ### Physical Design At the top level Folly uses the classic "stuttering" scheme `folly/folly` used by Boost and others. The first directory serves as an installation root of the library (with possible versioning a la `folly-1.0/`), and the second is to distinguish the library when including files, e.g. `#include `. The directory structure is flat (mimicking the namespace structure), i.e. we don't have an elaborate directory hierarchy (it is possible this will change in future versions). The subdirectory `experimental` contains files that are used inside folly and possibly at Facebook but not considered stable enough for client use. Your code should not use files in `folly/experimental` lest it may break when you update Folly. The `folly/folly/test` subdirectory includes the unittests for all components, usually named `ComponentXyzTest.cpp` for each `ComponentXyz.*`. The `folly/folly/docs` directory contains documentation. ### What's in it? Because of folly's fairly flat structure, the best way to see what's in it is to look at the headers in [top level `folly/` directory](https://github.com/facebook/folly/tree/master/folly). You can also check the [`docs` folder](folly/docs) for documentation, starting with the [overview](folly/docs/Overview.md). Folly is published on GitHub at https://github.com/facebook/folly ### Build Notes #### Dependencies folly supports gcc (5.1+), clang, or MSVC. It should run on Linux (x86-32, x86-64, and ARM), iOS, macOS, and Windows (x86-64). The CMake build is only tested on some of these platforms; at a minimum, we aim to support macOS and Linux (on the latest Ubuntu LTS release or newer.) folly requires a version of boost compiled with C++14 support. googletest is required to build and run folly's tests. You can download it from https://github.com/google/googletest/archive/release-1.8.0.tar.gz The following commands can be used to download and install it: ``` wget https://github.com/google/googletest/archive/release-1.8.0.tar.gz && \ tar zxf release-1.8.0.tar.gz && \ rm -f release-1.8.0.tar.gz && \ cd googletest-release-1.8.0 && \ cmake . && \ make && \ make install ``` #### Finding dependencies in non-default locations If you have boost, gtest, or other dependencies installed in a non-default location, you can use the `CMAKE_INCLUDE_PATH` and `CMAKE_LIBRARY_PATH` variables to make CMAKE look also look for header files and libraries in non-standard locations. For example, to also search the directories `/alt/include/path1` and `/alt/include/path2` for header files and the directories `/alt/lib/path1` and `/alt/lib/path2` for libraries, you can invoke `cmake` as follows: ``` cmake \ -DCMAKE_INCLUDE_PATH=/alt/include/path1:/alt/include/path2 \ -DCMAKE_LIBRARY_PATH=/alt/lib/path1:/alt/lib/path2 ... ``` #### Building tests By default, building the tests is disabled as part of the CMake `all` target. To build the tests, specify `-DBUILD_TESTS=ON` to CMake at configure time. #### Ubuntu 16.04 LTS The following packages are required (feel free to cut and paste the apt-get command below): ``` sudo apt-get install \ g++ \ cmake \ libboost-all-dev \ libevent-dev \ libdouble-conversion-dev \ libgoogle-glog-dev \ libgflags-dev \ libiberty-dev \ liblz4-dev \ liblzma-dev \ libsnappy-dev \ make \ zlib1g-dev \ binutils-dev \ libjemalloc-dev \ libssl-dev \ pkg-config \ libunwind-dev ``` Folly relies on [fmt](https://github.com/fmtlib/fmt) which needs to be installed from source. The following commands will download, compile, and install fmt. ``` git clone https://github.com/fmtlib/fmt.git && cd fmt mkdir _build && cd _build cmake .. make -j$(nproc) sudo make install ``` If advanced debugging functionality is required, use: ``` sudo apt-get install \ libunwind8-dev \ libelf-dev \ libdwarf-dev ``` In the folly directory (e.g. the checkout root or the archive unpack root), run: ``` mkdir _build && cd _build cmake .. make -j $(nproc) make install # with either sudo or DESTDIR as necessary ``` #### OS X (Homebrew) folly is available as a Formula and releases may be built via `brew install folly`. You may also use `folly/build/bootstrap-osx-homebrew.sh` to build against `master`: ``` ./folly/build/bootstrap-osx-homebrew.sh ``` This will create a build directory `_build` in the top-level. #### OS X (MacPorts) Install the required packages from MacPorts: ``` sudo port install \ boost \ cmake \ gflags \ git \ google-glog \ libevent \ libtool \ lz4 \ lzma \ openssl \ snappy \ xz \ zlib ``` Download and install double-conversion: ``` git clone https://github.com/google/double-conversion.git cd double-conversion cmake -DBUILD_SHARED_LIBS=ON . make sudo make install ``` Download and install folly with the parameters listed below: ``` git clone https://github.com/facebook/folly.git cd folly mkdir _build cd _build cmake .. make sudo make install ``` #### Windows (Vcpkg) folly is available in [Vcpkg](https://github.com/Microsoft/vcpkg#vcpkg) and releases may be built via `vcpkg install folly:x64-windows`. You may also use `vcpkg install folly:x64-windows --head` to build against `master`. #### Other Linux distributions - double-conversion (https://github.com/google/double-conversion) Download and build double-conversion. You may need to tell cmake where to find it. [double-conversion/] `ln -s src double-conversion` [folly/] `mkdir build && cd build` [folly/build/] `cmake "-DCMAKE_INCLUDE_PATH=$DOUBLE_CONVERSION_HOME/include" "-DCMAKE_LIBRARY_PATH=$DOUBLE_CONVERSION_HOME/lib" ..` [folly/build/] `make` - additional platform specific dependencies: Fedora >= 21 64-bit (last tested on Fedora 28 64-bit) - gcc - gcc-c++ - cmake - automake - boost-devel - libtool - lz4-devel - lzma-devel - snappy-devel - zlib-devel - glog-devel - gflags-devel - scons - double-conversion-devel - openssl-devel - libevent-devel - fmt-devel - libsodium-devel Optional - libdwarf-devel - elfutils-libelf-devel - libunwind-devel ================================================ FILE: extern/folly/UninitializedMemoryHacks.h ================================================ /* * Copyright (c) Facebook, Inc. and its affiliates. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #include #include namespace { // This struct is different in every translation unit. We use template // instantiations to define inline freestanding methods. Since the // methods are inline it is fine to define them in multiple translation // units, but the instantiation itself would be an ODR violation if it is // present in the program more than once. By tagging the instantiations // with this struct, we avoid ODR problems for the instantiation while // allowing the resulting methods to be inline-able. If you think that // seems hacky keep reading... struct FollyMemoryDetailTranslationUnitTag {}; } // namespace namespace folly { namespace detail { template void unsafeStringSetLargerSize(std::basic_string& s, std::size_t n); template void unsafeVectorSetLargerSize(std::vector& v, std::size_t n); } // namespace detail /* * This file provides helper functions resizeWithoutInitialization() * that can resize std::basic_string or std::vector without constructing * or initializing new elements. * * IMPORTANT: These functions can be unsafe if used improperly. If you * don't write to an element with index >= oldSize and < newSize, reading * the element can expose arbitrary memory contents to the world, including * the contents of old strings. If you're lucky you'll get a segfault, * because the kernel is only required to fault in new pages on write * access. MSAN should be able to catch problems in the common case that * the string or vector wasn't previously shrunk. * * Pay extra attention to your failure paths. For example, if you try * to read directly into a caller-provided string, make sure to clear * the string when you get an I/O error. * * You should only use this if you have profiling data from production * that shows that this is not a premature optimization. This code is * designed for retroactively optimizing code where touching every element * twice (or touching never-used elements once) shows up in profiling, * and where restructuring the code to use fixed-length arrays or IOBuf-s * would be difficult. * * NOTE: Just because .resize() shows up in your profile (probably * via one of the intrinsic memset implementations) doesn't mean that * these functions will make your program faster. A lot of the cost * of memset comes from cache misses, so avoiding the memset can mean * that the cache miss cost just gets pushed to the following code. * resizeWithoutInitialization can be a win when the contents are bigger * than a cache level, because the second access isn't free in that case. * It can be a win when the memory is already cached, so touching it * doesn't help later code. It can also be a win if the final length * of the string or vector isn't actually known, so the suffix will be * chopped off with a second call to .resize(). */ /** * Like calling s.resize(n), but when growing the string does not * initialize new elements. It is undefined behavior to read from * any element added to the string by this method unless it has been * written to by an operation that follows this call. * * Use the FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT(T) macro to * declare (and inline define) the internals required to call * resizeWithoutInitialization for a std::basic_string. * See detailed description of a similar macro for std::vector below. * * IMPORTANT: Read the warning at the top of this header file. */ template < typename T, typename = typename std::enable_if::value>::type> inline void resizeWithoutInitialization( std::basic_string& s, std::size_t n) { if (n <= s.size()) { s.resize(n); } else { // careful not to call reserve unless necessary, as it causes // shrink_to_fit on many platforms if (n > s.capacity()) { s.reserve(n); } detail::unsafeStringSetLargerSize(s, n); } } /** * Like calling v.resize(n), but when growing the vector does not construct * or initialize new elements. It is undefined behavior to read from any * element added to the vector by this method unless it has been written * to by an operation that follows this call. * * Use the FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT(T) macro to * declare (and inline define) the internals required to call * resizeWithoutInitialization for a std::vector. This must * be done exactly once in each translation unit that wants to call * resizeWithoutInitialization(std::vector&,size_t). char and unsigned * char are provided by default. If you don't do this you will get linker * errors about folly::detail::unsafeVectorSetLargerSize. Requiring that * T be trivially_destructible is only an approximation of the property * required of T. In fact what is required is that any random sequence of * bytes may be safely reinterpreted as a T and passed to T's destructor. * * std::vector has specialized internals and is not supported. * * IMPORTANT: Read the warning at the top of this header file. */ template < typename T, typename = typename std::enable_if< std::is_trivially_destructible::value && !std::is_same::value>::type> void resizeWithoutInitialization(std::vector& v, std::size_t n) { if (n <= v.size()) { v.resize(n); } else { if (n > v.capacity()) { v.reserve(n); } detail::unsafeVectorSetLargerSize(v, n); } } namespace detail { // This machinery bridges template expansion and macro expansion #define FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT_IMPL(TYPE) \ namespace folly { \ namespace detail { \ void unsafeStringSetLargerSizeImpl(std::basic_string& s, std::size_t); \ template <> \ inline void unsafeStringSetLargerSize( \ std::basic_string & s, \ std::size_t n) { \ unsafeStringSetLargerSizeImpl(s, n); \ } \ } \ } #if defined(_LIBCPP_STRING) // libc++ template struct MakeUnsafeStringSetLargerSize { friend void unsafeStringSetLargerSizeImpl( std::basic_string& s, std::size_t n) { // s.__set_size(n); (s.*Ptr__set_size)(n); (&s[0])[n] = '\0'; } }; #define FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT(TYPE) \ template void std::basic_string::__set_size(std::size_t); \ template struct folly::detail::MakeUnsafeStringSetLargerSize< \ FollyMemoryDetailTranslationUnitTag, \ TYPE, \ void (std::basic_string::*)(std::size_t), \ &std::basic_string::__set_size>; \ FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT_IMPL(TYPE) #elif defined(_GLIBCXX_STRING) && _GLIBCXX_USE_CXX11_ABI // libstdc++ new implementation with SSO template struct MakeUnsafeStringSetLargerSize { friend void unsafeStringSetLargerSizeImpl( std::basic_string& s, std::size_t n) { // s._M_set_length(n); (s.*Ptr_M_set_length)(n); } }; #define FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT(TYPE) \ template void std::basic_string::_M_set_length(std::size_t); \ template struct folly::detail::MakeUnsafeStringSetLargerSize< \ FollyMemoryDetailTranslationUnitTag, \ TYPE, \ void (std::basic_string::*)(std::size_t), \ &std::basic_string::_M_set_length>; \ FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT_IMPL(TYPE) #elif defined(_GLIBCXX_STRING) // libstdc++ old implementation template < typename Tag, typename T, typename A, A Ptr_M_rep, typename B, B Ptr_M_set_length_and_sharable> struct MakeUnsafeStringSetLargerSize { friend void unsafeStringSetLargerSizeImpl( std::basic_string& s, std::size_t n) { // s._M_rep()->_M_set_length_and_sharable(n); auto rep = (s.*Ptr_M_rep)(); (rep->*Ptr_M_set_length_and_sharable)(n); } }; #define FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT(TYPE) \ template std::basic_string::_Rep* std::basic_string::_M_rep() \ const; \ template void std::basic_string::_Rep::_M_set_length_and_sharable( \ std::size_t); \ template struct folly::detail::MakeUnsafeStringSetLargerSize< \ FollyMemoryDetailTranslationUnitTag, \ TYPE, \ std::basic_string::_Rep* (std::basic_string::*)() const, \ &std::basic_string::_M_rep, \ void (std::basic_string::_Rep::*)(std::size_t), \ &std::basic_string::_Rep::_M_set_length_and_sharable>; \ FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT_IMPL(TYPE) #elif defined(_MSC_VER) // MSVC template struct MakeUnsafeStringSetLargerSize { friend void unsafeStringSetLargerSizeImpl( std::basic_string& s, std::size_t n) { // _Eos method is public for _MSC_VER <= 1916, private after // s._Eos(n); (s.*Ptr_Eos)(n); } }; #define FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT(TYPE) \ template void std::basic_string::_Eos(std::size_t); \ template struct folly::detail::MakeUnsafeStringSetLargerSize< \ FollyMemoryDetailTranslationUnitTag, \ TYPE, \ void (std::basic_string::*)(std::size_t), \ &std::basic_string::_Eos>; \ FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT_IMPL(TYPE) #else #warning \ "No implementation for resizeWithoutInitialization of std::basic_string" #endif } // namespace detail } // namespace folly #if defined(FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT) FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT(char) FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT(wchar_t) #endif namespace folly { namespace detail { // This machinery bridges template expansion and macro expansion #define FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT_IMPL(TYPE) \ namespace folly { \ namespace detail { \ void unsafeVectorSetLargerSizeImpl(std::vector& v, std::size_t); \ template <> \ inline void unsafeVectorSetLargerSize( \ std::vector & v, \ std::size_t n) { \ unsafeVectorSetLargerSizeImpl(v, n); \ } \ } \ } #if defined(_LIBCPP_VECTOR) // libc++ template < typename Tag, typename T, typename A, A Ptr__end_, typename B, B Ptr__annotate_contiguous_container_> struct MakeUnsafeVectorSetLargerSize { friend void unsafeVectorSetLargerSizeImpl(std::vector& v, std::size_t n) { // v.__end_ += (n - v.size()); using Base = std::__vector_base>; static_assert( std::is_standard_layout>::value && sizeof(std::vector) == sizeof(Base), "reinterpret_cast safety conditions not met"); const auto old_size = v.size(); reinterpret_cast(v).*Ptr__end_ += (n - v.size()); // libc++ contiguous containers use special annotation functions that help // the address sanitizer to detect improper memory accesses. When ASAN is // enabled we need to call the appropriate annotation functions in order to // stop ASAN from reporting false positives. When ASAN is disabled, the // annotation function is a no-op. (v.*Ptr__annotate_contiguous_container_)( v.data(), v.data() + v.capacity(), v.data() + old_size, v.data() + v.size()); } }; #define FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT(TYPE) \ template struct folly::detail::MakeUnsafeVectorSetLargerSize< \ FollyMemoryDetailTranslationUnitTag, \ TYPE, \ TYPE*(std::__vector_base>::*), \ &std::vector::__end_, \ void (std::vector::*)( \ const void*, const void*, const void*, const void*) const, \ &std::vector::__annotate_contiguous_container>; \ FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT_IMPL(TYPE) #elif defined(_GLIBCXX_VECTOR) // libstdc++ template < typename Tag, typename T, typename A, A Ptr_M_impl, typename B, B Ptr_M_finish> struct MakeUnsafeVectorSetLargerSize : std::vector { friend void unsafeVectorSetLargerSizeImpl(std::vector& v, std::size_t n) { // v._M_impl._M_finish += (n - v.size()); (v.*Ptr_M_impl).*Ptr_M_finish += (n - v.size()); } }; #define FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT(TYPE) \ template struct folly::detail::MakeUnsafeVectorSetLargerSize< \ FollyMemoryDetailTranslationUnitTag, \ TYPE, \ decltype(&std::vector::_M_impl), \ &std::vector::_M_impl, \ decltype(&std::vector::_Vector_impl::_M_finish), \ &std::vector::_Vector_impl::_M_finish>; \ FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT_IMPL(TYPE) #elif defined(_MSC_VER) && _MSC_VER <= 1916 // MSVC <= VS2017 template struct MakeUnsafeVectorSetLargerSize : std::vector { friend void unsafeVectorSetLargerSizeImpl(std::vector& v, std::size_t n) { v._Mylast() += (n - v.size()); } }; #define FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT(TYPE) \ template struct folly::detail::MakeUnsafeVectorSetLargerSize< \ FollyMemoryDetailTranslationUnitTag, \ TYPE>; \ FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT_IMPL(TYPE) #elif defined(_MSC_VER) && _MSC_VER > 1916 // MSVC >= VS2019 template < typename Tag, typename T, typename A, A Ptr_Mypair, typename B, B Ptr_Myval2, typename C, C Ptr_Mylast> struct MakeUnsafeVectorSetLargerSize : std::vector { friend void unsafeVectorSetLargerSizeImpl(std::vector& v, std::size_t n) { // v._Mypair._Myval2._Mylast += (n - v.size()); ((v.*Ptr_Mypair).*Ptr_Myval2).*Ptr_Mylast += (n - v.size()); } }; #define FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT(TYPE) \ template struct folly::detail::MakeUnsafeVectorSetLargerSize< \ FollyMemoryDetailTranslationUnitTag, \ TYPE, \ decltype(&std::vector::_Mypair), \ &std::vector::_Mypair, \ decltype(&decltype(std::declval>()._Mypair)::_Myval2), \ &decltype(std::declval>()._Mypair)::_Myval2, \ decltype(&decltype( \ std::declval>()._Mypair._Myval2)::_Mylast), \ &decltype(std::declval>()._Mypair._Myval2)::_Mylast>; \ FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT_IMPL(TYPE) #else #warning "No implementation for resizeWithoutInitialization of std::vector" #endif } // namespace detail } // namespace folly #if defined(FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT) FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT(char) FOLLY_DECLARE_VECTOR_RESIZE_WITHOUT_INIT(unsigned char) #endif ================================================ FILE: extern/tsl-ordered-map/CMakeLists.txt ================================================ add_library(lib_orderedmap ordered_hash.h ordered_map.h ordered_set.h ) set_target_properties(lib_orderedmap PROPERTIES LINKER_LANGUAGE CXX) target_link_libraries(lib_orderedmap Qt5::Core ) ================================================ FILE: extern/tsl-ordered-map/LICENSE ================================================ MIT License Copyright (c) 2017 Tessil Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: extern/tsl-ordered-map/ordered_hash.h ================================================ /** * MIT License * * Copyright (c) 2017 Tessil * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef TSL_ORDERED_HASH_H #define TSL_ORDERED_HASH_H #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /** * Macros for compatibility with GCC 4.8 */ #if (defined(__GNUC__) && (__GNUC__ == 4) && (__GNUC_MINOR__ < 9)) # define TSL_OH_NO_CONTAINER_ERASE_CONST_ITERATOR # define TSL_OH_NO_CONTAINER_EMPLACE_CONST_ITERATOR #endif /** * Only activate tsl_oh_assert if TSL_DEBUG is defined. * This way we avoid the performance hit when NDEBUG is not defined with assert as tsl_oh_assert is used a lot * (people usually compile with "-O3" and not "-O3 -DNDEBUG"). */ #ifdef TSL_DEBUG # define tsl_oh_assert(expr) assert(expr) #else # define tsl_oh_assert(expr) (static_cast(0)) #endif /** * If exceptions are enabled, throw the exception passed in parameter, otherwise call std::terminate. */ #if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || (defined (_MSC_VER) && defined (_CPPUNWIND))) && !defined(TSL_NO_EXCEPTIONS) # define TSL_OH_THROW_OR_TERMINATE(ex, msg) throw ex(msg) #else # define TSL_OH_NO_EXCEPTIONS # ifdef NDEBUG # define TSL_OH_THROW_OR_TERMINATE(ex, msg) std::terminate() # else # include # define TSL_OH_THROW_OR_TERMINATE(ex, msg) do { std::cerr << msg << std::endl; std::terminate(); } while(0) # endif #endif namespace tsl { namespace detail_ordered_hash { template struct make_void { using type = void; }; template struct has_is_transparent: std::false_type { }; template struct has_is_transparent::type>: std::true_type { }; template struct is_vector: std::false_type { }; template struct is_vector>::value >::type>: std::true_type { }; template static T numeric_cast(U value, const char* error_message = "numeric_cast() failed.") { T ret = static_cast(value); if(static_cast(ret) != value) { TSL_OH_THROW_OR_TERMINATE(std::runtime_error, error_message); } const bool is_same_signedness = (std::is_unsigned::value && std::is_unsigned::value) || (std::is_signed::value && std::is_signed::value); if(!is_same_signedness && (ret < T{}) != (value < U{})) { TSL_OH_THROW_OR_TERMINATE(std::runtime_error, error_message); } return ret; } /** * Fixed size type used to represent size_type values on serialization. Need to be big enough * to represent a std::size_t on 32 and 64 bits platforms, and must be the same size on both platforms. */ using slz_size_type = std::uint64_t; static_assert(std::numeric_limits::max() >= std::numeric_limits::max(), "slz_size_type must be >= std::size_t"); template static T deserialize_value(Deserializer& deserializer) { // MSVC < 2017 is not conformant, circumvent the problem by removing the template keyword #if defined (_MSC_VER) && _MSC_VER < 1910 return deserializer.Deserializer::operator()(); #else return deserializer.Deserializer::template operator()(); #endif } /** * Each bucket entry stores an index which is the index in m_values corresponding to the bucket's value * and a hash (which may be truncated to 32 bits depending on IndexType) corresponding to the hash of the value. * * The size of IndexType limits the size of the hash table to std::numeric_limits::max() - 1 elements (-1 due to * a reserved value used to mark a bucket as empty). */ template class bucket_entry { static_assert(std::is_unsigned::value, "IndexType must be an unsigned value."); static_assert(std::numeric_limits::max() <= std::numeric_limits::max(), "std::numeric_limits::max() must be <= std::numeric_limits::max()."); public: using index_type = IndexType; using truncated_hash_type = typename std::conditional::max() <= std::numeric_limits::max(), std::uint_least32_t, std::size_t>::type; bucket_entry() noexcept: m_index(EMPTY_MARKER_INDEX), m_hash(0) { } bool empty() const noexcept { return m_index == EMPTY_MARKER_INDEX; } void clear() noexcept { m_index = EMPTY_MARKER_INDEX; } index_type index() const noexcept { tsl_oh_assert(!empty()); return m_index; } index_type& index_ref() noexcept { tsl_oh_assert(!empty()); return m_index; } void set_index(index_type index) noexcept { tsl_oh_assert(index <= max_size()); m_index = index; } truncated_hash_type truncated_hash() const noexcept { tsl_oh_assert(!empty()); return m_hash; } truncated_hash_type& truncated_hash_ref() noexcept { tsl_oh_assert(!empty()); return m_hash; } void set_hash(std::size_t hash) noexcept { m_hash = truncate_hash(hash); } template void serialize(Serializer& serializer) const { const slz_size_type index = m_index; serializer(index); const slz_size_type hash = m_hash; serializer(hash); } template static bucket_entry deserialize(Deserializer& deserializer) { const slz_size_type index = deserialize_value(deserializer); const slz_size_type hash = deserialize_value(deserializer); bucket_entry bentry; bentry.m_index = numeric_cast(index, "Deserialized index is too big."); bentry.m_hash = numeric_cast(hash, "Deserialized hash is too big."); return bentry; } static truncated_hash_type truncate_hash(std::size_t hash) noexcept { return truncated_hash_type(hash); } static std::size_t max_size() noexcept { return static_cast(std::numeric_limits::max()) - NB_RESERVED_INDEXES; } private: static const index_type EMPTY_MARKER_INDEX = std::numeric_limits::max(); static const std::size_t NB_RESERVED_INDEXES = 1; index_type m_index; truncated_hash_type m_hash; }; /** * Internal common class used by ordered_map and ordered_set. * * ValueType is what will be stored by ordered_hash (usually std::pair for map and Key for set). * * KeySelect should be a FunctionObject which takes a ValueType in parameter and return a reference to the key. * * ValueSelect should be a FunctionObject which takes a ValueType in parameter and return a reference to the value. * ValueSelect should be void if there is no value (in set for example). * * ValueTypeContainer is the container which will be used to store ValueType values. * Usually a std::deque or std::vector. * * * * The orderd_hash structure is a hash table which preserves the order of insertion of the elements. * To do so, it stores the values in the ValueTypeContainer (m_values) using emplace_back at each * insertion of a new element. Another structure (m_buckets of type std::vector) will * serve as buckets array for the hash table part. Each bucket stores an index which corresponds to * the index in m_values where the bucket's value is and the (truncated) hash of this value. An index * is used instead of a pointer to the value to reduce the size of each bucket entry. * * To resolve collisions in the buckets array, the structures use robin hood linear probing with * backward shift deletion. */ template class ordered_hash: private Hash, private KeyEqual { private: template using has_mapped_type = typename std::integral_constant::value>; static_assert(std::is_same::value, "ValueTypeContainer::value_type != ValueType. " "Check that the ValueTypeContainer has 'Key' as type for a set or 'std::pair' as type for a map."); static_assert(std::is_same::value, "ValueTypeContainer::allocator_type != Allocator. " "Check that the allocator for ValueTypeContainer is the same as Allocator."); static_assert(std::is_same::value, "Allocator::value_type != ValueType. " "Check that the allocator has 'Key' as type for a set or 'std::pair' as type for a map."); public: template class ordered_iterator; using key_type = typename KeySelect::key_type; using value_type = ValueType; using size_type = std::size_t; using difference_type = std::ptrdiff_t; using hasher = Hash; using key_equal = KeyEqual; using allocator_type = Allocator; using reference = value_type&; using const_reference = const value_type&; using pointer = value_type*; using const_pointer = const value_type*; using iterator = ordered_iterator; using const_iterator = ordered_iterator; using reverse_iterator = std::reverse_iterator; using const_reverse_iterator = std::reverse_iterator; using values_container_type = ValueTypeContainer; public: template class ordered_iterator { friend class ordered_hash; private: using iterator = typename std::conditional::type; ordered_iterator(iterator it) noexcept: m_iterator(it) { } public: using iterator_category = std::random_access_iterator_tag; using value_type = const typename ordered_hash::value_type; using difference_type = typename iterator::difference_type; using reference = value_type&; using pointer = value_type*; ordered_iterator() noexcept { } // Copy constructor from iterator to const_iterator. template::type* = nullptr> ordered_iterator(const ordered_iterator& other) noexcept: m_iterator(other.m_iterator) { } ordered_iterator(const ordered_iterator& other) = default; ordered_iterator(ordered_iterator&& other) = default; ordered_iterator& operator=(const ordered_iterator& other) = default; ordered_iterator& operator=(ordered_iterator&& other) = default; const typename ordered_hash::key_type& key() const { return KeySelect()(*m_iterator); } template::value && IsConst>::type* = nullptr> const typename U::value_type& value() const { return U()(*m_iterator); } template::value && !IsConst>::type* = nullptr> typename U::value_type& value() { return U()(*m_iterator); } reference operator*() const { return *m_iterator; } pointer operator->() const { return m_iterator.operator->(); } ordered_iterator& operator++() { ++m_iterator; return *this; } ordered_iterator& operator--() { --m_iterator; return *this; } ordered_iterator operator++(int) { ordered_iterator tmp(*this); ++(*this); return tmp; } ordered_iterator operator--(int) { ordered_iterator tmp(*this); --(*this); return tmp; } reference operator[](difference_type n) const { return m_iterator[n]; } ordered_iterator& operator+=(difference_type n) { m_iterator += n; return *this; } ordered_iterator& operator-=(difference_type n) { m_iterator -= n; return *this; } ordered_iterator operator+(difference_type n) { ordered_iterator tmp(*this); tmp += n; return tmp; } ordered_iterator operator-(difference_type n) { ordered_iterator tmp(*this); tmp -= n; return tmp; } friend bool operator==(const ordered_iterator& lhs, const ordered_iterator& rhs) { return lhs.m_iterator == rhs.m_iterator; } friend bool operator!=(const ordered_iterator& lhs, const ordered_iterator& rhs) { return lhs.m_iterator != rhs.m_iterator; } friend bool operator<(const ordered_iterator& lhs, const ordered_iterator& rhs) { return lhs.m_iterator < rhs.m_iterator; } friend bool operator>(const ordered_iterator& lhs, const ordered_iterator& rhs) { return lhs.m_iterator > rhs.m_iterator; } friend bool operator<=(const ordered_iterator& lhs, const ordered_iterator& rhs) { return lhs.m_iterator <= rhs.m_iterator; } friend bool operator>=(const ordered_iterator& lhs, const ordered_iterator& rhs) { return lhs.m_iterator >= rhs.m_iterator; } friend ordered_iterator operator+(difference_type n, const ordered_iterator& it) { return n + it.m_iterator; } friend difference_type operator-(const ordered_iterator& lhs, const ordered_iterator& rhs) { return lhs.m_iterator - rhs.m_iterator; } private: iterator m_iterator; }; private: using bucket_entry = tsl::detail_ordered_hash::bucket_entry; using buckets_container_allocator = typename std::allocator_traits::template rebind_alloc; using buckets_container_type = std::vector; using truncated_hash_type = typename bucket_entry::truncated_hash_type; using index_type = typename bucket_entry::index_type; public: ordered_hash(size_type bucket_count, const Hash& hash, const KeyEqual& equal, const Allocator& alloc, float max_load_factor): Hash(hash), KeyEqual(equal), m_buckets_data(alloc), m_buckets(static_empty_bucket_ptr()), m_mask(0), m_values(alloc), m_grow_on_next_insert(false) { if(bucket_count > max_bucket_count()) { TSL_OH_THROW_OR_TERMINATE(std::length_error, "The map exceeds its maxmimum size."); } if(bucket_count > 0) { bucket_count = round_up_to_power_of_two(bucket_count); m_buckets_data.resize(bucket_count); m_buckets = m_buckets_data.data(), m_mask = bucket_count - 1; } this->max_load_factor(max_load_factor); } ordered_hash(const ordered_hash& other): Hash(other), KeyEqual(other), m_buckets_data(other.m_buckets_data), m_buckets(m_buckets_data.empty()?static_empty_bucket_ptr(): m_buckets_data.data()), m_mask(other.m_mask), m_values(other.m_values), m_grow_on_next_insert(other.m_grow_on_next_insert), m_max_load_factor(other.m_max_load_factor), m_load_threshold(other.m_load_threshold) { } ordered_hash(ordered_hash&& other) noexcept(std::is_nothrow_move_constructible::value && std::is_nothrow_move_constructible::value && std::is_nothrow_move_constructible::value && std::is_nothrow_move_constructible::value) : Hash(std::move(static_cast(other))), KeyEqual(std::move(static_cast(other))), m_buckets_data(std::move(other.m_buckets_data)), m_buckets(m_buckets_data.empty()?static_empty_bucket_ptr(): m_buckets_data.data()), m_mask(other.m_mask), m_values(std::move(other.m_values)), m_grow_on_next_insert(other.m_grow_on_next_insert), m_max_load_factor(other.m_max_load_factor), m_load_threshold(other.m_load_threshold) { other.m_buckets_data.clear(); other.m_buckets = static_empty_bucket_ptr(); other.m_mask = 0; other.m_values.clear(); other.m_grow_on_next_insert = false; other.m_load_threshold = 0; } ordered_hash& operator=(const ordered_hash& other) { if(&other != this) { Hash::operator=(other); KeyEqual::operator=(other); m_buckets_data = other.m_buckets_data; m_buckets = m_buckets_data.empty()?static_empty_bucket_ptr(): m_buckets_data.data(); m_mask = other.m_mask; m_values = other.m_values; m_grow_on_next_insert = other.m_grow_on_next_insert; m_max_load_factor = other.m_max_load_factor; m_load_threshold = other.m_load_threshold; } return *this; } ordered_hash& operator=(ordered_hash&& other) { other.swap(*this); other.clear(); return *this; } allocator_type get_allocator() const { return m_values.get_allocator(); } /* * Iterators */ iterator begin() noexcept { return iterator(m_values.begin()); } const_iterator begin() const noexcept { return cbegin(); } const_iterator cbegin() const noexcept { return const_iterator(m_values.cbegin()); } iterator end() noexcept { return iterator(m_values.end()); } const_iterator end() const noexcept { return cend(); } const_iterator cend() const noexcept { return const_iterator(m_values.cend()); } reverse_iterator rbegin() noexcept { return reverse_iterator(m_values.end()); } const_reverse_iterator rbegin() const noexcept { return rcbegin(); } const_reverse_iterator rcbegin() const noexcept { return const_reverse_iterator(m_values.cend()); } reverse_iterator rend() noexcept { return reverse_iterator(m_values.begin()); } const_reverse_iterator rend() const noexcept { return rcend(); } const_reverse_iterator rcend() const noexcept { return const_reverse_iterator(m_values.cbegin()); } /* * Capacity */ bool empty() const noexcept { return m_values.empty(); } size_type size() const noexcept { return m_values.size(); } size_type max_size() const noexcept { return std::min(bucket_entry::max_size(), m_values.max_size()); } /* * Modifiers */ void clear() noexcept { for(auto& bucket: m_buckets_data) { bucket.clear(); } m_values.clear(); m_grow_on_next_insert = false; } template std::pair insert(P&& value) { return insert_impl(KeySelect()(value), std::forward

(value)); } template iterator insert_hint(const_iterator hint, P&& value) { if(hint != cend() && compare_keys(KeySelect()(*hint), KeySelect()(value))) { return mutable_iterator(hint); } return insert(std::forward

(value)).first; } template void insert(InputIt first, InputIt last) { if(std::is_base_of::iterator_category>::value) { const auto nb_elements_insert = std::distance(first, last); const size_type nb_free_buckets = m_load_threshold - size(); tsl_oh_assert(m_load_threshold >= size()); if(nb_elements_insert > 0 && nb_free_buckets < size_type(nb_elements_insert)) { reserve(size() + size_type(nb_elements_insert)); } } for(; first != last; ++first) { insert(*first); } } template std::pair insert_or_assign(K&& key, M&& value) { auto it = try_emplace(std::forward(key), std::forward(value)); if(!it.second) { it.first.value() = std::forward(value); } return it; } template iterator insert_or_assign(const_iterator hint, K&& key, M&& obj) { if(hint != cend() && compare_keys(KeySelect()(*hint), key)) { auto it = mutable_iterator(hint); it.value() = std::forward(obj); return it; } return insert_or_assign(std::forward(key), std::forward(obj)).first; } template std::pair emplace(Args&&... args) { return insert(value_type(std::forward(args)...)); } template iterator emplace_hint(const_iterator hint, Args&&... args) { return insert_hint(hint, value_type(std::forward(args)...)); } template std::pair try_emplace(K&& key, Args&&... value_args) { return insert_impl(key, std::piecewise_construct, std::forward_as_tuple(std::forward(key)), std::forward_as_tuple(std::forward(value_args)...)); } template iterator try_emplace_hint(const_iterator hint, K&& key, Args&&... args) { if(hint != cend() && compare_keys(KeySelect()(*hint), key)) { return mutable_iterator(hint); } return try_emplace(std::forward(key), std::forward(args)...).first; } /** * Here to avoid `template size_type erase(const K& key)` being used when * we use an `iterator` instead of a `const_iterator`. */ iterator erase(iterator pos) { return erase(const_iterator(pos)); } iterator erase(const_iterator pos) { tsl_oh_assert(pos != cend()); const std::size_t index_erase = iterator_to_index(pos); auto it_bucket = find_key(pos.key(), hash_key(pos.key())); tsl_oh_assert(it_bucket != m_buckets_data.end()); erase_value_from_bucket(it_bucket); /* * One element was removed from m_values, due to the left shift the next element * is now at the position of the previous element (or end if none). */ return begin() + index_erase; } iterator erase(const_iterator first, const_iterator last) { if(first == last) { return mutable_iterator(first); } tsl_oh_assert(std::distance(first, last) > 0); const std::size_t start_index = iterator_to_index(first); const std::size_t nb_values = std::size_t(std::distance(first, last)); const std::size_t end_index = start_index + nb_values; // Delete all values #ifdef TSL_OH_NO_CONTAINER_ERASE_CONST_ITERATOR auto next_it = m_values.erase(mutable_iterator(first).m_iterator, mutable_iterator(last).m_iterator); #else auto next_it = m_values.erase(first.m_iterator, last.m_iterator); #endif /* * Mark the buckets corresponding to the values as empty and do a backward shift. * * Also, the erase operation on m_values has shifted all the values on the right of last.m_iterator. * Adapt the indexes for these values. */ std::size_t ibucket = 0; while(ibucket < m_buckets_data.size()) { if(m_buckets[ibucket].empty()) { ibucket++; } else if(m_buckets[ibucket].index() >= start_index && m_buckets[ibucket].index() < end_index) { m_buckets[ibucket].clear(); backward_shift(ibucket); // Don't increment ibucket, backward_shift may have replaced current bucket. } else if(m_buckets[ibucket].index() >= end_index) { m_buckets[ibucket].set_index(index_type(m_buckets[ibucket].index() - nb_values)); ibucket++; } else { ibucket++; } } return iterator(next_it); } template size_type erase(const K& key) { return erase(key, hash_key(key)); } template size_type erase(const K& key, std::size_t hash) { return erase_impl(key, hash); } void swap(ordered_hash& other) { using std::swap; swap(static_cast(*this), static_cast(other)); swap(static_cast(*this), static_cast(other)); swap(m_buckets_data, other.m_buckets_data); swap(m_buckets, other.m_buckets); swap(m_mask, other.m_mask); swap(m_values, other.m_values); swap(m_grow_on_next_insert, other.m_grow_on_next_insert); swap(m_max_load_factor, other.m_max_load_factor); swap(m_load_threshold, other.m_load_threshold); } /* * Lookup */ template::value>::type* = nullptr> typename U::value_type& at(const K& key) { return at(key, hash_key(key)); } template::value>::type* = nullptr> typename U::value_type& at(const K& key, std::size_t hash) { return const_cast(static_cast(this)->at(key, hash)); } template::value>::type* = nullptr> const typename U::value_type& at(const K& key) const { return at(key, hash_key(key)); } template::value>::type* = nullptr> const typename U::value_type& at(const K& key, std::size_t hash) const { auto it = find(key, hash); if(it != end()) { return it.value(); } else { TSL_OH_THROW_OR_TERMINATE(std::out_of_range, "Couldn't find the key."); } } template::value>::type* = nullptr> typename U::value_type& operator[](K&& key) { return try_emplace(std::forward(key)).first.value(); } template size_type count(const K& key) const { return count(key, hash_key(key)); } template size_type count(const K& key, std::size_t hash) const { if(find(key, hash) == cend()) { return 0; } else { return 1; } } template iterator find(const K& key) { return find(key, hash_key(key)); } template iterator find(const K& key, std::size_t hash) { auto it_bucket = find_key(key, hash); return (it_bucket != m_buckets_data.end())?iterator(m_values.begin() + it_bucket->index()):end(); } template const_iterator find(const K& key) const { return find(key, hash_key(key)); } template const_iterator find(const K& key, std::size_t hash) const { auto it_bucket = find_key(key, hash); return (it_bucket != m_buckets_data.cend())?const_iterator(m_values.begin() + it_bucket->index()):end(); } template std::pair equal_range(const K& key) { return equal_range(key, hash_key(key)); } template std::pair equal_range(const K& key, std::size_t hash) { iterator it = find(key, hash); return std::make_pair(it, (it == end())?it:std::next(it)); } template std::pair equal_range(const K& key) const { return equal_range(key, hash_key(key)); } template std::pair equal_range(const K& key, std::size_t hash) const { const_iterator it = find(key, hash); return std::make_pair(it, (it == cend())?it:std::next(it)); } /* * Bucket interface */ size_type bucket_count() const { return m_buckets_data.size(); } size_type max_bucket_count() const { return m_buckets_data.max_size(); } /* * Hash policy */ float load_factor() const { if(bucket_count() == 0) { return 0; } return float(size())/float(bucket_count()); } float max_load_factor() const { return m_max_load_factor; } void max_load_factor(float ml) { m_max_load_factor = std::max(0.1f, std::min(ml, 0.95f)); m_load_threshold = size_type(float(bucket_count())*m_max_load_factor); } void rehash(size_type count) { count = std::max(count, size_type(std::ceil(float(size())/max_load_factor()))); rehash_impl(count); } void reserve(size_type count) { reserve_space_for_values(count); count = size_type(std::ceil(float(count)/max_load_factor())); rehash(count); } /* * Observers */ hasher hash_function() const { return static_cast(*this); } key_equal key_eq() const { return static_cast(*this); } /* * Other */ iterator mutable_iterator(const_iterator pos) { return iterator(m_values.begin() + iterator_to_index(pos)); } iterator nth(size_type index) { tsl_oh_assert(index <= size()); return iterator(m_values.begin() + index); } const_iterator nth(size_type index) const { tsl_oh_assert(index <= size()); return const_iterator(m_values.cbegin() + index); } const_reference front() const { tsl_oh_assert(!empty()); return m_values.front(); } const_reference back() const { tsl_oh_assert(!empty()); return m_values.back(); } const values_container_type& values_container() const noexcept { return m_values; } template::value>::type* = nullptr> const typename values_container_type::value_type* data() const noexcept { return m_values.data(); } template::value>::type* = nullptr> size_type capacity() const noexcept { return m_values.capacity(); } void shrink_to_fit() { m_values.shrink_to_fit(); } template std::pair insert_at_position(const_iterator pos, P&& value) { return insert_at_position_impl(pos.m_iterator, KeySelect()(value), std::forward

(value)); } template std::pair emplace_at_position(const_iterator pos, Args&&... args) { return insert_at_position(pos, value_type(std::forward(args)...)); } template std::pair try_emplace_at_position(const_iterator pos, K&& key, Args&&... value_args) { return insert_at_position_impl(pos.m_iterator, key, std::piecewise_construct, std::forward_as_tuple(std::forward(key)), std::forward_as_tuple(std::forward(value_args)...)); } void pop_back() { tsl_oh_assert(!empty()); erase(std::prev(end())); } /** * Here to avoid `template size_type unordered_erase(const K& key)` being used when * we use a iterator instead of a const_iterator. */ iterator unordered_erase(iterator pos) { return unordered_erase(const_iterator(pos)); } iterator unordered_erase(const_iterator pos) { const std::size_t index_erase = iterator_to_index(pos); unordered_erase(pos.key()); /* * One element was deleted, index_erase now points to the next element as the elements after * the deleted value were shifted to the left in m_values (will be end() if we deleted the last element). */ return begin() + index_erase; } template size_type unordered_erase(const K& key) { return unordered_erase(key, hash_key(key)); } template size_type unordered_erase(const K& key, std::size_t hash) { auto it_bucket_key = find_key(key, hash); if(it_bucket_key == m_buckets_data.end()) { return 0; } /** * If we are not erasing the last element in m_values, we swap * the element we are erasing with the last element. We then would * just have to do a pop_back() in m_values. */ if(!compare_keys(key, KeySelect()(back()))) { auto it_bucket_last_elem = find_key(KeySelect()(back()), hash_key(KeySelect()(back()))); tsl_oh_assert(it_bucket_last_elem != m_buckets_data.end()); tsl_oh_assert(it_bucket_last_elem->index() == m_values.size() - 1); using std::swap; swap(m_values[it_bucket_key->index()], m_values[it_bucket_last_elem->index()]); swap(it_bucket_key->index_ref(), it_bucket_last_elem->index_ref()); } erase_value_from_bucket(it_bucket_key); return 1; } template void serialize(Serializer& serializer) const { serialize_impl(serializer); } template void deserialize(Deserializer& deserializer, bool hash_compatible) { deserialize_impl(deserializer, hash_compatible); } friend bool operator==(const ordered_hash& lhs, const ordered_hash& rhs) { return lhs.m_values == rhs.m_values; } friend bool operator!=(const ordered_hash& lhs, const ordered_hash& rhs) { return lhs.m_values != rhs.m_values; } friend bool operator<(const ordered_hash& lhs, const ordered_hash& rhs) { return lhs.m_values < rhs.m_values; } friend bool operator<=(const ordered_hash& lhs, const ordered_hash& rhs) { return lhs.m_values <= rhs.m_values; } friend bool operator>(const ordered_hash& lhs, const ordered_hash& rhs) { return lhs.m_values > rhs.m_values; } friend bool operator>=(const ordered_hash& lhs, const ordered_hash& rhs) { return lhs.m_values >= rhs.m_values; } private: template std::size_t hash_key(const K& key) const { return Hash::operator()(key); } template bool compare_keys(const K1& key1, const K2& key2) const { return KeyEqual::operator()(key1, key2); } template typename buckets_container_type::iterator find_key(const K& key, std::size_t hash) { auto it = static_cast(this)->find_key(key, hash); return m_buckets_data.begin() + std::distance(m_buckets_data.cbegin(), it); } /** * Return bucket which has the key 'key' or m_buckets_data.end() if none. * * From the bucket_for_hash, search for the value until we either find an empty bucket * or a bucket which has a value with a distance from its ideal bucket longer * than the probe length for the value we are looking for. */ template typename buckets_container_type::const_iterator find_key(const K& key, std::size_t hash) const { for(std::size_t ibucket = bucket_for_hash(hash), dist_from_ideal_bucket = 0; ; ibucket = next_bucket(ibucket), dist_from_ideal_bucket++) { if(m_buckets[ibucket].empty()) { return m_buckets_data.end(); } else if(m_buckets[ibucket].truncated_hash() == bucket_entry::truncate_hash(hash) && compare_keys(key, KeySelect()(m_values[m_buckets[ibucket].index()]))) { return m_buckets_data.begin() + ibucket; } else if(dist_from_ideal_bucket > distance_from_ideal_bucket(ibucket)) { return m_buckets_data.end(); } } } void rehash_impl(size_type bucket_count) { tsl_oh_assert(bucket_count >= size_type(std::ceil(float(size())/max_load_factor()))); if(bucket_count > max_bucket_count()) { TSL_OH_THROW_OR_TERMINATE(std::length_error, "The map exceeds its maxmimum size."); } if(bucket_count > 0) { bucket_count = round_up_to_power_of_two(bucket_count); } if(bucket_count == this->bucket_count()) { return; } buckets_container_type old_buckets(bucket_count); m_buckets_data.swap(old_buckets); m_buckets = m_buckets_data.empty()?static_empty_bucket_ptr(): m_buckets_data.data(); // Everything should be noexcept from here. m_mask = (bucket_count > 0)?(bucket_count - 1):0; this->max_load_factor(m_max_load_factor); m_grow_on_next_insert = false; for(const bucket_entry& old_bucket: old_buckets) { if(old_bucket.empty()) { continue; } truncated_hash_type insert_hash = old_bucket.truncated_hash(); index_type insert_index = old_bucket.index(); for(std::size_t ibucket = bucket_for_hash(insert_hash), dist_from_ideal_bucket = 0; ; ibucket = next_bucket(ibucket), dist_from_ideal_bucket++) { if(m_buckets[ibucket].empty()) { m_buckets[ibucket].set_index(insert_index); m_buckets[ibucket].set_hash(insert_hash); break; } const std::size_t distance = distance_from_ideal_bucket(ibucket); if(dist_from_ideal_bucket > distance) { std::swap(insert_index, m_buckets[ibucket].index_ref()); std::swap(insert_hash, m_buckets[ibucket].truncated_hash_ref()); dist_from_ideal_bucket = distance; } } } } template::value>::type* = nullptr> void reserve_space_for_values(size_type count) { m_values.reserve(count); } template::value>::type* = nullptr> void reserve_space_for_values(size_type /*count*/) { } /** * Swap the empty bucket with the values on its right until we cross another empty bucket * or if the other bucket has a distance_from_ideal_bucket == 0. */ void backward_shift(std::size_t empty_ibucket) noexcept { tsl_oh_assert(m_buckets[empty_ibucket].empty()); std::size_t previous_ibucket = empty_ibucket; for(std::size_t current_ibucket = next_bucket(previous_ibucket); !m_buckets[current_ibucket].empty() && distance_from_ideal_bucket(current_ibucket) > 0; previous_ibucket = current_ibucket, current_ibucket = next_bucket(current_ibucket)) { std::swap(m_buckets[current_ibucket], m_buckets[previous_ibucket]); } } void erase_value_from_bucket(typename buckets_container_type::iterator it_bucket) { tsl_oh_assert(it_bucket != m_buckets_data.end() && !it_bucket->empty()); m_values.erase(m_values.begin() + it_bucket->index()); /* * m_values.erase shifted all the values on the right of the erased value, * shift the indexes by -1 in the buckets array for these values. */ if(it_bucket->index() != m_values.size()) { shift_indexes_in_buckets(it_bucket->index(), -1); } // Mark the bucket as empty and do a backward shift of the values on the right it_bucket->clear(); backward_shift(std::size_t(std::distance(m_buckets_data.begin(), it_bucket))); } /** * Go through each value from [from_ivalue, m_values.size()) in m_values and for each * bucket corresponding to the value, shift the index by delta. * * delta must be equal to 1 or -1. */ void shift_indexes_in_buckets(index_type from_ivalue, int delta) noexcept { tsl_oh_assert(delta == 1 || delta == -1); for(std::size_t ivalue = from_ivalue; ivalue < m_values.size(); ivalue++) { // All the values in m_values have been shifted by delta. Find the bucket corresponding // to the value m_values[ivalue] const index_type old_index = static_cast(ivalue - delta); std::size_t ibucket = bucket_for_hash(hash_key(KeySelect()(m_values[ivalue]))); while(m_buckets[ibucket].index() != old_index) { ibucket = next_bucket(ibucket); } m_buckets[ibucket].set_index(index_type(ivalue)); } } template size_type erase_impl(const K& key, std::size_t hash) { auto it_bucket = find_key(key, hash); if(it_bucket != m_buckets_data.end()) { erase_value_from_bucket(it_bucket); return 1; } else { return 0; } } /** * Insert the element at the end. */ template std::pair insert_impl(const K& key, Args&&... value_type_args) { const std::size_t hash = hash_key(key); std::size_t ibucket = bucket_for_hash(hash); std::size_t dist_from_ideal_bucket = 0; while(!m_buckets[ibucket].empty() && dist_from_ideal_bucket <= distance_from_ideal_bucket(ibucket)) { if(m_buckets[ibucket].truncated_hash() == bucket_entry::truncate_hash(hash) && compare_keys(key, KeySelect()(m_values[m_buckets[ibucket].index()]))) { return std::make_pair(begin() + m_buckets[ibucket].index(), false); } ibucket = next_bucket(ibucket); dist_from_ideal_bucket++; } if(size() >= max_size()) { TSL_OH_THROW_OR_TERMINATE(std::length_error, "We reached the maximum size for the hash table."); } if(grow_on_high_load()) { ibucket = bucket_for_hash(hash); dist_from_ideal_bucket = 0; } m_values.emplace_back(std::forward(value_type_args)...); insert_index(ibucket, dist_from_ideal_bucket, index_type(m_values.size() - 1), bucket_entry::truncate_hash(hash)); return std::make_pair(std::prev(end()), true); } /** * Insert the element before insert_position. */ template std::pair insert_at_position_impl(typename values_container_type::const_iterator insert_position, const K& key, Args&&... value_type_args) { const std::size_t hash = hash_key(key); std::size_t ibucket = bucket_for_hash(hash); std::size_t dist_from_ideal_bucket = 0; while(!m_buckets[ibucket].empty() && dist_from_ideal_bucket <= distance_from_ideal_bucket(ibucket)) { if(m_buckets[ibucket].truncated_hash() == bucket_entry::truncate_hash(hash) && compare_keys(key, KeySelect()(m_values[m_buckets[ibucket].index()]))) { return std::make_pair(begin() + m_buckets[ibucket].index(), false); } ibucket = next_bucket(ibucket); dist_from_ideal_bucket++; } if(size() >= max_size()) { TSL_OH_THROW_OR_TERMINATE(std::length_error, "We reached the maximum size for the hash table."); } if(grow_on_high_load()) { ibucket = bucket_for_hash(hash); dist_from_ideal_bucket = 0; } const index_type index_insert_position = index_type(std::distance(m_values.cbegin(), insert_position)); #ifdef TSL_OH_NO_CONTAINER_EMPLACE_CONST_ITERATOR m_values.emplace(m_values.begin() + std::distance(m_values.cbegin(), insert_position), std::forward(value_type_args)...); #else m_values.emplace(insert_position, std::forward(value_type_args)...); #endif insert_index(ibucket, dist_from_ideal_bucket, index_insert_position, bucket_entry::truncate_hash(hash)); /* * The insertion didn't happend at the end of the m_values container, * we need to shift the indexes in m_buckets_data. */ if(index_insert_position != m_values.size() - 1) { shift_indexes_in_buckets(index_insert_position + 1, 1); } return std::make_pair(iterator(m_values.begin() + index_insert_position), true); } void insert_index(std::size_t ibucket, std::size_t dist_from_ideal_bucket, index_type index_insert, truncated_hash_type hash_insert) noexcept { while(!m_buckets[ibucket].empty()) { const std::size_t distance = distance_from_ideal_bucket(ibucket); if(dist_from_ideal_bucket > distance) { std::swap(index_insert, m_buckets[ibucket].index_ref()); std::swap(hash_insert, m_buckets[ibucket].truncated_hash_ref()); dist_from_ideal_bucket = distance; } ibucket = next_bucket(ibucket); dist_from_ideal_bucket++; if(dist_from_ideal_bucket > REHASH_ON_HIGH_NB_PROBES__NPROBES && !m_grow_on_next_insert && load_factor() >= REHASH_ON_HIGH_NB_PROBES__MIN_LOAD_FACTOR) { // We don't want to grow the map now as we need this method to be noexcept. // Do it on next insert. m_grow_on_next_insert = true; } } m_buckets[ibucket].set_index(index_insert); m_buckets[ibucket].set_hash(hash_insert); } std::size_t distance_from_ideal_bucket(std::size_t ibucket) const noexcept { const std::size_t ideal_bucket = bucket_for_hash(m_buckets[ibucket].truncated_hash()); if(ibucket >= ideal_bucket) { return ibucket - ideal_bucket; } // If the bucket is smaller than the ideal bucket for the value, there was a wrapping at the end of the // bucket array due to the modulo. else { return (bucket_count() + ibucket) - ideal_bucket; } } std::size_t next_bucket(std::size_t index) const noexcept { tsl_oh_assert(index < m_buckets_data.size()); index++; return (index < m_buckets_data.size())?index:0; } std::size_t bucket_for_hash(std::size_t hash) const noexcept { return hash & m_mask; } std::size_t iterator_to_index(const_iterator it) const noexcept { const auto dist = std::distance(cbegin(), it); tsl_oh_assert(dist >= 0); return std::size_t(dist); } /** * Return true if the map has been rehashed. */ bool grow_on_high_load() { if(m_grow_on_next_insert || size() >= m_load_threshold) { rehash_impl(std::max(size_type(1), bucket_count() * 2)); m_grow_on_next_insert = false; return true; } else { return false; } } template void serialize_impl(Serializer& serializer) const { const slz_size_type version = SERIALIZATION_PROTOCOL_VERSION; serializer(version); const slz_size_type nb_elements = m_values.size(); serializer(nb_elements); const slz_size_type bucket_count = m_buckets_data.size(); serializer(bucket_count); const float max_load_factor = m_max_load_factor; serializer(max_load_factor); for(const value_type& value: m_values) { serializer(value); } for(const bucket_entry& bucket: m_buckets_data) { bucket.serialize(serializer); } } template void deserialize_impl(Deserializer& deserializer, bool hash_compatible) { tsl_oh_assert(m_buckets_data.empty()); // Current hash table must be empty const slz_size_type version = deserialize_value(deserializer); // For now we only have one version of the serialization protocol. // If it doesn't match there is a problem with the file. if(version != SERIALIZATION_PROTOCOL_VERSION) { TSL_OH_THROW_OR_TERMINATE(std::runtime_error, "Can't deserialize the ordered_map/set. " "The protocol version header is invalid."); } const slz_size_type nb_elements = deserialize_value(deserializer); const slz_size_type bucket_count_ds = deserialize_value(deserializer); const float max_load_factor = deserialize_value(deserializer); this->max_load_factor(max_load_factor); if(bucket_count_ds == 0) { tsl_oh_assert(nb_elements == 0); return; } if(!hash_compatible) { reserve(numeric_cast(nb_elements, "Deserialized nb_elements is too big.")); for(slz_size_type el = 0; el < nb_elements; el++) { insert(deserialize_value(deserializer)); } } else { m_buckets_data.reserve(numeric_cast(bucket_count_ds, "Deserialized bucket_count is too big.")); m_buckets = m_buckets_data.data(), m_mask = m_buckets_data.capacity() - 1; reserve_space_for_values(numeric_cast(nb_elements, "Deserialized nb_elements is too big.")); for(slz_size_type el = 0; el < nb_elements; el++) { m_values.push_back(deserialize_value(deserializer)); } for(slz_size_type b = 0; b < bucket_count_ds; b++) { m_buckets_data.push_back(bucket_entry::deserialize(deserializer)); } if(load_factor() > this->max_load_factor()) { TSL_OH_THROW_OR_TERMINATE(std::runtime_error, "Invalid max_load_factor. Check that the serializer " "and deserializer supports floats correctly as they " "can be converted implicitely to ints."); } } } static std::size_t round_up_to_power_of_two(std::size_t value) { if(is_power_of_two(value)) { return value; } if(value == 0) { return 1; } --value; for(std::size_t i = 1; i < sizeof(std::size_t) * CHAR_BIT; i *= 2) { value |= value >> i; } return value + 1; } static constexpr bool is_power_of_two(std::size_t value) { return value != 0 && (value & (value - 1)) == 0; } public: static const size_type DEFAULT_INIT_BUCKETS_SIZE = 0; static constexpr float DEFAULT_MAX_LOAD_FACTOR = 0.75f; private: static const size_type REHASH_ON_HIGH_NB_PROBES__NPROBES = 128; static constexpr float REHASH_ON_HIGH_NB_PROBES__MIN_LOAD_FACTOR = 0.15f; /** * Protocol version currenlty used for serialization. */ static const slz_size_type SERIALIZATION_PROTOCOL_VERSION = 1; /** * Return an always valid pointer to an static empty bucket_entry with last_bucket() == true. */ bucket_entry* static_empty_bucket_ptr() { static bucket_entry empty_bucket; return &empty_bucket; } private: buckets_container_type m_buckets_data; /** * Points to m_buckets_data.data() if !m_buckets_data.empty() otherwise points to static_empty_bucket_ptr. * This variable is useful to avoid the cost of checking if m_buckets_data is empty when trying * to find an element. * * TODO Remove m_buckets_data and only use a pointer+size instead of a pointer+vector to save some space in the ordered_hash object. */ bucket_entry* m_buckets; size_type m_mask; values_container_type m_values; bool m_grow_on_next_insert; float m_max_load_factor; size_type m_load_threshold; }; } // end namespace detail_ordered_hash } // end namespace tsl #endif ================================================ FILE: extern/tsl-ordered-map/ordered_map.h ================================================ /** * MIT License * * Copyright (c) 2017 Tessil * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef TSL_ORDERED_MAP_H #define TSL_ORDERED_MAP_H #include #include #include #include #include #include #include #include #include #include "ordered_hash.h" namespace tsl { /** * Implementation of an hash map using open adressing with robin hood with backshift delete to resolve collisions. * * The particularity of this hash map is that it remembers the order in which the elements were added and * provide a way to access the structure which stores these values through the 'values_container()' method. * The used container is defined by ValueTypeContainer, by default a std::deque is used (grows faster) but * a std::vector may be used. In this case the map provides a 'data()' method which give a direct access * to the memory used to store the values (which can be usefull to communicate with C API's). * * The Key and T must be copy constructible and/or move constructible. To use `unordered_erase` they both * must be swappable. * * The behaviour of the hash map is undefinded if the destructor of Key or T throws an exception. * * By default the maximum size of a map is limited to 2^32 - 1 values, if needed this can be changed through * the IndexType template parameter. Using an `uint64_t` will raise this limit to 2^64 - 1 values but each * bucket will use 16 bytes instead of 8 bytes in addition to the space needed to store the values. * * Iterators invalidation: * - clear, operator=, reserve, rehash: always invalidate the iterators (also invalidate end()). * - insert, emplace, emplace_hint, operator[]: when a std::vector is used as ValueTypeContainer * and if size() < capacity(), only end(). * Otherwise all the iterators are invalidated if an insert occurs. * - erase, unordered_erase: when a std::vector is used as ValueTypeContainer invalidate the iterator of * the erased element and all the ones after the erased element (including end()). * Otherwise all the iterators are invalidated if an erase occurs. */ template, class KeyEqual = std::equal_to, class Allocator = std::allocator>, class ValueTypeContainer = std::deque, Allocator>, class IndexType = std::uint_least32_t> class ordered_map { private: template using has_is_transparent = tsl::detail_ordered_hash::has_is_transparent; class KeySelect { public: using key_type = Key; const key_type& operator()(const std::pair& key_value) const noexcept { return key_value.first; } key_type& operator()(std::pair& key_value) noexcept { return key_value.first; } }; class ValueSelect { public: using value_type = T; const value_type& operator()(const std::pair& key_value) const noexcept { return key_value.second; } value_type& operator()(std::pair& key_value) noexcept { return key_value.second; } }; using ht = detail_ordered_hash::ordered_hash, KeySelect, ValueSelect, Hash, KeyEqual, Allocator, ValueTypeContainer, IndexType>; public: using key_type = typename ht::key_type; using mapped_type = T; using value_type = typename ht::value_type; using size_type = typename ht::size_type; using difference_type = typename ht::difference_type; using hasher = typename ht::hasher; using key_equal = typename ht::key_equal; using allocator_type = typename ht::allocator_type; using reference = typename ht::reference; using const_reference = typename ht::const_reference; using pointer = typename ht::pointer; using const_pointer = typename ht::const_pointer; using iterator = typename ht::iterator; using const_iterator = typename ht::const_iterator; using reverse_iterator = typename ht::reverse_iterator; using const_reverse_iterator = typename ht::const_reverse_iterator; using values_container_type = typename ht::values_container_type; /* * Constructors */ ordered_map(): ordered_map(ht::DEFAULT_INIT_BUCKETS_SIZE) { } explicit ordered_map(size_type bucket_count, const Hash& hash = Hash(), const KeyEqual& equal = KeyEqual(), const Allocator& alloc = Allocator()): m_ht(bucket_count, hash, equal, alloc, ht::DEFAULT_MAX_LOAD_FACTOR) { } ordered_map(size_type bucket_count, const Allocator& alloc): ordered_map(bucket_count, Hash(), KeyEqual(), alloc) { } ordered_map(size_type bucket_count, const Hash& hash, const Allocator& alloc): ordered_map(bucket_count, hash, KeyEqual(), alloc) { } explicit ordered_map(const Allocator& alloc): ordered_map(ht::DEFAULT_INIT_BUCKETS_SIZE, alloc) { } template ordered_map(InputIt first, InputIt last, size_type bucket_count = ht::DEFAULT_INIT_BUCKETS_SIZE, const Hash& hash = Hash(), const KeyEqual& equal = KeyEqual(), const Allocator& alloc = Allocator()): ordered_map(bucket_count, hash, equal, alloc) { insert(first, last); } template ordered_map(InputIt first, InputIt last, size_type bucket_count, const Allocator& alloc): ordered_map(first, last, bucket_count, Hash(), KeyEqual(), alloc) { } template ordered_map(InputIt first, InputIt last, size_type bucket_count, const Hash& hash, const Allocator& alloc): ordered_map(first, last, bucket_count, hash, KeyEqual(), alloc) { } ordered_map(std::initializer_list init, size_type bucket_count = ht::DEFAULT_INIT_BUCKETS_SIZE, const Hash& hash = Hash(), const KeyEqual& equal = KeyEqual(), const Allocator& alloc = Allocator()): ordered_map(init.begin(), init.end(), bucket_count, hash, equal, alloc) { } ordered_map(std::initializer_list init, size_type bucket_count, const Allocator& alloc): ordered_map(init.begin(), init.end(), bucket_count, Hash(), KeyEqual(), alloc) { } ordered_map(std::initializer_list init, size_type bucket_count, const Hash& hash, const Allocator& alloc): ordered_map(init.begin(), init.end(), bucket_count, hash, KeyEqual(), alloc) { } ordered_map& operator=(std::initializer_list ilist) { m_ht.clear(); m_ht.reserve(ilist.size()); m_ht.insert(ilist.begin(), ilist.end()); return *this; } allocator_type get_allocator() const { return m_ht.get_allocator(); } /* * Iterators */ iterator begin() noexcept { return m_ht.begin(); } const_iterator begin() const noexcept { return m_ht.begin(); } const_iterator cbegin() const noexcept { return m_ht.cbegin(); } iterator end() noexcept { return m_ht.end(); } const_iterator end() const noexcept { return m_ht.end(); } const_iterator cend() const noexcept { return m_ht.cend(); } reverse_iterator rbegin() noexcept { return m_ht.rbegin(); } const_reverse_iterator rbegin() const noexcept { return m_ht.rbegin(); } const_reverse_iterator rcbegin() const noexcept { return m_ht.rcbegin(); } reverse_iterator rend() noexcept { return m_ht.rend(); } const_reverse_iterator rend() const noexcept { return m_ht.rend(); } const_reverse_iterator rcend() const noexcept { return m_ht.rcend(); } /* * Capacity */ bool empty() const noexcept { return m_ht.empty(); } size_type size() const noexcept { return m_ht.size(); } size_type max_size() const noexcept { return m_ht.max_size(); } /* * Modifiers */ void clear() noexcept { m_ht.clear(); } std::pair insert(const value_type& value) { return m_ht.insert(value); } template::value>::type* = nullptr> std::pair insert(P&& value) { return m_ht.emplace(std::forward

(value)); } std::pair insert(value_type&& value) { return m_ht.insert(std::move(value)); } iterator insert(const_iterator hint, const value_type& value) { return m_ht.insert_hint(hint, value); } template::value>::type* = nullptr> iterator insert(const_iterator hint, P&& value) { return m_ht.emplace_hint(hint, std::forward

(value)); } iterator insert(const_iterator hint, value_type&& value) { return m_ht.insert_hint(hint, std::move(value)); } template void insert(InputIt first, InputIt last) { m_ht.insert(first, last); } void insert(std::initializer_list ilist) { m_ht.insert(ilist.begin(), ilist.end()); } template std::pair insert_or_assign(const key_type& k, M&& obj) { return m_ht.insert_or_assign(k, std::forward(obj)); } template std::pair insert_or_assign(key_type&& k, M&& obj) { return m_ht.insert_or_assign(std::move(k), std::forward(obj)); } template iterator insert_or_assign(const_iterator hint, const key_type& k, M&& obj) { return m_ht.insert_or_assign(hint, k, std::forward(obj)); } template iterator insert_or_assign(const_iterator hint, key_type&& k, M&& obj) { return m_ht.insert_or_assign(hint, std::move(k), std::forward(obj)); } /** * Due to the way elements are stored, emplace will need to move or copy the key-value once. * The method is equivalent to insert(value_type(std::forward(args)...)); * * Mainly here for compatibility with the std::unordered_map interface. */ template std::pair emplace(Args&&... args) { return m_ht.emplace(std::forward(args)...); } /** * Due to the way elements are stored, emplace_hint will need to move or copy the key-value once. * The method is equivalent to insert(hint, value_type(std::forward(args)...)); * * Mainly here for compatibility with the std::unordered_map interface. */ template iterator emplace_hint(const_iterator hint, Args&&... args) { return m_ht.emplace_hint(hint, std::forward(args)...); } template std::pair try_emplace(const key_type& k, Args&&... args) { return m_ht.try_emplace(k, std::forward(args)...); } template std::pair try_emplace(key_type&& k, Args&&... args) { return m_ht.try_emplace(std::move(k), std::forward(args)...); } template iterator try_emplace(const_iterator hint, const key_type& k, Args&&... args) { return m_ht.try_emplace_hint(hint, k, std::forward(args)...); } template iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args) { return m_ht.try_emplace_hint(hint, std::move(k), std::forward(args)...); } /** * When erasing an element, the insert order will be preserved and no holes will be present in the container * returned by 'values_container()'. * * The method is in O(n), if the order is not important 'unordered_erase(...)' method is faster with an O(1) * average complexity. */ iterator erase(iterator pos) { return m_ht.erase(pos); } /** * @copydoc erase(iterator pos) */ iterator erase(const_iterator pos) { return m_ht.erase(pos); } /** * @copydoc erase(iterator pos) */ iterator erase(const_iterator first, const_iterator last) { return m_ht.erase(first, last); } /** * @copydoc erase(iterator pos) */ size_type erase(const key_type& key) { return m_ht.erase(key); } /** * @copydoc erase(iterator pos) * * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup to the value if you already have the hash. */ size_type erase(const key_type& key, std::size_t precalculated_hash) { return m_ht.erase(key, precalculated_hash); } /** * @copydoc erase(iterator pos) * * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. * If so, K must be hashable and comparable to Key. */ template::value>::type* = nullptr> size_type erase(const K& key) { return m_ht.erase(key); } /** * @copydoc erase(const key_type& key, std::size_t precalculated_hash) * * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. * If so, K must be hashable and comparable to Key. */ template::value>::type* = nullptr> size_type erase(const K& key, std::size_t precalculated_hash) { return m_ht.erase(key, precalculated_hash); } void swap(ordered_map& other) { other.m_ht.swap(m_ht); } /* * Lookup */ T& at(const Key& key) { return m_ht.at(key); } /** * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ T& at(const Key& key, std::size_t precalculated_hash) { return m_ht.at(key, precalculated_hash); } const T& at(const Key& key) const { return m_ht.at(key); } /** * @copydoc at(const Key& key, std::size_t precalculated_hash) */ const T& at(const Key& key, std::size_t precalculated_hash) const { return m_ht.at(key, precalculated_hash); } /** * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. * If so, K must be hashable and comparable to Key. */ template::value>::type* = nullptr> T& at(const K& key) { return m_ht.at(key); } /** * @copydoc at(const K& key) * * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ template::value>::type* = nullptr> T& at(const K& key, std::size_t precalculated_hash) { return m_ht.at(key, precalculated_hash); } /** * @copydoc at(const K& key) */ template::value>::type* = nullptr> const T& at(const K& key) const { return m_ht.at(key); } /** * @copydoc at(const K& key, std::size_t precalculated_hash) */ template::value>::type* = nullptr> const T& at(const K& key, std::size_t precalculated_hash) const { return m_ht.at(key, precalculated_hash); } T& operator[](const Key& key) { return m_ht[key]; } T& operator[](Key&& key) { return m_ht[std::move(key)]; } size_type count(const Key& key) const { return m_ht.count(key); } /** * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ size_type count(const Key& key, std::size_t precalculated_hash) const { return m_ht.count(key, precalculated_hash); } /** * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. * If so, K must be hashable and comparable to Key. */ template::value>::type* = nullptr> size_type count(const K& key) const { return m_ht.count(key); } /** * @copydoc count(const K& key) const * * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ template::value>::type* = nullptr> size_type count(const K& key, std::size_t precalculated_hash) const { return m_ht.count(key, precalculated_hash); } iterator find(const Key& key) { return m_ht.find(key); } /** * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ iterator find(const Key& key, std::size_t precalculated_hash) { return m_ht.find(key, precalculated_hash); } const_iterator find(const Key& key) const { return m_ht.find(key); } /** * @copydoc find(const Key& key, std::size_t precalculated_hash) */ const_iterator find(const Key& key, std::size_t precalculated_hash) const { return m_ht.find(key, precalculated_hash); } /** * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. * If so, K must be hashable and comparable to Key. */ template::value>::type* = nullptr> iterator find(const K& key) { return m_ht.find(key); } /** * @copydoc find(const K& key) * * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ template::value>::type* = nullptr> iterator find(const K& key, std::size_t precalculated_hash) { return m_ht.find(key, precalculated_hash); } /** * @copydoc find(const K& key) */ template::value>::type* = nullptr> const_iterator find(const K& key) const { return m_ht.find(key); } /** * @copydoc find(const K& key) * * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ template::value>::type* = nullptr> const_iterator find(const K& key, std::size_t precalculated_hash) const { return m_ht.find(key, precalculated_hash); } std::pair equal_range(const Key& key) { return m_ht.equal_range(key); } /** * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ std::pair equal_range(const Key& key, std::size_t precalculated_hash) { return m_ht.equal_range(key, precalculated_hash); } std::pair equal_range(const Key& key) const { return m_ht.equal_range(key); } /** * @copydoc equal_range(const Key& key, std::size_t precalculated_hash) */ std::pair equal_range(const Key& key, std::size_t precalculated_hash) const { return m_ht.equal_range(key, precalculated_hash); } /** * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. * If so, K must be hashable and comparable to Key. */ template::value>::type* = nullptr> std::pair equal_range(const K& key) { return m_ht.equal_range(key); } /** * @copydoc equal_range(const K& key) * * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ template::value>::type* = nullptr> std::pair equal_range(const K& key, std::size_t precalculated_hash) { return m_ht.equal_range(key, precalculated_hash); } /** * @copydoc equal_range(const K& key) */ template::value>::type* = nullptr> std::pair equal_range(const K& key) const { return m_ht.equal_range(key); } /** * @copydoc equal_range(const K& key, std::size_t precalculated_hash) */ template::value>::type* = nullptr> std::pair equal_range(const K& key, std::size_t precalculated_hash) const { return m_ht.equal_range(key, precalculated_hash); } /* * Bucket interface */ size_type bucket_count() const { return m_ht.bucket_count(); } size_type max_bucket_count() const { return m_ht.max_bucket_count(); } /* * Hash policy */ float load_factor() const { return m_ht.load_factor(); } float max_load_factor() const { return m_ht.max_load_factor(); } void max_load_factor(float ml) { m_ht.max_load_factor(ml); } void rehash(size_type count) { m_ht.rehash(count); } void reserve(size_type count) { m_ht.reserve(count); } /* * Observers */ hasher hash_function() const { return m_ht.hash_function(); } key_equal key_eq() const { return m_ht.key_eq(); } /* * Other */ /** * Convert a const_iterator to an iterator. */ iterator mutable_iterator(const_iterator pos) { return m_ht.mutable_iterator(pos); } /** * Requires index <= size(). * * Return an iterator to the element at index. Return end() if index == size(). */ iterator nth(size_type index) { return m_ht.nth(index); } /** * @copydoc nth(size_type index) */ const_iterator nth(size_type index) const { return m_ht.nth(index); } /** * Return const_reference to the first element. Requires the container to not be empty. */ const_reference front() const { return m_ht.front(); } /** * Return const_reference to the last element. Requires the container to not be empty. */ const_reference back() const { return m_ht.back(); } /** * Only available if ValueTypeContainer is a std::vector. Same as calling 'values_container().data()'. */ template::value>::type* = nullptr> const typename values_container_type::value_type* data() const noexcept { return m_ht.data(); } /** * Return the container in which the values are stored. The values are in the same order as the insertion order * and are contiguous in the structure, no holes (size() == values_container().size()). */ const values_container_type& values_container() const noexcept { return m_ht.values_container(); } template::value>::type* = nullptr> size_type capacity() const noexcept { return m_ht.capacity(); } void shrink_to_fit() { m_ht.shrink_to_fit(); } /** * Insert the value before pos shifting all the elements on the right of pos (including pos) one position * to the right. * * Amortized linear time-complexity in the distance between pos and end(). */ std::pair insert_at_position(const_iterator pos, const value_type& value) { return m_ht.insert_at_position(pos, value); } /** * @copydoc insert_at_position(const_iterator pos, const value_type& value) */ std::pair insert_at_position(const_iterator pos, value_type&& value) { return m_ht.insert_at_position(pos, std::move(value)); } /** * @copydoc insert_at_position(const_iterator pos, const value_type& value) * * Same as insert_at_position(pos, value_type(std::forward(args)...), mainly * here for coherence. */ template std::pair emplace_at_position(const_iterator pos, Args&&... args) { return m_ht.emplace_at_position(pos, std::forward(args)...); } /** * @copydoc insert_at_position(const_iterator pos, const value_type& value) */ template std::pair try_emplace_at_position(const_iterator pos, const key_type& k, Args&&... args) { return m_ht.try_emplace_at_position(pos, k, std::forward(args)...); } /** * @copydoc insert_at_position(const_iterator pos, const value_type& value) */ template std::pair try_emplace_at_position(const_iterator pos, key_type&& k, Args&&... args) { return m_ht.try_emplace_at_position(pos, std::move(k), std::forward(args)...); } void pop_back() { m_ht.pop_back(); } /** * Faster erase operation with an O(1) average complexity but it doesn't preserve the insertion order. * * If an erasure occurs, the last element of the map will take the place of the erased element. */ iterator unordered_erase(iterator pos) { return m_ht.unordered_erase(pos); } /** * @copydoc unordered_erase(iterator pos) */ iterator unordered_erase(const_iterator pos) { return m_ht.unordered_erase(pos); } /** * @copydoc unordered_erase(iterator pos) */ size_type unordered_erase(const key_type& key) { return m_ht.unordered_erase(key); } /** * @copydoc unordered_erase(iterator pos) * * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ size_type unordered_erase(const key_type& key, std::size_t precalculated_hash) { return m_ht.unordered_erase(key, precalculated_hash); } /** * @copydoc unordered_erase(iterator pos) * * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. * If so, K must be hashable and comparable to Key. */ template::value>::type* = nullptr> size_type unordered_erase(const K& key) { return m_ht.unordered_erase(key); } /** * @copydoc unordered_erase(const K& key) * * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ template::value>::type* = nullptr> size_type unordered_erase(const K& key, std::size_t precalculated_hash) { return m_ht.unordered_erase(key, precalculated_hash); } /** * Serialize the map through the `serializer` parameter. * * The `serializer` parameter must be a function object that supports the following call: * - `template void operator()(const U& value);` where the types `std::uint64_t`, `float` and `std::pair` must be supported for U. * * The implementation leaves binary compatibilty (endianness, IEEE 754 for floats, ...) of the types it serializes * in the hands of the `Serializer` function object if compatibilty is required. */ template void serialize(Serializer& serializer) const { m_ht.serialize(serializer); } /** * Deserialize a previouly serialized map through the `deserializer` parameter. * * The `deserializer` parameter must be a function object that supports the following calls: * - `template U operator()();` where the types `std::uint64_t`, `float` and `std::pair` must be supported for U. * * If the deserialized hash map type is hash compatible with the serialized map, the deserialization process can be * sped up by setting `hash_compatible` to true. To be hash compatible, the Hash and KeyEqual must behave the same way * than the ones used on the serialized map. The `std::size_t` must also be of the same size as the one on the platform used * to serialize the map, the same apply for `IndexType`. If these criteria are not met, the behaviour is undefined with * `hash_compatible` sets to true. * * The behaviour is undefined if the type `Key` and `T` of the `ordered_map` are not the same as the * types used during serialization. * * The implementation leaves binary compatibilty (endianness, IEEE 754 for floats, size of int, ...) of the types it * deserializes in the hands of the `Deserializer` function object if compatibilty is required. */ template static ordered_map deserialize(Deserializer& deserializer, bool hash_compatible = false) { ordered_map map(0); map.m_ht.deserialize(deserializer, hash_compatible); return map; } friend bool operator==(const ordered_map& lhs, const ordered_map& rhs) { return lhs.m_ht == rhs.m_ht; } friend bool operator!=(const ordered_map& lhs, const ordered_map& rhs) { return lhs.m_ht != rhs.m_ht; } friend bool operator<(const ordered_map& lhs, const ordered_map& rhs) { return lhs.m_ht < rhs.m_ht; } friend bool operator<=(const ordered_map& lhs, const ordered_map& rhs) { return lhs.m_ht <= rhs.m_ht; } friend bool operator>(const ordered_map& lhs, const ordered_map& rhs) { return lhs.m_ht > rhs.m_ht; } friend bool operator>=(const ordered_map& lhs, const ordered_map& rhs) { return lhs.m_ht >= rhs.m_ht; } friend void swap(ordered_map& lhs, ordered_map& rhs) { lhs.swap(rhs); } private: ht m_ht; }; } // end namespace tsl #endif ================================================ FILE: extern/tsl-ordered-map/ordered_set.h ================================================ /** * MIT License * * Copyright (c) 2017 Tessil * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef TSL_ORDERED_SET_H #define TSL_ORDERED_SET_H #include #include #include #include #include #include #include #include #include #include "ordered_hash.h" namespace tsl { /** * Implementation of an hash set using open adressing with robin hood with backshift delete to resolve collisions. * * The particularity of this hash set is that it remembers the order in which the elements were added and * provide a way to access the structure which stores these values through the 'values_container()' method. * The used container is defined by ValueTypeContainer, by default a std::deque is used (grows faster) but * a std::vector may be used. In this case the set provides a 'data()' method which give a direct access * to the memory used to store the values (which can be usefull to communicate with C API's). * * The Key must be copy constructible and/or move constructible. To use `unordered_erase` it also must be swappable. * * The behaviour of the hash set is undefinded if the destructor of Key throws an exception. * * By default the maximum size of a set is limited to 2^32 - 1 values, if needed this can be changed through * the IndexType template parameter. Using an `uint64_t` will raise this limit to 2^64 - 1 values but each * bucket will use 16 bytes instead of 8 bytes in addition to the space needed to store the values. * * Iterators invalidation: * - clear, operator=, reserve, rehash: always invalidate the iterators (also invalidate end()). * - insert, emplace, emplace_hint, operator[]: when a std::vector is used as ValueTypeContainer * and if size() < capacity(), only end(). * Otherwise all the iterators are invalidated if an insert occurs. * - erase, unordered_erase: when a std::vector is used as ValueTypeContainer invalidate the iterator of * the erased element and all the ones after the erased element (including end()). * Otherwise all the iterators are invalidated if an erase occurs. */ template, class KeyEqual = std::equal_to, class Allocator = std::allocator, class ValueTypeContainer = std::deque, class IndexType = std::uint_least32_t> class ordered_set { private: template using has_is_transparent = tsl::detail_ordered_hash::has_is_transparent; class KeySelect { public: using key_type = Key; const key_type& operator()(const Key& key) const noexcept { return key; } key_type& operator()(Key& key) noexcept { return key; } }; using ht = detail_ordered_hash::ordered_hash; public: using key_type = typename ht::key_type; using value_type = typename ht::value_type; using size_type = typename ht::size_type; using difference_type = typename ht::difference_type; using hasher = typename ht::hasher; using key_equal = typename ht::key_equal; using allocator_type = typename ht::allocator_type; using reference = typename ht::reference; using const_reference = typename ht::const_reference; using pointer = typename ht::pointer; using const_pointer = typename ht::const_pointer; using iterator = typename ht::iterator; using const_iterator = typename ht::const_iterator; using reverse_iterator = typename ht::reverse_iterator; using const_reverse_iterator = typename ht::const_reverse_iterator; using values_container_type = typename ht::values_container_type; /* * Constructors */ ordered_set(): ordered_set(ht::DEFAULT_INIT_BUCKETS_SIZE) { } explicit ordered_set(size_type bucket_count, const Hash& hash = Hash(), const KeyEqual& equal = KeyEqual(), const Allocator& alloc = Allocator()): m_ht(bucket_count, hash, equal, alloc, ht::DEFAULT_MAX_LOAD_FACTOR) { } ordered_set(size_type bucket_count, const Allocator& alloc): ordered_set(bucket_count, Hash(), KeyEqual(), alloc) { } ordered_set(size_type bucket_count, const Hash& hash, const Allocator& alloc): ordered_set(bucket_count, hash, KeyEqual(), alloc) { } explicit ordered_set(const Allocator& alloc): ordered_set(ht::DEFAULT_INIT_BUCKETS_SIZE, alloc) { } template ordered_set(InputIt first, InputIt last, size_type bucket_count = ht::DEFAULT_INIT_BUCKETS_SIZE, const Hash& hash = Hash(), const KeyEqual& equal = KeyEqual(), const Allocator& alloc = Allocator()): ordered_set(bucket_count, hash, equal, alloc) { insert(first, last); } template ordered_set(InputIt first, InputIt last, size_type bucket_count, const Allocator& alloc): ordered_set(first, last, bucket_count, Hash(), KeyEqual(), alloc) { } template ordered_set(InputIt first, InputIt last, size_type bucket_count, const Hash& hash, const Allocator& alloc): ordered_set(first, last, bucket_count, hash, KeyEqual(), alloc) { } ordered_set(std::initializer_list init, size_type bucket_count = ht::DEFAULT_INIT_BUCKETS_SIZE, const Hash& hash = Hash(), const KeyEqual& equal = KeyEqual(), const Allocator& alloc = Allocator()): ordered_set(init.begin(), init.end(), bucket_count, hash, equal, alloc) { } ordered_set(std::initializer_list init, size_type bucket_count, const Allocator& alloc): ordered_set(init.begin(), init.end(), bucket_count, Hash(), KeyEqual(), alloc) { } ordered_set(std::initializer_list init, size_type bucket_count, const Hash& hash, const Allocator& alloc): ordered_set(init.begin(), init.end(), bucket_count, hash, KeyEqual(), alloc) { } ordered_set& operator=(std::initializer_list ilist) { m_ht.clear(); m_ht.reserve(ilist.size()); m_ht.insert(ilist.begin(), ilist.end()); return *this; } allocator_type get_allocator() const { return m_ht.get_allocator(); } /* * Iterators */ iterator begin() noexcept { return m_ht.begin(); } const_iterator begin() const noexcept { return m_ht.begin(); } const_iterator cbegin() const noexcept { return m_ht.cbegin(); } iterator end() noexcept { return m_ht.end(); } const_iterator end() const noexcept { return m_ht.end(); } const_iterator cend() const noexcept { return m_ht.cend(); } reverse_iterator rbegin() noexcept { return m_ht.rbegin(); } const_reverse_iterator rbegin() const noexcept { return m_ht.rbegin(); } const_reverse_iterator rcbegin() const noexcept { return m_ht.rcbegin(); } reverse_iterator rend() noexcept { return m_ht.rend(); } const_reverse_iterator rend() const noexcept { return m_ht.rend(); } const_reverse_iterator rcend() const noexcept { return m_ht.rcend(); } /* * Capacity */ bool empty() const noexcept { return m_ht.empty(); } size_type size() const noexcept { return m_ht.size(); } size_type max_size() const noexcept { return m_ht.max_size(); } /* * Modifiers */ void clear() noexcept { m_ht.clear(); } std::pair insert(const value_type& value) { return m_ht.insert(value); } std::pair insert(value_type&& value) { return m_ht.insert(std::move(value)); } iterator insert(const_iterator hint, const value_type& value) { return m_ht.insert_hint(hint, value); } iterator insert(const_iterator hint, value_type&& value) { return m_ht.insert_hint(hint, std::move(value)); } template void insert(InputIt first, InputIt last) { m_ht.insert(first, last); } void insert(std::initializer_list ilist) { m_ht.insert(ilist.begin(), ilist.end()); } /** * Due to the way elements are stored, emplace will need to move or copy the key-value once. * The method is equivalent to insert(value_type(std::forward(args)...)); * * Mainly here for compatibility with the std::unordered_map interface. */ template std::pair emplace(Args&&... args) { return m_ht.emplace(std::forward(args)...); } /** * Due to the way elements are stored, emplace_hint will need to move or copy the key-value once. * The method is equivalent to insert(hint, value_type(std::forward(args)...)); * * Mainly here for compatibility with the std::unordered_map interface. */ template iterator emplace_hint(const_iterator hint, Args&&... args) { return m_ht.emplace_hint(hint, std::forward(args)...); } /** * When erasing an element, the insert order will be preserved and no holes will be present in the container * returned by 'values_container()'. * * The method is in O(n), if the order is not important 'unordered_erase(...)' method is faster with an O(1) * average complexity. */ iterator erase(iterator pos) { return m_ht.erase(pos); } /** * @copydoc erase(iterator pos) */ iterator erase(const_iterator pos) { return m_ht.erase(pos); } /** * @copydoc erase(iterator pos) */ iterator erase(const_iterator first, const_iterator last) { return m_ht.erase(first, last); } /** * @copydoc erase(iterator pos) */ size_type erase(const key_type& key) { return m_ht.erase(key); } /** * @copydoc erase(iterator pos) * * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup to the value if you already have the hash. */ size_type erase(const key_type& key, std::size_t precalculated_hash) { return m_ht.erase(key, precalculated_hash); } /** * @copydoc erase(iterator pos) * * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. * If so, K must be hashable and comparable to Key. */ template::value>::type* = nullptr> size_type erase(const K& key) { return m_ht.erase(key); } /** * @copydoc erase(const key_type& key, std::size_t precalculated_hash) * * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. * If so, K must be hashable and comparable to Key. */ template::value>::type* = nullptr> size_type erase(const K& key, std::size_t precalculated_hash) { return m_ht.erase(key, precalculated_hash); } void swap(ordered_set& other) { other.m_ht.swap(m_ht); } /* * Lookup */ size_type count(const Key& key) const { return m_ht.count(key); } /** * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ size_type count(const Key& key, std::size_t precalculated_hash) const { return m_ht.count(key, precalculated_hash); } /** * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. * If so, K must be hashable and comparable to Key. */ template::value>::type* = nullptr> size_type count(const K& key) const { return m_ht.count(key); } /** * @copydoc count(const K& key) const * * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ template::value>::type* = nullptr> size_type count(const K& key, std::size_t precalculated_hash) const { return m_ht.count(key, precalculated_hash); } iterator find(const Key& key) { return m_ht.find(key); } /** * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ iterator find(const Key& key, std::size_t precalculated_hash) { return m_ht.find(key, precalculated_hash); } const_iterator find(const Key& key) const { return m_ht.find(key); } /** * @copydoc find(const Key& key, std::size_t precalculated_hash) */ const_iterator find(const Key& key, std::size_t precalculated_hash) const { return m_ht.find(key, precalculated_hash); } /** * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. * If so, K must be hashable and comparable to Key. */ template::value>::type* = nullptr> iterator find(const K& key) { return m_ht.find(key); } /** * @copydoc find(const K& key) * * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ template::value>::type* = nullptr> iterator find(const K& key, std::size_t precalculated_hash) { return m_ht.find(key, precalculated_hash); } /** * @copydoc find(const K& key) */ template::value>::type* = nullptr> const_iterator find(const K& key) const { return m_ht.find(key); } /** * @copydoc find(const K& key) * * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ template::value>::type* = nullptr> const_iterator find(const K& key, std::size_t precalculated_hash) const { return m_ht.find(key, precalculated_hash); } std::pair equal_range(const Key& key) { return m_ht.equal_range(key); } /** * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ std::pair equal_range(const Key& key, std::size_t precalculated_hash) { return m_ht.equal_range(key, precalculated_hash); } std::pair equal_range(const Key& key) const { return m_ht.equal_range(key); } /** * @copydoc equal_range(const Key& key, std::size_t precalculated_hash) */ std::pair equal_range(const Key& key, std::size_t precalculated_hash) const { return m_ht.equal_range(key, precalculated_hash); } /** * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. * If so, K must be hashable and comparable to Key. */ template::value>::type* = nullptr> std::pair equal_range(const K& key) { return m_ht.equal_range(key); } /** * @copydoc equal_range(const K& key) * * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ template::value>::type* = nullptr> std::pair equal_range(const K& key, std::size_t precalculated_hash) { return m_ht.equal_range(key, precalculated_hash); } /** * @copydoc equal_range(const K& key) */ template::value>::type* = nullptr> std::pair equal_range(const K& key) const { return m_ht.equal_range(key); } /** * @copydoc equal_range(const K& key, std::size_t precalculated_hash) */ template::value>::type* = nullptr> std::pair equal_range(const K& key, std::size_t precalculated_hash) const { return m_ht.equal_range(key, precalculated_hash); } /* * Bucket interface */ size_type bucket_count() const { return m_ht.bucket_count(); } size_type max_bucket_count() const { return m_ht.max_bucket_count(); } /* * Hash policy */ float load_factor() const { return m_ht.load_factor(); } float max_load_factor() const { return m_ht.max_load_factor(); } void max_load_factor(float ml) { m_ht.max_load_factor(ml); } void rehash(size_type count) { m_ht.rehash(count); } void reserve(size_type count) { m_ht.reserve(count); } /* * Observers */ hasher hash_function() const { return m_ht.hash_function(); } key_equal key_eq() const { return m_ht.key_eq(); } /* * Other */ /** * Convert a const_iterator to an iterator. */ iterator mutable_iterator(const_iterator pos) { return m_ht.mutable_iterator(pos); } /** * Requires index <= size(). * * Return an iterator to the element at index. Return end() if index == size(). */ iterator nth(size_type index) { return m_ht.nth(index); } /** * @copydoc nth(size_type index) */ const_iterator nth(size_type index) const { return m_ht.nth(index); } /** * Return const_reference to the first element. Requires the container to not be empty. */ const_reference front() const { return m_ht.front(); } /** * Return const_reference to the last element. Requires the container to not be empty. */ const_reference back() const { return m_ht.back(); } /** * Only available if ValueTypeContainer is a std::vector. Same as calling 'values_container().data()'. */ template::value>::type* = nullptr> const typename values_container_type::value_type* data() const noexcept { return m_ht.data(); } /** * Return the container in which the values are stored. The values are in the same order as the insertion order * and are contiguous in the structure, no holes (size() == values_container().size()). */ const values_container_type& values_container() const noexcept { return m_ht.values_container(); } template::value>::type* = nullptr> size_type capacity() const noexcept { return m_ht.capacity(); } void shrink_to_fit() { m_ht.shrink_to_fit(); } /** * Insert the value before pos shifting all the elements on the right of pos (including pos) one position * to the right. * * Amortized linear time-complexity in the distance between pos and end(). */ std::pair insert_at_position(const_iterator pos, const value_type& value) { return m_ht.insert_at_position(pos, value); } /** * @copydoc insert_at_position(const_iterator pos, const value_type& value) */ std::pair insert_at_position(const_iterator pos, value_type&& value) { return m_ht.insert_at_position(pos, std::move(value)); } /** * @copydoc insert_at_position(const_iterator pos, const value_type& value) * * Same as insert_at_position(pos, value_type(std::forward(args)...), mainly * here for coherence. */ template std::pair emplace_at_position(const_iterator pos, Args&&... args) { return m_ht.emplace_at_position(pos, std::forward(args)...); } void pop_back() { m_ht.pop_back(); } /** * Faster erase operation with an O(1) average complexity but it doesn't preserve the insertion order. * * If an erasure occurs, the last element of the map will take the place of the erased element. */ iterator unordered_erase(iterator pos) { return m_ht.unordered_erase(pos); } /** * @copydoc unordered_erase(iterator pos) */ iterator unordered_erase(const_iterator pos) { return m_ht.unordered_erase(pos); } /** * @copydoc unordered_erase(iterator pos) */ size_type unordered_erase(const key_type& key) { return m_ht.unordered_erase(key); } /** * @copydoc unordered_erase(iterator pos) * * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ size_type unordered_erase(const key_type& key, std::size_t precalculated_hash) { return m_ht.unordered_erase(key, precalculated_hash); } /** * @copydoc unordered_erase(iterator pos) * * This overload only participates in the overload resolution if the typedef KeyEqual::is_transparent exists. * If so, K must be hashable and comparable to Key. */ template::value>::type* = nullptr> size_type unordered_erase(const K& key) { return m_ht.unordered_erase(key); } /** * @copydoc unordered_erase(const K& key) * * Use the hash value 'precalculated_hash' instead of hashing the key. The hash value should be the same * as hash_function()(key). Usefull to speed-up the lookup if you already have the hash. */ template::value>::type* = nullptr> size_type unordered_erase(const K& key, std::size_t precalculated_hash) { return m_ht.unordered_erase(key, precalculated_hash); } /** * Serialize the set through the `serializer` parameter. * * The `serializer` parameter must be a function object that supports the following call: * - `void operator()(const U& value);` where the types `std::uint64_t`, `float` and `Key` must be supported for U. * * The implementation leaves binary compatibilty (endianness, IEEE 754 for floats, ...) of the types it serializes * in the hands of the `Serializer` function object if compatibilty is required. */ template void serialize(Serializer& serializer) const { m_ht.serialize(serializer); } /** * Deserialize a previouly serialized set through the `deserializer` parameter. * * The `deserializer` parameter must be a function object that supports the following calls: * - `template U operator()();` where the types `std::uint64_t`, `float` and `Key` must be supported for U. * * If the deserialized hash set type is hash compatible with the serialized set, the deserialization process can be * sped up by setting `hash_compatible` to true. To be hash compatible, the Hash and KeyEqual must behave the same way * than the ones used on the serialized map. The `std::size_t` must also be of the same size as the one on the platform used * to serialize the map, the same apply for `IndexType`. If these criteria are not met, the behaviour is undefined with * `hash_compatible` sets to true. * * The behaviour is undefined if the type `Key` of the `ordered_set` is not the same as the * type used during serialization. * * The implementation leaves binary compatibilty (endianness, IEEE 754 for floats, size of int, ...) of the types it * deserializes in the hands of the `Deserializer` function object if compatibilty is required. */ template static ordered_set deserialize(Deserializer& deserializer, bool hash_compatible = false) { ordered_set set(0); set.m_ht.deserialize(deserializer, hash_compatible); return set; } friend bool operator==(const ordered_set& lhs, const ordered_set& rhs) { return lhs.m_ht == rhs.m_ht; } friend bool operator!=(const ordered_set& lhs, const ordered_set& rhs) { return lhs.m_ht != rhs.m_ht; } friend bool operator<(const ordered_set& lhs, const ordered_set& rhs) { return lhs.m_ht < rhs.m_ht; } friend bool operator<=(const ordered_set& lhs, const ordered_set& rhs) { return lhs.m_ht <= rhs.m_ht; } friend bool operator>(const ordered_set& lhs, const ordered_set& rhs) { return lhs.m_ht > rhs.m_ht; } friend bool operator>=(const ordered_set& lhs, const ordered_set& rhs) { return lhs.m_ht >= rhs.m_ht; } friend void swap(ordered_set& lhs, ordered_set& rhs) { lhs.swap(rhs); } private: ht m_ht; }; } // end namespace tsl #endif ================================================ FILE: extern/xxHash/.gitattributes ================================================ # Set the default behavior * text eol=lf # Explicitly declare source files *.c text eol=lf *.h text eol=lf # Denote files that should not be modified. *.odt binary ================================================ FILE: extern/xxHash/.gitignore ================================================ # objects *.o # libraries libxxhash.* # Executables xxh32sum xxh64sum xxhsum xxhsum32 xxhsum_privateXXH xxhsum_inlinedXXH # Mac OS-X artefacts *.dSYM .DS_Store ================================================ FILE: extern/xxHash/.travis.yml ================================================ language: c compiler: gcc script: make -B test-all before_install: - sudo apt-get update -qq - sudo apt-get install -qq gcc-arm-linux-gnueabi - sudo apt-get install -qq clang - sudo apt-get install -qq g++-multilib - sudo apt-get install -qq gcc-multilib ================================================ FILE: extern/xxHash/LICENSE ================================================ xxHash Library Copyright (c) 2012-2014, Yann Collet All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: extern/xxHash/Makefile ================================================ # ################################################################ # xxHash Makefile # Copyright (C) Yann Collet 2012-2015 # # GPL v2 License # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # You can contact the author at : # - xxHash source repository : http://code.google.com/p/xxhash/ # ################################################################ # xxhsum : provides 32/64 bits hash of one or multiple files, or stdin # ################################################################ # Version numbers LIBVER_MAJOR_SCRIPT:=`sed -n '/define XXH_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < xxhash.h` LIBVER_MINOR_SCRIPT:=`sed -n '/define XXH_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < xxhash.h` LIBVER_PATCH_SCRIPT:=`sed -n '/define XXH_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < xxhash.h` LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) LIBVER := $(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH) # SSE4 detection HAVE_SSE4 := $(shell $(CC) -dM -E - < /dev/null | grep "SSE4" > /dev/null && echo 1 || echo 0) ifeq ($(HAVE_SSE4), 1) NOSSE4 := -mno-sse4 else NOSSE4 := endif CFLAGS ?= -O2 $(NOSSE4) # disables potential auto-vectorization CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $(MOREFLAGS) XXHSUM_VERSION=$(LIBVER) MD2ROFF = ronn MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="xxhsum $(XXHSUM_VERSION)" # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) EXT =.exe else EXT = endif # OS X linker doesn't support -soname, and use different extension # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html ifeq ($(shell uname), Darwin) SHARED_EXT = dylib SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT) SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT) SONAME_FLAGS = -install_name $(LIBDIR)/libxxhash.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER) else SONAME_FLAGS = -Wl,-soname=libxxhash.$(SHARED_EXT).$(LIBVER_MAJOR) SHARED_EXT = so SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR) SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) endif LIBXXH = libxxhash.$(SHARED_EXT_VER) .PHONY: default default: lib xxhsum_and_links .PHONY: all all: lib xxhsum xxhsum_inlinedXXH xxhsum32: CFLAGS += -m32 xxhsum xxhsum32: xxhash.c xxhsum.c $(CC) $(FLAGS) $^ -o $@$(EXT) .PHONY: xxhsum_and_links xxhsum_and_links: xxhsum ln -sf xxhsum xxh32sum ln -sf xxhsum xxh64sum xxhsum_inlinedXXH: xxhsum.c $(CC) $(FLAGS) -DXXH_PRIVATE_API $^ -o $@$(EXT) # library libxxhash.a: ARFLAGS = rcs libxxhash.a: xxhash.o @echo compiling static library @$(AR) $(ARFLAGS) $@ $^ $(LIBXXH): LDFLAGS += -shared ifeq (,$(filter Windows%,$(OS))) $(LIBXXH): LDFLAGS += -fPIC endif $(LIBXXH): xxhash.c @echo compiling dynamic library $(LIBVER) @$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ @echo creating versioned links @ln -sf $@ libxxhash.$(SHARED_EXT_MAJOR) @ln -sf $@ libxxhash.$(SHARED_EXT) libxxhash : $(LIBXXH) lib: libxxhash.a libxxhash # tests .PHONY: check check: xxhsum # stdin ./xxhsum < xxhash.c # multiple files ./xxhsum xxhash.* xxhsum.* # internal bench ./xxhsum -bi1 # file bench ./xxhsum -bi1 xxhash.c .PHONY: test-mem test-mem: xxhsum # memory tests valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -bi1 xxhash.c valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -H0 xxhash.c valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -H1 xxhash.c .PHONY: test32 test32: clean xxhsum32 @echo ---- test 32-bit ---- ./xxhsum32 -bi1 xxhash.c test-xxhsum-c: xxhsum # xxhsum to/from pipe ./xxhsum lib* | ./xxhsum -c - ./xxhsum -H0 lib* | ./xxhsum -c - # xxhsum to/from file, shell redirection ./xxhsum lib* > .test.xxh64 ./xxhsum -H0 lib* > .test.xxh32 ./xxhsum -c .test.xxh64 ./xxhsum -c .test.xxh32 ./xxhsum -c < .test.xxh64 ./xxhsum -c < .test.xxh32 # xxhsum -c warns improperly format lines. cat .test.xxh64 .test.xxh32 | ./xxhsum -c - cat .test.xxh32 .test.xxh64 | ./xxhsum -c - # Expects "FAILED" echo "0000000000000000 LICENSE" | ./xxhsum -c -; test $$? -eq 1 echo "00000000 LICENSE" | ./xxhsum -c -; test $$? -eq 1 # Expects "FAILED open or read" echo "0000000000000000 test-expects-file-not-found" | ./xxhsum -c -; test $$? -eq 1 echo "00000000 test-expects-file-not-found" | ./xxhsum -c -; test $$? -eq 1 @$(RM) -f .test.xxh32 .test.xxh64 armtest: clean @echo ---- test ARM compilation ---- $(MAKE) xxhsum CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror -static" clangtest: clean @echo ---- test clang compilation ---- $(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion" gpptest: clean @echo ---- test g++ compilation ---- $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" c90test: clean @echo ---- test strict C90 compilation [xxh32 only] ---- $(CC) -std=c90 -Werror -pedantic -DXXH_NO_LONG_LONG -c xxhash.c $(RM) xxhash.o usan: CC=clang usan: clean @echo ---- check undefined behavior - sanitize ---- $(MAKE) clean test CC=$(CC) MOREFLAGS="-g -fsanitize=undefined -fno-sanitize-recover=all" staticAnalyze: clean @echo ---- static analyzer - scan-build ---- CFLAGS="-g -Werror" scan-build --status-bugs -v $(MAKE) all namespaceTest: $(CC) -c xxhash.c $(CC) -DXXH_NAMESPACE=TEST_ -c xxhash.c -o xxhash2.o $(CC) xxhash.o xxhash2.o xxhsum.c -o xxhsum2 # will fail if one namespace missing (symbol collision) $(RM) *.o xxhsum2 # clean xxhsum.1: xxhsum.1.md cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@ man: xxhsum.1 clean-man: $(RM) xxhsum.1 preview-man: clean-man man man ./xxhsum.1 test: all namespaceTest check test-xxhsum-c c90test test-all: test test32 armtest clangtest gpptest usan listL120 trailingWhitespace staticAnalyze .PHONY: listL120 listL120: # extract lines >= 120 characters in *.{c,h}, by Takayuki Matsuoka (note : $$, for Makefile compatibility) find . -type f -name '*.c' -o -name '*.h' | while read -r filename; do awk 'length > 120 {print FILENAME "(" FNR "): " $$0}' $$filename; done .PHONY: trailingWhitespace trailingWhitespace: ! grep -E "`printf '[ \\t]$$'`" *.1 *.c *.h LICENSE Makefile cmake_unofficial/CMakeLists.txt .PHONY: clean clean: @$(RM) -r *.dSYM # Mac OS-X specific @$(RM) core *.o libxxhash.* @$(RM) xxhsum$(EXT) xxhsum32$(EXT) xxhsum_inlinedXXH$(EXT) xxh32sum xxh64sum @echo cleaning completed #----------------------------------------------------------------------------- # make install is validated only for the following targets #----------------------------------------------------------------------------- ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) .PHONY: list list: @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs DESTDIR ?= # directory variables : GNU conventions prefer lowercase # see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html # support both lower and uppercase (BSD), use uppercase in script prefix ?= /usr/local PREFIX ?= $(prefix) exec_prefix ?= $(PREFIX) libdir ?= $(exec_prefix)/lib LIBDIR ?= $(libdir) includedir ?= $(PREFIX)/include INCLUDEDIR ?= $(includedir) bindir ?= $(exec_prefix)/bin BINDIR ?= $(bindir) datarootdir ?= $(PREFIX)/share mandir ?= $(datarootdir)/man man1dir ?= $(mandir)/man1 ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly SunOS)) MANDIR ?= $(PREFIX)/man/man1 else MANDIR ?= $(man1dir) endif ifneq (,$(filter $(shell uname),SunOS)) INSTALL ?= ginstall else INSTALL ?= install endif INSTALL_PROGRAM ?= $(INSTALL) INSTALL_DATA ?= $(INSTALL) -m 644 .PHONY: install install: lib xxhsum @echo Installing libxxhash @$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR) @$(INSTALL_DATA) libxxhash.a $(DESTDIR)$(LIBDIR) @$(INSTALL_PROGRAM) $(LIBXXH) $(DESTDIR)$(LIBDIR) @ln -sf $(LIBXXH) $(DESTDIR)$(LIBDIR)/libxxhash.$(SHARED_EXT_MAJOR) @ln -sf $(LIBXXH) $(DESTDIR)$(LIBDIR)/libxxhash.$(SHARED_EXT) @$(INSTALL) -d -m 755 $(DESTDIR)$(INCLUDEDIR) # includes @$(INSTALL_DATA) xxhash.h $(DESTDIR)$(INCLUDEDIR) @echo Installing xxhsum @$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/ @$(INSTALL_PROGRAM) xxhsum $(DESTDIR)$(BINDIR)/xxhsum @ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh32sum @ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh64sum @echo Installing man pages @$(INSTALL_DATA) xxhsum.1 $(DESTDIR)$(MANDIR)/xxhsum.1 @ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh32sum.1 @ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh64sum.1 @echo xxhash installation completed .PHONY: uninstall uninstall: @$(RM) $(DESTDIR)$(LIBDIR)/libxxhash.a @$(RM) $(DESTDIR)$(LIBDIR)/libxxhash.$(SHARED_EXT) @$(RM) $(DESTDIR)$(LIBDIR)/libxxhash.$(SHARED_EXT_MAJOR) @$(RM) $(DESTDIR)$(LIBDIR)/$(LIBXXH) @$(RM) $(DESTDIR)$(INCLUDEDIR)/xxhash.h @$(RM) $(DESTDIR)$(BINDIR)/xxh32sum @$(RM) $(DESTDIR)$(BINDIR)/xxh64sum @$(RM) $(DESTDIR)$(BINDIR)/xxhsum @$(RM) $(DESTDIR)$(MANDIR)/xxh32sum.1 @$(RM) $(DESTDIR)$(MANDIR)/xxh64sum.1 @$(RM) $(DESTDIR)$(MANDIR)/xxhsum.1 @echo xxhsum successfully uninstalled endif ================================================ FILE: extern/xxHash/README.md ================================================ xxHash - Extremely fast hash algorithm ====================================== xxHash is an Extremely fast Hash algorithm, running at RAM speed limits. It successfully completes the [SMHasher](http://code.google.com/p/smhasher/wiki/SMHasher) test suite which evaluates collision, dispersion and randomness qualities of hash functions. Code is highly portable, and hashes are identical on all platforms (little / big endian). |Branch |Status | |------------|---------| |master | [![Build Status](https://travis-ci.org/Cyan4973/xxHash.svg?branch=master)](https://travis-ci.org/Cyan4973/xxHash?branch=master) | |dev | [![Build Status](https://travis-ci.org/Cyan4973/xxHash.svg?branch=dev)](https://travis-ci.org/Cyan4973/xxHash?branch=dev) | Benchmarks ------------------------- The benchmark uses SMHasher speed test, compiled with Visual 2010 on a Windows Seven 32-bit box. The reference system uses a Core 2 Duo @3GHz | Name | Speed | Quality | Author | |---------------|----------|:-------:|------------------| | [xxHash] | 5.4 GB/s | 10 | Y.C. | | MurmurHash 3a | 2.7 GB/s | 10 | Austin Appleby | | SBox | 1.4 GB/s | 9 | Bret Mulvey | | Lookup3 | 1.2 GB/s | 9 | Bob Jenkins | | CityHash64 | 1.05 GB/s| 10 | Pike & Alakuijala| | FNV | 0.55 GB/s| 5 | Fowler, Noll, Vo | | CRC32 | 0.43 GB/s| 9 | | | MD5-32 | 0.33 GB/s| 10 | Ronald L.Rivest | | SHA1-32 | 0.28 GB/s| 10 | | [xxHash]: http://www.xxhash.com Q.Score is a measure of quality of the hash function. It depends on successfully passing SMHasher test set. 10 is a perfect score. Algorithms with a score < 5 are not listed on this table. A more recent version, XXH64, has been created thanks to [Mathias Westerdahl](https://github.com/JCash), which offers superior speed and dispersion for 64-bit systems. Note however that 32-bit applications will still run faster using the 32-bit version. SMHasher speed test, compiled using GCC 4.8.2, on Linux Mint 64-bit. The reference system uses a Core i5-3340M @2.7GHz | Version | Speed on 64-bit | Speed on 32-bit | |------------|------------------|------------------| | XXH64 | 13.8 GB/s | 1.9 GB/s | | XXH32 | 6.8 GB/s | 6.0 GB/s | This project also includes a command line utility, named `xxhsum`, offering similar features as `md5sum`, thanks to [Takayuki Matsuoka](https://github.com/t-mat) contributions. ### License The library files `xxhash.c` and `xxhash.h` are BSD licensed. The utility `xxhsum` is GPL licensed. ### Build modifiers The following macros can be set at compilation time, they modify xxhash behavior. They are all disabled by default. - `XXH_INLINE_ALL` : Make all functions `inline`, with bodies directly included within `xxhash.h`. There is no need for an `xxhash.o` module in this case. Inlining functions is generally beneficial for speed on small keys. It's especially effective when key length is a compile time constant, with observed performance improvement in the +200% range . See [this article](https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html) for details. - `XXH_ACCEPT_NULL_INPUT_POINTER` : if set to `1`, when input is a null-pointer, xxhash result is the same as a zero-length key (instead of a dereference segfault). - `XXH_FORCE_MEMORY_ACCESS` : default method `0` uses a portable `memcpy()` notation. Method `1` uses a gcc-specific `packed` attribute, which can provide better performance for some targets. Method `2` forces unaligned reads, which is not standard compliant, but might sometimes be the only way to extract better performance. - `XXH_CPU_LITTLE_ENDIAN` : by default, endianess is determined at compile time. It's possible to skip auto-detection and force format to little-endian, by setting this macro to 1. Setting it to 0 forces big-endian. - `XXH_FORCE_NATIVE_FORMAT` : on big-endian systems : use native number representation. Breaks consistency with little-endian results. - `XXH_PRIVATE_API` : same impact as `XXH_INLINE_ALL`. Name underlines that symbols will not be published on library public interface. - `XXH_NAMESPACE` : prefix all symbols with the value of `XXH_NAMESPACE`. Useful to evade symbol naming collisions, in case of multiple inclusions of xxHash source code. Client applications can still use regular function name, symbols are automatically translated through `xxhash.h`. - `XXH_STATIC_LINKING_ONLY` : gives access to state declaration for static allocation. Incompatible with dynamic linking, due to risks of ABI changes. - `XXH_NO_LONG_LONG` : removes support for XXH64, for targets without 64-bit support. ### Example Calling xxhash 64-bit variant from a C program : ``` #include "xxhash.h" unsigned long long calcul_hash(const void* buffer, size_t length) { unsigned long long const seed = 0; /* or any other value */ unsigned long long const hash = XXH64(buffer, length, seed); return hash; } ``` Using streaming variant is more involved, but makes it possible to provide data in multiple rounds : ``` #include "stdlib.h" /* abort() */ #include "xxhash.h" unsigned long long calcul_hash_streaming(someCustomType handler) { XXH64_state_t* const state = XXH64_createState(); if (state==NULL) abort(); size_t const bufferSize = SOME_VALUE; void* const buffer = malloc(bufferSize); if (buffer==NULL) abort(); unsigned long long const seed = 0; /* or any other value */ XXH_errorcode const resetResult = XXH64_reset(state, seed); if (resetResult == XXH_ERROR) abort(); (...) while ( /* any condition */ ) { size_t const length = get_more_data(buffer, bufferSize, handler); /* undescribed */ XXH_errorcode const addResult = XXH64_update(state, buffer, length); if (addResult == XXH_ERROR) abort(); (...) } (...) unsigned long long const hash = XXH64_digest(state); free(buffer); XXH64_freeState(state); return hash; } ``` ### Other programming languages Beyond the C reference version, xxHash is also available on many programming languages, thanks to great contributors. They are [listed here](http://www.xxhash.com/#other-languages). ### Branch Policy > - The "master" branch is considered stable, at all times. > - The "dev" branch is the one where all contributions must be merged before being promoted to master. > + If you plan to propose a patch, please commit into the "dev" branch, or its own feature branch. Direct commit to "master" are not permitted. ================================================ FILE: extern/xxHash/appveyor.yml ================================================ version: 1.0.{build} environment: matrix: - COMPILER: "gcc" PLATFORM: "mingw64" - COMPILER: "gcc" PLATFORM: "mingw32" install: - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% - MKDIR bin - if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH% - if [%COMPILER%]==[gcc] ( SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe && COPY C:\MinGW\bin\gcc.exe C:\MinGW\bin\cc.exe ) else ( IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;") ) build_script: - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - if [%PLATFORM%]==[clang] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - ECHO *** && ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION% && ECHO *** - if [%PLATFORM%]==[clang] (clang -v) - if [%COMPILER%]==[gcc] (gcc -v) - if [%COMPILER%]==[gcc] ( echo ----- && make -v && echo ----- && if not [%PLATFORM%]==[clang] ( make -B clean test MOREFLAGS=-Werror ) ELSE ( make -B clean test CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion" ) ) - if [%COMPILER%]==[visual] ( ECHO *** && ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% && ECHO *** && msbuild "visual\VS2010\lz4.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /p:EnableWholeProgramOptimization=true /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && ECHO *** && ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% && ECHO *** && msbuild "visual\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && ECHO *** && ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% && ECHO *** && msbuild "visual\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && ECHO *** && ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && ECHO *** && msbuild "visual\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && COPY visual\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe programs\ ) test_script: - ECHO *** && ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION% && ECHO *** - if not [%COMPILER%]==[unknown] ( xxhsum -h && xxhsum xxhsum.exe && xxhsum -bi1 && echo ------- xxhsum tested ------- ) ================================================ FILE: extern/xxHash/cmake_unofficial/.gitignore ================================================ # cmake artifacts CMakeCache.txt CMakeFiles Makefile cmake_install.cmake # make compilation results libxxhash.0.6.3.dylib libxxhash.0.dylib libxxhash.a libxxhash.dylib ================================================ FILE: extern/xxHash/cmake_unofficial/CMakeLists.txt ================================================ # To the extent possible under law, the author(s) have dedicated all # copyright and related and neighboring rights to this software to # the public domain worldwide. This software is distributed without # any warranty. # # For details, see . set(XXHASH_DIR "${CMAKE_CURRENT_SOURCE_DIR}/..") file(STRINGS "${XXHASH_DIR}/xxhash.h" XXHASH_VERSION_MAJOR REGEX "^#define XXH_VERSION_MAJOR +([0-9]+) *$") string(REGEX REPLACE "^#define XXH_VERSION_MAJOR +([0-9]+) *$" "\\1" XXHASH_VERSION_MAJOR "${XXHASH_VERSION_MAJOR}") file(STRINGS "${XXHASH_DIR}/xxhash.h" XXHASH_VERSION_MINOR REGEX "^#define XXH_VERSION_MINOR +([0-9]+) *$") string(REGEX REPLACE "^#define XXH_VERSION_MINOR +([0-9]+) *$" "\\1" XXHASH_VERSION_MINOR "${XXHASH_VERSION_MINOR}") file(STRINGS "${XXHASH_DIR}/xxhash.h" XXHASH_VERSION_RELEASE REGEX "^#define XXH_VERSION_RELEASE +([0-9]+) *$") string(REGEX REPLACE "^#define XXH_VERSION_RELEASE +([0-9]+) *$" "\\1" XXHASH_VERSION_RELEASE "${XXHASH_VERSION_RELEASE}") set(XXHASH_VERSION_STRING "${XXHASH_VERSION_MAJOR}.${XXHASH_VERSION_MINOR}.${XXHASH_VERSION_RELEASE}") set(XXHASH_LIB_VERSION ${XXHASH_VERSION_STRING}) set(XXHASH_LIB_SOVERSION "${XXHASH_VERSION_MAJOR}") mark_as_advanced(XXHASH_VERSION_MAJOR XXHASH_VERSION_MINOR XXHASH_VERSION_RELEASE XXHASH_VERSION_STRING XXHASH_LIB_VERSION XXHASH_LIB_SOVERSION) option(BUILD_XXHSUM "Build the xxhsum binary" OFF) option(BUILD_SHARED_LIBS "Build shared library" OFF) if("${CMAKE_VERSION}" VERSION_LESS "3.0") project(XXHASH C) else() cmake_policy (SET CMP0048 NEW) project(XXHASH VERSION ${XXHASH_VERSION_STRING} LANGUAGES C) endif() cmake_minimum_required (VERSION 3.6) # If XXHASH is being bundled in another project, we don't want to # install anything. However, we want to let people override this, so # we'll use the XXHASH_BUNDLED_MODE variable to let them do that; just # set it to OFF in your project before you add_subdirectory(xxhash/contrib/cmake_unofficial). if(CMAKE_CURRENT_SOURCE_DIR STREQUAL "${CMAKE_SOURCE_DIR}") # Bundled mode hasn't been set one way or the other, set the default # depending on whether or not we are the top-level project. if("${XXHASH_PARENT_DIRECTORY}" STREQUAL "") set(XXHASH_BUNDLED_MODE OFF) else() set(XXHASH_BUNDLED_MODE ON) endif() endif() mark_as_advanced(XXHASH_BUNDLED_MODE) # Allow people to choose whether to build shared or static libraries # via the BUILD_SHARED_LIBS option unless we are in bundled mode, in # which case we always use static libraries. include(CMakeDependentOption) CMAKE_DEPENDENT_OPTION(BUILD_SHARED_LIBS "Build shared libraries" ON "NOT XXHASH_BUNDLED_MODE" OFF) include_directories("${XXHASH_DIR}") # libxxhash add_library(xxhash "${XXHASH_DIR}/xxhash.c") set_target_properties(xxhash PROPERTIES SOVERSION "${XXHASH_VERSION_STRING}" VERSION "${XXHASH_VERSION_STRING}") # xxhsum add_executable(xxhsum "${XXHASH_DIR}/xxhsum.c") target_link_libraries(xxhsum xxhash) # Extra warning flags include (CheckCCompilerFlag) foreach (flag -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef) # Because https://gcc.gnu.org/wiki/FAQ#wnowarning string(REGEX REPLACE "\\-Wno\\-(.+)" "-W\\1" flag_to_test "${flag}") string(REGEX REPLACE "[^a-zA-Z0-9]+" "_" test_name "CFLAG_${flag_to_test}") check_c_compiler_flag("${ADD_COMPILER_FLAGS_PREPEND} ${flag_to_test}" ${test_name}) if(${test_name}) set(CMAKE_C_FLAGS "${flag} ${CMAKE_C_FLAGS}") endif() unset(test_name) unset(flag_to_test) endforeach (flag) if(NOT XXHASH_BUNDLED_MODE) include(GNUInstallDirs) install(TARGETS xxhsum RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") install(TARGETS xxhash LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}") install(FILES "${XXHASH_DIR}/xxhash.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") install(FILES "${XXHASH_DIR}/xxhsum.1" DESTINATION "${CMAKE_INSTALL_MANDIR}/man1") endif(NOT XXHASH_BUNDLED_MODE) ================================================ FILE: extern/xxHash/cmake_unofficial/README.md ================================================ The `cmake` script present in this directory offers the following options : - `BUILD_XXHSUM` : build the command line binary. ON by default - `BUILD_SHARED_LIBS` : build dynamic library. ON by default. ================================================ FILE: extern/xxHash/doc/xxhash_spec.md ================================================ xxHash fast digest algorithm ====================== ### Notices Copyright (c) Yann Collet Permission is granted to copy and distribute this document for any purpose and without charge, including translations into other languages and incorporation into compilations, provided that the copyright notice and this notice are preserved, and that any substantive changes or deletions from the original are clearly marked. Distribution of this document is unlimited. ### Version 0.1.0 (15/01/18) Table of Contents --------------------- - [Introduction](#introduction) - [XXH32 algorithm description](#xxh32-algorithm-description) - [XXH64 algorithm description](#xxh64-algorithm-description) - [Performance considerations](#performance-considerations) - [Reference Implementation](#reference-implementation) Introduction ---------------- This document describes the xxHash digest algorithm, for both 32 and 64 variants, named `XXH32` and `XXH64`. The algorithm takes as input a message of arbitrary length and an optional seed value, it then produces an output of 32 or 64-bit as "fingerprint" or "digest". xxHash is primarily designed for speed. It is labelled non-cryptographic, and is not meant to avoid intentional collisions (same digest for 2 different messages), or to prevent producing a message with predefined digest. XXH32 is designed to be fast on 32-bits machines. XXH64 is designed to be fast on 64-bits machines. Both variants produce different output. However, a given variant shall produce exactly the same output, irrespective of the cpu / os used. In particular, the result remains identical whatever the endianness and width of the cpu. ### Operation notations All operations are performed modulo {32,64} bits. Arithmetic overflows are expected. `XXH32` uses 32-bit modular operations. `XXH64` uses 64-bit modular operations. - `+` : denote modular addition - `*` : denote modular multiplication - `X <<< s` : denote the value obtained by circularly shifting (rotating) `X` left by `s` bit positions. - `X >> s` : denote the value obtained by shifting `X` right by s bit positions. Upper `s` bits become `0`. - `X xor Y` : denote the bit-wise XOR of `X` and `Y` (same width). XXH32 Algorithm Description ------------------------------------- ### Overview We begin by supposing that we have a message of any length `L` as input, and that we wish to find its digest. Here `L` is an arbitrary nonnegative integer; `L` may be zero. The following steps are performed to compute the digest of the message. The algorithm collect and transform input in _stripes_ of 16 bytes. The transforms are stored inside 4 "accumulators", each one storing an unsigned 32-bit value. Each accumulator can be processed independently in parallel, speeding up processing for cpu with multiple execution units. The algorithm uses 32-bits addition, multiplication, rotate, shift and xor operations. Many operations require some 32-bits prime number constants, all defined below : static const u32 PRIME32_1 = 2654435761U; static const u32 PRIME32_2 = 2246822519U; static const u32 PRIME32_3 = 3266489917U; static const u32 PRIME32_4 = 668265263U; static const u32 PRIME32_5 = 374761393U; ### Step 1. Initialise internal accumulators Each accumulator gets an initial value based on optional `seed` input. Since the `seed` is optional, it can be `0`. u32 acc1 = seed + PRIME32_1 + PRIME32_2; u32 acc2 = seed + PRIME32_2; u32 acc3 = seed + 0; u32 acc4 = seed - PRIME32_1; #### Special case : input is less than 16 bytes When input is too small (< 16 bytes), the algorithm will not process any stripe. Consequently, it will not make use of parallel accumulators. In which case, a simplified initialization is performed, using a single accumulator : u32 acc = seed + PRIME32_5; The algorithm then proceeds directly to step 4. ### Step 2. Process stripes A stripe is a contiguous segment of 16 bytes. It is evenly divided into 4 _lanes_, of 4 bytes each. The first lane is used to update accumulator 1, the second lane is used to update accumulator 2, and so on. Each lane read its associated 32-bit value using __little-endian__ convention. For each {lane, accumulator}, the update process is called a _round_, and applies the following formula : accN = accN + (laneN * PRIME32_2); accN = accN <<< 13; accN = accN * PRIME32_1; This shuffles the bits so that any bit from input _lane_ impacts several bits in output _accumulator_. All operations are performed modulo 2^32. Input is consumed one full stripe at a time. Step 2 is looped as many times as necessary to consume the whole input, except the last remaining bytes which cannot form a stripe (< 16 bytes). When that happens, move to step 3. ### Step 3. Accumulator convergence All 4 lane accumulators from previous steps are merged to produce a single remaining accumulator of same width (32-bit). The associated formula is as follows : acc = (acc1 <<< 1) + (acc2 <<< 7) + (acc3 <<< 12) + (acc4 <<< 18); ### Step 4. Add input length The input total length is presumed known at this stage. This step is just about adding the length to accumulator, so that it participates to final mixing. acc = acc + (u32)inputLength; Note that, if input length is so large that it requires more than 32-bits, only the lower 32-bits are added to the accumulator. ### Step 5. Consume remaining input There may be up to 15 bytes remaining to consume from the input. The final stage will digest them according to following pseudo-code : while (remainingLength >= 4) { lane = read_32bit_little_endian(input_ptr); acc = acc + lane * PRIME32_3; acc = (acc <<< 17) * PRIME32_4; input_ptr += 4; remainingLength -= 4; } while (remainingLength >= 1) { lane = read_byte(input_ptr); acc = acc + lane * PRIME32_5; acc = (acc <<< 11) * PRIME32_1; input_ptr += 1; remainingLength -= 1; } This process ensures that all input bytes are present in the final mix. ### Step 6. Final mix (avalanche) The final mix ensures that all input bits have a chance to impact any bit in the output digest, resulting in an unbiased distribution. This is also called avalanche effect. acc = acc xor (acc >> 15); acc = acc * PRIME32_2; acc = acc xor (acc >> 13); acc = acc * PRIME32_3; acc = acc xor (acc >> 16); ### Step 7. Output The `XXH32()` function produces an unsigned 32-bit value as output. For systems which require to store and/or display the result in binary or hexadecimal format, the canonical format is defined to reproduce the same value as the natural decimal format, hence follows __big-endian__ convention (most significant byte first). XXH64 Algorithm Description ------------------------------------- ### Overview `XXH64` algorithm structure is very similar to `XXH32` one. The major difference is that `XXH64` uses 64-bit arithmetic, speeding up memory transfer for 64-bit compliant systems, but also relying on cpu capability to efficiently perform 64-bit operations. The algorithm collects and transforms input in _stripes_ of 32 bytes. The transforms are stored inside 4 "accumulators", each one storing an unsigned 64-bit value. Each accumulator can be processed independently in parallel, speeding up processing for cpu with multiple execution units. The algorithm uses 64-bit addition, multiplication, rotate, shift and xor operations. Many operations require some 64-bit prime number constants, all defined below : static const u64 PRIME64_1 = 11400714785074694791ULL; static const u64 PRIME64_2 = 14029467366897019727ULL; static const u64 PRIME64_3 = 1609587929392839161ULL; static const u64 PRIME64_4 = 9650029242287828579ULL; static const u64 PRIME64_5 = 2870177450012600261ULL; ### Step 1. Initialise internal accumulators Each accumulator gets an initial value based on optional `seed` input. Since the `seed` is optional, it can be `0`. u64 acc1 = seed + PRIME64_1 + PRIME64_2; u64 acc2 = seed + PRIME64_2; u64 acc3 = seed + 0; u64 acc4 = seed - PRIME64_1; #### Special case : input is less than 32 bytes When input is too small (< 32 bytes), the algorithm will not process any stripe. Consequently, it will not make use of parallel accumulators. In which case, a simplified initialization is performed, using a single accumulator : u64 acc = seed + PRIME64_5; The algorithm then proceeds directly to step 4. ### Step 2. Process stripes A stripe is a contiguous segment of 32 bytes. It is evenly divided into 4 _lanes_, of 8 bytes each. The first lane is used to update accumulator 1, the second lane is used to update accumulator 2, and so on. Each lane read its associated 64-bit value using __little-endian__ convention. For each {lane, accumulator}, the update process is called a _round_, and applies the following formula : round(accN,laneN): accN = accN + (laneN * PRIME64_2); accN = accN <<< 31; return accN * PRIME64_1; This shuffles the bits so that any bit from input _lane_ impacts several bits in output _accumulator_. All operations are performed modulo 2^64. Input is consumed one full stripe at a time. Step 2 is looped as many times as necessary to consume the whole input, except the last remaining bytes which cannot form a stripe (< 32 bytes). When that happens, move to step 3. ### Step 3. Accumulator convergence All 4 lane accumulators from previous steps are merged to produce a single remaining accumulator of same width (64-bit). The associated formula is as follows. Note that accumulator convergence is more complex than 32-bit variant, and requires to define another function called _mergeAccumulator()_ : mergeAccumulator(acc,accN): acc = acc xor round(0, accN); acc = acc * PRIME64_1 return acc + PRIME64_4; which is then used in the convergence formula : acc = (acc1 <<< 1) + (acc2 <<< 7) + (acc3 <<< 12) + (acc4 <<< 18); acc = mergeAccumulator(acc, acc1); acc = mergeAccumulator(acc, acc2); acc = mergeAccumulator(acc, acc3); acc = mergeAccumulator(acc, acc4); ### Step 4. Add input length The input total length is presumed known at this stage. This step is just about adding the length to accumulator, so that it participates to final mixing. acc = acc + inputLength; ### Step 5. Consume remaining input There may be up to 31 bytes remaining to consume from the input. The final stage will digest them according to following pseudo-code : while (remainingLength >= 8) { lane = read_64bit_little_endian(input_ptr); acc = acc xor round(0, lane); acc = (acc <<< 27) * PRIME64_1; acc = acc + PRIME64_4; input_ptr += 8; remainingLength -= 8; } if (remainingLength >= 4) { lane = read_32bit_little_endian(input_ptr); acc = acc xor (lane * PRIME64_1); acc = (acc <<< 23) * PRIME64_2; acc = acc + PRIME64_3; input_ptr += 4; remainingLength -= 4; } while (remainingLength >= 1) { lane = read_byte(input_ptr); acc = acc xor (lane * PRIME64_5); acc = (acc <<< 11) * PRIME64_1; input_ptr += 1; remainingLength -= 1; } This process ensures that all input bytes are present in the final mix. ### Step 6. Final mix (avalanche) The final mix ensures that all input bits have a chance to impact any bit in the output digest, resulting in an unbiased distribution. This is also called avalanche effect. acc = acc xor (acc >> 33); acc = acc * PRIME64_2; acc = acc xor (acc >> 29); acc = acc * PRIME64_3; acc = acc xor (acc >> 32); ### Step 7. Output The `XXH64()` function produces an unsigned 64-bit value as output. For systems which require to store and/or display the result in binary or hexadecimal format, the canonical format is defined to reproduce the same value as the natural decimal format, hence follows __big-endian__ convention (most significant byte first). Performance considerations ---------------------------------- The xxHash algorithms are simple and compact to implement. They provide a system independent "fingerprint" or digest of a message of arbitrary length. The algorithm allows input to be streamed and processed in multiple steps. In such case, an internal buffer is needed to ensure data is presented to the algorithm in full stripes. On 64-bit systems, the 64-bit variant `XXH64` is generally faster to compute, so it is a recommended variant, even when only 32-bit are needed. On 32-bit systems though, positions are reversed : `XXH64` performance is reduced, due to its usage of 64-bit arithmetic. `XXH32` becomes a faster variant. Reference Implementation ---------------------------------------- A reference library written in C is available at http://www.xxhash.com . The web page also links to multiple other implementations written in many different languages. It links to the [github project page](https://github.com/Cyan4973/xxHash) where an [issue board](https://github.com/Cyan4973/xxHash/issues) can be used for further public discussions on the topic. Version changes -------------------- v0.1.0 : initial release ================================================ FILE: extern/xxHash/xxhash.c ================================================ /* * xxHash - Fast Hash algorithm * Copyright (C) 2012-2016, Yann Collet * * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following disclaimer * in the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * You can contact the author at : * - xxHash homepage: http://www.xxhash.com * - xxHash source repository : https://github.com/Cyan4973/xxHash */ /* ************************************* * Tuning parameters ***************************************/ /*!XXH_FORCE_MEMORY_ACCESS : * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. * The below switch allow to select different access method for improved performance. * Method 0 (default) : use `memcpy()`. Safe and portable. * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. * It can generate buggy code on targets which do not support unaligned memory accesses. * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) * See http://stackoverflow.com/a/32095106/646947 for details. * Prefer these methods in priority order (0 > 1 > 2) */ #ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \ || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) \ || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define XXH_FORCE_MEMORY_ACCESS 2 # elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \ || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) \ || defined(__ARM_ARCH_7S__) )) # define XXH_FORCE_MEMORY_ACCESS 1 # endif #endif /*!XXH_ACCEPT_NULL_INPUT_POINTER : * If input pointer is NULL, xxHash default behavior is to dereference it, triggering a segfault. * When this macro is enabled, xxHash actively checks input for null pointer. * It it is, result for null input pointers is the same as a null-length input. */ #ifndef XXH_ACCEPT_NULL_INPUT_POINTER /* can be defined externally */ # define XXH_ACCEPT_NULL_INPUT_POINTER 0 #endif /*!XXH_FORCE_NATIVE_FORMAT : * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. * Results are therefore identical for little-endian and big-endian CPU. * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. * Should endian-independence be of no importance for your application, you may set the #define below to 1, * to improve speed for Big-endian CPU. * This option has no impact on Little_Endian CPU. */ #ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ # define XXH_FORCE_NATIVE_FORMAT 0 #endif /*!XXH_FORCE_ALIGN_CHECK : * This is a minor performance trick, only useful with lots of very small keys. * It means : check for aligned/unaligned input. * The check costs one initial branch per hash; * set it to 0 when the input is guaranteed to be aligned, * or when alignment doesn't matter for performance. */ #ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ # if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) # define XXH_FORCE_ALIGN_CHECK 0 # else # define XXH_FORCE_ALIGN_CHECK 1 # endif #endif /* ************************************* * Includes & Memory related functions ***************************************/ /*! Modify the local functions below should you wish to use some other memory routines * for malloc(), free() */ #include static void* XXH_malloc(size_t s) { return malloc(s); } static void XXH_free (void* p) { free(p); } /*! and for memcpy() */ #include static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } #include /* assert */ #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" /* ************************************* * Compiler Specific Options ***************************************/ #ifdef _MSC_VER /* Visual Studio */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # define FORCE_INLINE static __forceinline #else # if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # ifdef __GNUC__ # define FORCE_INLINE static inline __attribute__((always_inline)) # else # define FORCE_INLINE static inline # endif # else # define FORCE_INLINE static # endif /* __STDC_VERSION__ */ #endif /* ************************************* * Basic Types ***************************************/ #ifndef MEM_MODULE # if !defined (__VMS) \ && (defined (__cplusplus) \ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include typedef uint8_t BYTE; typedef uint16_t U16; typedef uint32_t U32; # else typedef unsigned char BYTE; typedef unsigned short U16; typedef unsigned int U32; # endif #endif #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) /* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ /* currently only defined for gcc and icc */ typedef union { U32 u32; } __attribute__((packed)) unalign; static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } #else /* portable and safe solution. Generally efficient. * see : http://stackoverflow.com/a/32095106/646947 */ static U32 XXH_read32(const void* memPtr) { U32 val; memcpy(&val, memPtr, sizeof(val)); return val; } #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ /* **************************************** * Compiler-specific Functions and Macros ******************************************/ #define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) /* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ #if defined(_MSC_VER) # define XXH_rotl32(x,r) _rotl(x,r) # define XXH_rotl64(x,r) _rotl64(x,r) #else # define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) # define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) #endif #if defined(_MSC_VER) /* Visual Studio */ # define XXH_swap32 _byteswap_ulong #elif XXH_GCC_VERSION >= 403 # define XXH_swap32 __builtin_bswap32 #else static U32 XXH_swap32 (U32 x) { return ((x << 24) & 0xff000000 ) | ((x << 8) & 0x00ff0000 ) | ((x >> 8) & 0x0000ff00 ) | ((x >> 24) & 0x000000ff ); } #endif /* ************************************* * Architecture Macros ***************************************/ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ #ifndef XXH_CPU_LITTLE_ENDIAN static int XXH_isLittleEndian(void) { const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ return one.c[0]; } # define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() #endif /* *************************** * Memory reads *****************************/ typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); else return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); } FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); } static U32 XXH_readBE32(const void* ptr) { return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); } /* ************************************* * Macros ***************************************/ #define XXH_STATIC_ASSERT(c) { enum { XXH_sa = 1/(int)(!!(c)) }; } /* use after variable declarations */ XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } /* ******************************************************************* * 32-bit hash functions *********************************************************************/ static const U32 PRIME32_1 = 2654435761U; static const U32 PRIME32_2 = 2246822519U; static const U32 PRIME32_3 = 3266489917U; static const U32 PRIME32_4 = 668265263U; static const U32 PRIME32_5 = 374761393U; static U32 XXH32_round(U32 seed, U32 input) { seed += input * PRIME32_2; seed = XXH_rotl32(seed, 13); seed *= PRIME32_1; return seed; } /* mix all bits */ static U32 XXH32_avalanche(U32 h32) { h32 ^= h32 >> 15; h32 *= PRIME32_2; h32 ^= h32 >> 13; h32 *= PRIME32_3; h32 ^= h32 >> 16; return(h32); } #define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) static U32 XXH32_finalize(U32 h32, const void* ptr, size_t len, XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)ptr; #define PROCESS1 \ h32 += (*p) * PRIME32_5; \ p++; \ h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; #define PROCESS4 \ h32 += XXH_get32bits(p) * PRIME32_3; \ p+=4; \ h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; switch(len&15) /* or switch(bEnd - p) */ { case 12: PROCESS4; /* fallthrough */ case 8: PROCESS4; /* fallthrough */ case 4: PROCESS4; return XXH32_avalanche(h32); case 13: PROCESS4; /* fallthrough */ case 9: PROCESS4; /* fallthrough */ case 5: PROCESS4; PROCESS1; return XXH32_avalanche(h32); case 14: PROCESS4; /* fallthrough */ case 10: PROCESS4; /* fallthrough */ case 6: PROCESS4; PROCESS1; PROCESS1; return XXH32_avalanche(h32); case 15: PROCESS4; /* fallthrough */ case 11: PROCESS4; /* fallthrough */ case 7: PROCESS4; /* fallthrough */ case 3: PROCESS1; /* fallthrough */ case 2: PROCESS1; /* fallthrough */ case 1: PROCESS1; /* fallthrough */ case 0: return XXH32_avalanche(h32); } assert(0); return h32; /* reaching this point is deemed impossible */ } FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; const BYTE* bEnd = p + len; U32 h32; #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) if (p==NULL) { len=0; bEnd=p=(const BYTE*)(size_t)16; } #endif if (len>=16) { const BYTE* const limit = bEnd - 15; U32 v1 = seed + PRIME32_1 + PRIME32_2; U32 v2 = seed + PRIME32_2; U32 v3 = seed + 0; U32 v4 = seed - PRIME32_1; do { v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; } while (p < limit); h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); } else { h32 = seed + PRIME32_5; } h32 += (U32)len; return XXH32_finalize(h32, p, len&15, endian, align); } XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) { #if 0 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ XXH32_state_t state; XXH32_reset(&state, seed); XXH32_update(&state, input, len); return XXH32_digest(&state); #else XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; if (XXH_FORCE_ALIGN_CHECK) { if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); else return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); } } if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); else return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); #endif } /*====== Hash streaming ======*/ XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) { return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); } XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) { XXH_free(statePtr); return XXH_OK; } XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) { memcpy(dstState, srcState, sizeof(*dstState)); } XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) { XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ memset(&state, 0, sizeof(state)); state.v1 = seed + PRIME32_1 + PRIME32_2; state.v2 = seed + PRIME32_2; state.v3 = seed + 0; state.v4 = seed - PRIME32_1; /* do not write into reserved, planned to be removed in a future version */ memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); return XXH_OK; } FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) { const BYTE* p = (const BYTE*)input; const BYTE* const bEnd = p + len; if (input==NULL) #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) return XXH_OK; #else return XXH_ERROR; #endif state->total_len_32 += (unsigned)len; state->large_len |= (len>=16) | (state->total_len_32>=16); if (state->memsize + len < 16) { /* fill in tmp buffer */ XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); state->memsize += (unsigned)len; return XXH_OK; } if (state->memsize) { /* some data left from previous update */ XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); { const U32* p32 = state->mem32; state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); } p += 16-state->memsize; state->memsize = 0; } if (p <= bEnd-16) { const BYTE* const limit = bEnd - 16; U32 v1 = state->v1; U32 v2 = state->v2; U32 v3 = state->v3; U32 v4 = state->v4; do { v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; } while (p<=limit); state->v1 = v1; state->v2 = v2; state->v3 = v3; state->v4 = v4; } if (p < bEnd) { XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); state->memsize = (unsigned)(bEnd-p); } return XXH_OK; } XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH32_update_endian(state_in, input, len, XXH_littleEndian); else return XXH32_update_endian(state_in, input, len, XXH_bigEndian); } FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) { U32 h32; if (state->large_len) { h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); } else { h32 = state->v3 /* == seed */ + PRIME32_5; } h32 += state->total_len_32; return XXH32_finalize(h32, state->mem32, state->memsize, endian, XXH_aligned); } XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH32_digest_endian(state_in, XXH_littleEndian); else return XXH32_digest_endian(state_in, XXH_bigEndian); } /*====== Canonical representation ======*/ /*! Default XXH result types are basic unsigned 32 and 64 bits. * The canonical representation follows human-readable write convention, aka big-endian (large digits first). * These functions allow transformation of hash result into and from its canonical format. * This way, hash values can be written into a file or buffer, remaining comparable across different systems. */ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) { XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); memcpy(dst, &hash, sizeof(*dst)); } XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) { return XXH_readBE32(src); } #ifndef XXH_NO_LONG_LONG /* ******************************************************************* * 64-bit hash functions *********************************************************************/ /*====== Memory access ======*/ #ifndef MEM_MODULE # define MEM_MODULE # if !defined (__VMS) \ && (defined (__cplusplus) \ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include typedef uint64_t U64; # else /* if compiler doesn't support unsigned long long, replace by another 64-bit type */ typedef unsigned long long U64; # endif #endif #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) /* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ /* currently only defined for gcc and icc */ typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64; static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; } #else /* portable and safe solution. Generally efficient. * see : http://stackoverflow.com/a/32095106/646947 */ static U64 XXH_read64(const void* memPtr) { U64 val; memcpy(&val, memPtr, sizeof(val)); return val; } #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ #if defined(_MSC_VER) /* Visual Studio */ # define XXH_swap64 _byteswap_uint64 #elif XXH_GCC_VERSION >= 403 # define XXH_swap64 __builtin_bswap64 #else static U64 XXH_swap64 (U64 x) { return ((x << 56) & 0xff00000000000000ULL) | ((x << 40) & 0x00ff000000000000ULL) | ((x << 24) & 0x0000ff0000000000ULL) | ((x << 8) & 0x000000ff00000000ULL) | ((x >> 8) & 0x00000000ff000000ULL) | ((x >> 24) & 0x0000000000ff0000ULL) | ((x >> 40) & 0x000000000000ff00ULL) | ((x >> 56) & 0x00000000000000ffULL); } #endif FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); else return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); } FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) { return XXH_readLE64_align(ptr, endian, XXH_unaligned); } static U64 XXH_readBE64(const void* ptr) { return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); } /*====== xxh64 ======*/ static const U64 PRIME64_1 = 11400714785074694791ULL; static const U64 PRIME64_2 = 14029467366897019727ULL; static const U64 PRIME64_3 = 1609587929392839161ULL; static const U64 PRIME64_4 = 9650029242287828579ULL; static const U64 PRIME64_5 = 2870177450012600261ULL; static U64 XXH64_round(U64 acc, U64 input) { acc += input * PRIME64_2; acc = XXH_rotl64(acc, 31); acc *= PRIME64_1; return acc; } static U64 XXH64_mergeRound(U64 acc, U64 val) { val = XXH64_round(0, val); acc ^= val; acc = acc * PRIME64_1 + PRIME64_4; return acc; } static U64 XXH64_avalanche(U64 h64) { h64 ^= h64 >> 33; h64 *= PRIME64_2; h64 ^= h64 >> 29; h64 *= PRIME64_3; h64 ^= h64 >> 32; return h64; } #define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) static U64 XXH64_finalize(U64 h64, const void* ptr, size_t len, XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)ptr; #define PROCESS1_64 \ h64 ^= (*p) * PRIME64_5; \ p++; \ h64 = XXH_rotl64(h64, 11) * PRIME64_1; #define PROCESS4_64 \ h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; \ p+=4; \ h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; #define PROCESS8_64 { \ U64 const k1 = XXH64_round(0, XXH_get64bits(p)); \ p+=8; \ h64 ^= k1; \ h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; \ } switch(len&31) { case 24: PROCESS8_64; /* fallthrough */ case 16: PROCESS8_64; /* fallthrough */ case 8: PROCESS8_64; return XXH64_avalanche(h64); case 28: PROCESS8_64; /* fallthrough */ case 20: PROCESS8_64; /* fallthrough */ case 12: PROCESS8_64; /* fallthrough */ case 4: PROCESS4_64; return XXH64_avalanche(h64); case 25: PROCESS8_64; /* fallthrough */ case 17: PROCESS8_64; /* fallthrough */ case 9: PROCESS8_64; PROCESS1_64; return XXH64_avalanche(h64); case 29: PROCESS8_64; /* fallthrough */ case 21: PROCESS8_64; /* fallthrough */ case 13: PROCESS8_64; /* fallthrough */ case 5: PROCESS4_64; PROCESS1_64; return XXH64_avalanche(h64); case 26: PROCESS8_64; /* fallthrough */ case 18: PROCESS8_64; /* fallthrough */ case 10: PROCESS8_64; PROCESS1_64; PROCESS1_64; return XXH64_avalanche(h64); case 30: PROCESS8_64; /* fallthrough */ case 22: PROCESS8_64; /* fallthrough */ case 14: PROCESS8_64; /* fallthrough */ case 6: PROCESS4_64; PROCESS1_64; PROCESS1_64; return XXH64_avalanche(h64); case 27: PROCESS8_64; /* fallthrough */ case 19: PROCESS8_64; /* fallthrough */ case 11: PROCESS8_64; PROCESS1_64; PROCESS1_64; PROCESS1_64; return XXH64_avalanche(h64); case 31: PROCESS8_64; /* fallthrough */ case 23: PROCESS8_64; /* fallthrough */ case 15: PROCESS8_64; /* fallthrough */ case 7: PROCESS4_64; /* fallthrough */ case 3: PROCESS1_64; /* fallthrough */ case 2: PROCESS1_64; /* fallthrough */ case 1: PROCESS1_64; /* fallthrough */ case 0: return XXH64_avalanche(h64); } /* impossible to reach */ assert(0); return 0; /* unreachable, but some compilers complain without it */ } FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; const BYTE* bEnd = p + len; U64 h64; #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) if (p==NULL) { len=0; bEnd=p=(const BYTE*)(size_t)32; } #endif if (len>=32) { const BYTE* const limit = bEnd - 32; U64 v1 = seed + PRIME64_1 + PRIME64_2; U64 v2 = seed + PRIME64_2; U64 v3 = seed + 0; U64 v4 = seed - PRIME64_1; do { v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; } while (p<=limit); h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); h64 = XXH64_mergeRound(h64, v1); h64 = XXH64_mergeRound(h64, v2); h64 = XXH64_mergeRound(h64, v3); h64 = XXH64_mergeRound(h64, v4); } else { h64 = seed + PRIME64_5; } h64 += (U64) len; return XXH64_finalize(h64, p, len, endian, align); } XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) { #if 0 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ XXH64_state_t state; XXH64_reset(&state, seed); XXH64_update(&state, input, len); return XXH64_digest(&state); #else XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; if (XXH_FORCE_ALIGN_CHECK) { if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); else return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); } } if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); else return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); #endif } /*====== Hash Streaming ======*/ XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) { return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); } XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) { XXH_free(statePtr); return XXH_OK; } XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState) { memcpy(dstState, srcState, sizeof(*dstState)); } XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) { XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ memset(&state, 0, sizeof(state)); state.v1 = seed + PRIME64_1 + PRIME64_2; state.v2 = seed + PRIME64_2; state.v3 = seed + 0; state.v4 = seed - PRIME64_1; /* do not write into reserved, planned to be removed in a future version */ memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); return XXH_OK; } FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) { const BYTE* p = (const BYTE*)input; const BYTE* const bEnd = p + len; if (input==NULL) #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) return XXH_OK; #else return XXH_ERROR; #endif state->total_len += len; if (state->memsize + len < 32) { /* fill in tmp buffer */ XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); state->memsize += (U32)len; return XXH_OK; } if (state->memsize) { /* tmp buffer is full */ XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); p += 32-state->memsize; state->memsize = 0; } if (p+32 <= bEnd) { const BYTE* const limit = bEnd - 32; U64 v1 = state->v1; U64 v2 = state->v2; U64 v3 = state->v3; U64 v4 = state->v4; do { v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; } while (p<=limit); state->v1 = v1; state->v2 = v2; state->v3 = v3; state->v4 = v4; } if (p < bEnd) { XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); state->memsize = (unsigned)(bEnd-p); } return XXH_OK; } XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH64_update_endian(state_in, input, len, XXH_littleEndian); else return XXH64_update_endian(state_in, input, len, XXH_bigEndian); } FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) { U64 h64; if (state->total_len >= 32) { U64 const v1 = state->v1; U64 const v2 = state->v2; U64 const v3 = state->v3; U64 const v4 = state->v4; h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); h64 = XXH64_mergeRound(h64, v1); h64 = XXH64_mergeRound(h64, v2); h64 = XXH64_mergeRound(h64, v3); h64 = XXH64_mergeRound(h64, v4); } else { h64 = state->v3 /*seed*/ + PRIME64_5; } h64 += (U64) state->total_len; return XXH64_finalize(h64, state->mem64, (size_t)state->total_len, endian, XXH_aligned); } XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH64_digest_endian(state_in, XXH_littleEndian); else return XXH64_digest_endian(state_in, XXH_bigEndian); } /*====== Canonical representation ======*/ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) { XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); memcpy(dst, &hash, sizeof(*dst)); } XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) { return XXH_readBE64(src); } #endif /* XXH_NO_LONG_LONG */ ================================================ FILE: extern/xxHash/xxhash.h ================================================ /* xxHash - Extremely Fast Hash algorithm Header File Copyright (C) 2012-2016, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - xxHash source repository : https://github.com/Cyan4973/xxHash */ /* Notice extracted from xxHash homepage : xxHash is an extremely fast Hash algorithm, running at RAM speed limits. It also successfully passes all tests from the SMHasher suite. Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) Name Speed Q.Score Author xxHash 5.4 GB/s 10 CrapWow 3.2 GB/s 2 Andrew MumurHash 3a 2.7 GB/s 10 Austin Appleby SpookyHash 2.0 GB/s 10 Bob Jenkins SBox 1.4 GB/s 9 Bret Mulvey Lookup3 1.2 GB/s 9 Bob Jenkins SuperFastHash 1.2 GB/s 1 Paul Hsieh CityHash64 1.05 GB/s 10 Pike & Alakuijala FNV 0.55 GB/s 5 Fowler, Noll, Vo CRC32 0.43 GB/s 9 MD5-32 0.33 GB/s 10 Ronald L. Rivest SHA1-32 0.28 GB/s 10 Q.Score is a measure of quality of the hash function. It depends on successfully passing SMHasher test set. 10 is a perfect score. A 64-bit version, named XXH64, is available since r35. It offers much better speed, but for 64-bit applications only. Name Speed on 64 bits Speed on 32 bits XXH64 13.8 GB/s 1.9 GB/s XXH32 6.8 GB/s 6.0 GB/s */ #ifndef XXHASH_H_5627135585666179 #define XXHASH_H_5627135585666179 1 #if defined (__cplusplus) extern "C" { #endif /* **************************** * Definitions ******************************/ #include /* size_t */ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; /* **************************** * API modifier ******************************/ /** XXH_INLINE_ALL (and XXH_PRIVATE_API) * This is useful to include xxhash functions in `static` mode * in order to inline them, and remove their symbol from the public list. * Inlining can offer dramatic performance improvement on small keys. * Methodology : * #define XXH_INLINE_ALL * #include "xxhash.h" * `xxhash.c` is automatically included. * It's not useful to compile and link it as a separate module. */ #if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) # ifndef XXH_STATIC_LINKING_ONLY # define XXH_STATIC_LINKING_ONLY # endif # if defined(__GNUC__) # define XXH_PUBLIC_API static __inline __attribute__((unused)) # elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) # define XXH_PUBLIC_API static inline # elif defined(_MSC_VER) # define XXH_PUBLIC_API static __inline # else /* this version may generate warnings for unused static functions */ # define XXH_PUBLIC_API static # endif #else # define XXH_PUBLIC_API /* do nothing */ #endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ /*! XXH_NAMESPACE, aka Namespace Emulation : * * If you want to include _and expose_ xxHash functions from within your own library, * but also want to avoid symbol collisions with other libraries which may also include xxHash, * * you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library * with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values). * * Note that no change is required within the calling program as long as it includes `xxhash.h` : * regular symbol name will be automatically translated by this header. */ #ifdef XXH_NAMESPACE # define XXH_CAT(A,B) A##B # define XXH_NAME2(A,B) XXH_CAT(A,B) # define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) # define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) # define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) # define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) # define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) # define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) # define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) # define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) # define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) # define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) # define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) # define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) # define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) # define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) # define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) # define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) # define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) # define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) # define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) #endif /* ************************************* * Version ***************************************/ #define XXH_VERSION_MAJOR 0 #define XXH_VERSION_MINOR 6 #define XXH_VERSION_RELEASE 5 #define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) XXH_PUBLIC_API unsigned XXH_versionNumber (void); /*-********************************************************************** * 32-bit hash ************************************************************************/ typedef unsigned int XXH32_hash_t; /*! XXH32() : Calculate the 32-bit hash of sequence "length" bytes stored at memory address "input". The memory between input & input+length must be valid (allocated and read-accessible). "seed" can be used to alter the result predictably. Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */ XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); /*====== Streaming ======*/ typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); /* * Streaming functions generate the xxHash of an input provided in multiple segments. * Note that, for small input, they are slower than single-call functions, due to state management. * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. * * XXH state must first be allocated, using XXH*_createState() . * * Start a new hash by initializing state with a seed, using XXH*_reset(). * * Then, feed the hash state by calling XXH*_update() as many times as necessary. * The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. * * Finally, a hash value can be produced anytime, by using XXH*_digest(). * This function returns the nn-bits hash as an int or long long. * * It's still possible to continue inserting input into the hash state after a digest, * and generate some new hashes later on, by calling again XXH*_digest(). * * When done, free XXH state space if it was allocated dynamically. */ /*====== Canonical representation ======*/ typedef struct { unsigned char digest[4]; } XXH32_canonical_t; XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); /* Default result type for XXH functions are primitive unsigned 32 and 64 bits. * The canonical representation uses human-readable write convention, aka big-endian (large digits first). * These functions allow transformation of hash result into and from its canonical format. * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. */ #ifndef XXH_NO_LONG_LONG /*-********************************************************************** * 64-bit hash ************************************************************************/ typedef unsigned long long XXH64_hash_t; /*! XXH64() : Calculate the 64-bit hash of sequence of length "len" stored at memory address "input". "seed" can be used to alter the result predictably. This function runs faster on 64-bit systems, but slower on 32-bit systems (see benchmark). */ XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); /*====== Streaming ======*/ typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); /*====== Canonical representation ======*/ typedef struct { unsigned char digest[8]; } XXH64_canonical_t; XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); #endif /* XXH_NO_LONG_LONG */ #ifdef XXH_STATIC_LINKING_ONLY /* ================================================================================================ This section contains declarations which are not guaranteed to remain stable. They may change in future versions, becoming incompatible with a different version of the library. These declarations should only be used with static linking. Never use them in association with dynamic linking ! =================================================================================================== */ /* These definitions are only present to allow * static allocation of XXH state, on stack or in a struct for example. * Never **ever** use members directly. */ #if !defined (__VMS) \ && (defined (__cplusplus) \ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include struct XXH32_state_s { uint32_t total_len_32; uint32_t large_len; uint32_t v1; uint32_t v2; uint32_t v3; uint32_t v4; uint32_t mem32[4]; uint32_t memsize; uint32_t reserved; /* never read nor write, might be removed in a future version */ }; /* typedef'd to XXH32_state_t */ struct XXH64_state_s { uint64_t total_len; uint64_t v1; uint64_t v2; uint64_t v3; uint64_t v4; uint64_t mem64[4]; uint32_t memsize; uint32_t reserved[2]; /* never read nor write, might be removed in a future version */ }; /* typedef'd to XXH64_state_t */ # else struct XXH32_state_s { unsigned total_len_32; unsigned large_len; unsigned v1; unsigned v2; unsigned v3; unsigned v4; unsigned mem32[4]; unsigned memsize; unsigned reserved; /* never read nor write, might be removed in a future version */ }; /* typedef'd to XXH32_state_t */ # ifndef XXH_NO_LONG_LONG /* remove 64-bit support */ struct XXH64_state_s { unsigned long long total_len; unsigned long long v1; unsigned long long v2; unsigned long long v3; unsigned long long v4; unsigned long long mem64[4]; unsigned memsize; unsigned reserved[2]; /* never read nor write, might be removed in a future version */ }; /* typedef'd to XXH64_state_t */ # endif # endif #if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) # include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */ #endif #endif /* XXH_STATIC_LINKING_ONLY */ #if defined (__cplusplus) } #endif #endif /* XXHASH_H_5627135585666179 */ ================================================ FILE: extern/xxHash/xxhsum.1 ================================================ . .TH "XXHSUM" "1" "September 2017" "xxhsum 0.6.3" "User Commands" . .SH "NAME" \fBxxhsum\fR \- print or check xxHash non\-cryptographic checksums . .SH "SYNOPSIS" \fBxxhsum [

shournal report

A ================================================ FILE: html-export/dist/main.js ================================================ /******/ (function(modules) { // webpackBootstrap /******/ // The module cache /******/ var installedModules = {}; /******/ /******/ // The require function /******/ function __webpack_require__(moduleId) { /******/ /******/ // Check if module is in cache /******/ if(installedModules[moduleId]) { /******/ return installedModules[moduleId].exports; /******/ } /******/ // Create a new module (and put it into the cache) /******/ var module = installedModules[moduleId] = { /******/ i: moduleId, /******/ l: false, /******/ exports: {} /******/ }; /******/ /******/ // Execute the module function /******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__); /******/ /******/ // Flag the module as loaded /******/ module.l = true; /******/ /******/ // Return the exports of the module /******/ return module.exports; /******/ } /******/ /******/ /******/ // expose the modules object (__webpack_modules__) /******/ __webpack_require__.m = modules; /******/ /******/ // expose the module cache /******/ __webpack_require__.c = installedModules; /******/ /******/ // define getter function for harmony exports /******/ __webpack_require__.d = function(exports, name, getter) { /******/ if(!__webpack_require__.o(exports, name)) { /******/ Object.defineProperty(exports, name, { enumerable: true, get: getter }); /******/ } /******/ }; /******/ /******/ // define __esModule on exports /******/ __webpack_require__.r = function(exports) { /******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) { /******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' }); /******/ } /******/ Object.defineProperty(exports, '__esModule', { value: true }); /******/ }; /******/ /******/ // create a fake namespace object /******/ // mode & 1: value is a module id, require it /******/ // mode & 2: merge all properties of value into the ns /******/ // mode & 4: return value when already ns object /******/ // mode & 8|1: behave like require /******/ __webpack_require__.t = function(value, mode) { /******/ if(mode & 1) value = __webpack_require__(value); /******/ if(mode & 8) return value; /******/ if((mode & 4) && typeof value === 'object' && value && value.__esModule) return value; /******/ var ns = Object.create(null); /******/ __webpack_require__.r(ns); /******/ Object.defineProperty(ns, 'default', { enumerable: true, value: value }); /******/ if(mode & 2 && typeof value != 'string') for(var key in value) __webpack_require__.d(ns, key, function(key) { return value[key]; }.bind(null, key)); /******/ return ns; /******/ }; /******/ /******/ // getDefaultExport function for compatibility with non-harmony modules /******/ __webpack_require__.n = function(module) { /******/ var getter = module && module.__esModule ? /******/ function getDefault() { return module['default']; } : /******/ function getModuleExports() { return module; }; /******/ __webpack_require__.d(getter, 'a', getter); /******/ return getter; /******/ }; /******/ /******/ // Object.prototype.hasOwnProperty.call /******/ __webpack_require__.o = function(object, property) { return Object.prototype.hasOwnProperty.call(object, property); }; /******/ /******/ // __webpack_public_path__ /******/ __webpack_require__.p = ""; /******/ /******/ /******/ // Load entry module and return exports /******/ return __webpack_require__(__webpack_require__.s = 2); /******/ }) /************************************************************************/ /******/ ([ /* 0 */ /***/ (function(module, exports, __webpack_require__) { "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); var Mutex_1 = __webpack_require__(1); exports.Mutex = Mutex_1.default; /***/ }), /* 1 */ /***/ (function(module, exports, __webpack_require__) { "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); var Mutex = /** @class */ (function () { function Mutex() { this._queue = []; this._pending = false; } Mutex.prototype.isLocked = function () { return this._pending; }; Mutex.prototype.acquire = function () { var _this = this; var ticket = new Promise(function (resolve) { return _this._queue.push(resolve); }); if (!this._pending) { this._dispatchNext(); } return ticket; }; Mutex.prototype.runExclusive = function (callback) { return this .acquire() .then(function (release) { var result; try { result = callback(); } catch (e) { release(); throw (e); } return Promise .resolve(result) .then(function (x) { return (release(), x); }, function (e) { release(); throw e; }); }); }; Mutex.prototype._dispatchNext = function () { if (this._queue.length > 0) { this._pending = true; this._queue.shift()(this._dispatchNext.bind(this)); } else { this._pending = false; } }; return Mutex; }()); exports.default = Mutex; /***/ }), /* 2 */ /***/ (function(module, __webpack_exports__, __webpack_require__) { "use strict"; __webpack_require__.r(__webpack_exports__); // CONCATENATED MODULE: ./src/generic_text_dialog.js class GenericTextDialog { constructor() { } show(title, content){ $("#genericModalTitle").html(title); $("#genericModalBody").html(content); $("#genericModal").modal('toggle'); } } // CONCATENATED MODULE: ./src/globals.js let humanDateFormat; let humanDateFormatOnlyDate; let humanDateFormatOnlyTime; let d3TimeParseIsoWithMil; let textDialog; let commandList; let sessionTimeline; function init(){ humanDateFormat = d3.timeFormat("%Y-%m-%d %H:%M"); humanDateFormatOnlyDate = d3.timeFormat("%Y-%m-%d"); humanDateFormatOnlyTime = d3.timeFormat("%H:%M"); d3TimeParseIsoWithMil = d3.timeParse("%Y-%m-%dT%H:%M:%S.%L"); textDialog = new GenericTextDialog(); } // CONCATENATED MODULE: ./src/command_manipulation.js /** * Parse the command-date into d3's date and assign session colors * @param {[Command]} commands */ function prepareCommands(commands){ commands.forEach(function(cmd) { cmd.startTime = d3TimeParseIsoWithMil(cmd.startTime); cmd.endTime = d3TimeParseIsoWithMil(cmd.endTime); }); _fillCommandSessionColors(commands); } /** * Can be passed to array.sort or similar functions. * @param {*} cmd1 * @param {*} cmd2 * @return {int} */ function compareStartDates(cmd1, cmd2) { return cmd1.startTime - cmd2.startTime; } /** * Can be passed to array.sort or similar functions. * @param {*} cmd1 * @param {*} cmd2 * @return {int} */ function compareEndDates(cmd1, cmd2) { return cmd1.endTime - cmd2.endTime; } /** * Assign session-colors to the given commands. * Each session gets a specific color, after n sessions occurred, colors * start from beginning again. * @param {[Command]} commands */ function _fillCommandSessionColors(commands){ const DISTINCT_COLORS = [ '#e6194b', '#3cb44b', '#ffe119', '#4363d8', '#f58231', '#911eb4', '#46f0f0', '#f032e6', '#bcf60c', '#fabebe', '#008080', '#e6beff', '#9a6324', '#fffac8', '#800000', '#aaffc3', '#808000', '#ffd8b1', '#000075', '#808080', ]; let lastColorIdx = 0; const sessionColorMap = new Map(); commands.forEach(function(cmd) { if(cmd.sessionUuid === null){ cmd.sessionColor = '#000000'; } else { let color = sessionColorMap.get(cmd.sessionUuid); if(color === undefined){ color = DISTINCT_COLORS[lastColorIdx]; sessionColorMap.set(cmd.sessionUuid, color); lastColorIdx++; if(lastColorIdx >= DISTINCT_COLORS.length){ lastColorIdx = 0; } } cmd.sessionColor = color; } }); } // CONCATENATED MODULE: ./src/html_util.js function insertAfter(newNode, referenceNode) { referenceNode.parentNode.insertBefore(newNode, referenceNode.nextSibling); } /** * Check if element is visible inside container - also partially at your wish. * @return {boolean} * @param {Element} element * @param {Element} container * @param {boolean} partial if true, return true, if not completely but partially * visible */ function isScrolledIntoView(element, container, partial) { // Get container properties const cTop = container.scrollTop; const cBottom = cTop + container.clientHeight; // Get element properties const eTop = element.offsetTop; const eBottom = eTop + element.clientHeight; // Check if in view const isTotal = (eTop >= cTop && eBottom <= cBottom); const isPartial = partial && ( (eTop < cTop && eBottom > cTop) || (eBottom > cBottom && eTop < cBottom) ); return (isTotal || isPartial); } // CONCATENATED MODULE: ./src/util.js class ErrorNotImplemented extends Error { constructor() { super('Required method not implemented'); } } function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } function getTime() { return new Date().getTime(); } function date_max(d1, d2){ return d1 > d2 ? d1 : d2; } function date_min(d1, d2){ return d1 < d2 ? d1 : d2; } function windowWidth() { return window.innerWidth || document.documentElement.clientWidth || document.body.clientWidth; } function windowHeight() { return window.innerHeight || document.documentElement.clientHeight || document.body.clientHeight; } function assert(condition, message) { if (!condition){ throw Error('Assert failed: ' + (message || '')); } } const DATE_MIN = new Date(-8640000000000000); /** * non-blocking .foreach array loop. * @param {*} array * @param {*} func */ async function timedForEach(array, func) { const maxTimePerChunk = 200; // max 200ms until next sleep function getTime() { return new Date().getTime(); } let lastStart = getTime(); for (let i=0; i < array.length; i++) { func(array[i], i, array); const now = getTime(); if(now - lastStart > maxTimePerChunk){ // enough computation time used await sleep(5); lastStart = now; } } } /** * Binary search. * @param {[]} ar sorted array, may contain duplicate elements. * If there are more than one equal elements in the array, * the returned value can be the index of any one of the equal elements. * @param {*} el element to search for * @param {function} compareFn A comparator function. The function takes two arguments: (a, b) and returns: * a negative number if a is less than b; * 0 if a is equal to b; * a positive number of a is greater than b. * @param {boolean} clipIdx see @return: * @return {int} if clipIdx is false: index of of the element in a sorted array or (-n-1) where n * is the insertion point for the new element. * If clipIdx is true: return an index within the array element bounds, independent of * wheter the element exists or not (the best matching existing index is returned). */ function binarySearch(ar, el, compareFn, clipIdx=false) { const clipIdxIfOn = (idx) => { if(! clipIdx){ return idx; } if (idx < 0) { idx = -(idx + 1); } if (idx >= ar.length) { return ar.length - 1; } return idx; }; let m = 0; let n = ar.length - 1; while (m <= n) { const k = (n + m) >> 1; const cmp = compareFn(el, ar[k]); if (cmp > 0) { m = k + 1; } else if(cmp < 0) { n = k - 1; } else { return clipIdxIfOn(k); } } return clipIdxIfOn(-m - 1); } /** * Get the directry of a unix path, e.g. the path /home/user/foo * would return /home/user. * @return {String} * @param {String} path */ function getDirFromAbsPath(path){ return path.substring(0,path.lastIndexOf("/")); } // CONCATENATED MODULE: ./src/conversions.js /** * @return {String} human readble byte-size-string * @param {int} bytes * @param {boolean} si if true: use 1000 as base (kB), else 1024 (KiB) */ function bytesToHuman(bytes, si = false) { const thresh = si ? 1000 : 1024; if (Math.abs(bytes) < thresh) { return bytes + ' B'; } const units = si ? ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'] : ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']; let u = -1; do { bytes /= thresh; ++u; } while (Math.abs(bytes) >= thresh && u < units.length - 1); return bytes.toFixed(1) + ' ' + units[u]; } // CONCATENATED MODULE: ./src/command_list.js class command_list_CommandList { constructor(commands) { this._CMDLISTPADDING = 18; this._CMDLISTBG = '#777'; const cmdListHeight = (() => { const boundClient = sessionTimeline.getSvg().node().getBoundingClientRect(); let h = windowHeight() - (boundClient.y + boundClient.height) - 30; // why minus 30? if (h < 200) { // screen too small (or too many command groups): allow for scrolling h = 300; } return h; })(); const cmdListScroll = d3.select('body').append('div') .attr('id', 'cmdListScroll') .style('height', cmdListHeight + 'px'); cmdListScroll.selectAll('.collapsibleCmd') .data(commands) .enter() .append('button') .attr('class', 'collapsibleCmd') .attr('id', (cmd) => { return 'cmdListEntry' + cmd.id; }) .html((cmd) => { // only display year,month,day of endTime if different from start const actualEndFormat = (cmd.startTime.getFullYear() == cmd.endTime.getFullYear() && cmd.startTime.getMonth() == cmd.endTime.getMonth() && cmd.startTime.getDay() == cmd.endTime.getDay() ) ? humanDateFormatOnlyTime : humanDateFormat; return humanDateFormat(cmd.startTime) + ' - ' + actualEndFormat(cmd.endTime) + ': ' + cmd.command; }) .style('padding', this._CMDLISTPADDING + 'px') .style('background', (cmd) => { return this._computeCmdBackground(cmd); }) .on("click", (cmd, idx) => { if (document.readyState !== "complete"){ // silently ignore clicks, until everything loaded... return; } this._handleClickOnCmd(cmd, idx); }); } /** * @param {*} cmd command-object to scroll to */ scrollToCmd(cmd) { const cmdElement = this._selectCmdEntry(cmd); const scroll = document.getElementById('cmdListScroll'); scroll.scrollTop = cmdElement.node().offsetTop; cmdElement .transition() .duration(1300) // miliseconds .style("background", "red") .on("end", () => { cmdElement.style('background', (cmd) => { return this._computeCmdBackground(cmd); }); }); } _selectCmdEntry(cmd){ return d3.select(`#cmdListEntry${cmd.id}`); } _computeCmdBackground(cmd){ return `linear-gradient(to right, ${cmd.sessionColor} 0px, ${cmd.sessionColor} ${this._CMDLISTPADDING - 1}px, ${this._CMDLISTBG} ${this._CMDLISTPADDING - 1}px, ${this._CMDLISTBG} 100%)`; } _handleClickOnCmd(cmd, idx){ let contentDiv = d3.select(`#cmdcontent${cmd.id}`); if (! contentDiv.empty()) { contentDiv.remove(); return; } contentDiv = d3.select('body').append('div') .attr('id', `cmdcontent${cmd.id}`) .attr('class', 'collapsibleCmdContent') .html(`Working directory: ${cmd.workingDir}
` + `Command exit status: ${cmd.returnValue}
` + `Session uuid: ${cmd.sessionUuid}
` + `Command id: ${cmd.id}
` + `Hostname: ${cmd.hostname}
`); const alternatingColor = '#D9D9D9'; if (cmd.fileWriteEvents_length > 0) { contentDiv.append('span') .html(cmd.fileWriteEvents_length + ' written files') .style('color', 'red') .style('display', 'block'); contentDiv.selectAll('.nonexistentClass') .data(cmd.fileWriteEvents) .enter() .append('span') .style('display', 'block') .style('background-color', (e, idx) => { return (idx % 2 === 0) ? 'transparent' : alternatingColor; }) .text((e) => { return `${e.path} (${bytesToHuman(e.size)}), Hash: ${e.hash}`; }); if (cmd.fileWriteEvents.length !== cmd.fileWriteEvents_length) { contentDiv.append('span').html( `... and ` + `${cmd.fileWriteEvents_length - cmd.fileWriteEvents.length}` + ` more (see shournal's query help to increase limits)
`); } } if (cmd.fileReadEvents_length > 0) { if(cmd.fileWriteEvents_length > 0){ contentDiv.append('span').html('
'); } contentDiv.append('span') .html(cmd.fileReadEvents_length + ' read files') .style('color', 'red') .style('display', 'block'); } contentDiv.selectAll('.nonexistentClass') .data(cmd.fileReadEvents) .enter() .append('span') .style('background-color', (e, idx) => { return (idx % 2 === 0) ? 'transparent' : alternatingColor; }) .style('color', (readFile) => { return (readFile.isStoredToDisk) ? 'blue' : 'black'; }) .style('cursor', (readFile) => { return (readFile.isStoredToDisk) ? 'pointer' : 'default'; }) .style('display', 'block') // only one read file per line .text((e) => { return `${e.path} (${bytesToHuman(e.size)}), Hash: ${e.hash}`; }) .on("click", (readFile) => { if (readFile.isStoredToDisk) { const mtimeHuman = humanDateFormat(d3.isoParse(readFile.mtime)); const title = `Read file ${readFile.path}
` + `mtime: ${mtimeHuman}
` + `size: ${bytesToHuman(readFile.size)}
` + `hash: ${readFile.hash}
`; const readFileContent = atob(readFileContentMap.get(readFile.id)); textDialog.show(title, readFileContent); } }); if (cmd.fileReadEvents.length !== cmd.fileReadEvents_length) { contentDiv.append('span').html( `... and ` + `${cmd.fileReadEvents_length - cmd.fileReadEvents.length}` + ` more (see shournal's query help to increase limits)
` ); } const cmdElement = this._selectCmdEntry(cmd); insertAfter(contentDiv.node(), cmdElement.node()); const cmdListScroll = document.getElementById('cmdListScroll'); if(! isScrolledIntoView(contentDiv.node(), cmdListScroll, true)){ // scroll down one element, so at least the beginning of content is visible: cmdListScroll.scrollTop += cmdElement.node().clientHeight; } } } // CONCATENATED MODULE: ./src/map_extended.js class MapExtended extends Map { /** * Like get() but insert and return a default, if the key * does not exist * @return {*} * @param {*} key * @param {Function} defaultFactory A parameterless function whose return value * is used as default. */ getDefault(key, defaultFactory) { if(defaultFactory === undefined){ throw Error('defaultValue must not be undefined'); } let val = this.get(key); if(val === undefined){ val = defaultFactory(); this.set(key, val); } return val; } } // CONCATENATED MODULE: ./node_modules/tinyqueue/index.js class TinyQueue { constructor(data = [], compare = defaultCompare) { this.data = data; this.length = this.data.length; this.compare = compare; if (this.length > 0) { for (let i = (this.length >> 1) - 1; i >= 0; i--) this._down(i); } } push(item) { this.data.push(item); this.length++; this._up(this.length - 1); } pop() { if (this.length === 0) return undefined; const top = this.data[0]; const bottom = this.data.pop(); this.length--; if (this.length > 0) { this.data[0] = bottom; this._down(0); } return top; } peek() { return this.data[0]; } _up(pos) { const {data, compare} = this; const item = data[pos]; while (pos > 0) { const parent = (pos - 1) >> 1; const current = data[parent]; if (compare(item, current) >= 0) break; data[pos] = current; pos = parent; } data[pos] = item; } _down(pos) { const {data, compare} = this; const halfLength = this.length >> 1; const item = data[pos]; while (pos < halfLength) { let left = (pos << 1) + 1; let best = data[left]; const right = left + 1; if (right < this.length && compare(data[right], best) < 0) { left = right; best = data[right]; } if (compare(best, item) >= 0) break; data[pos] = best; pos = left; } data[pos] = item; } } function defaultCompare(a, b) { return a < b ? -1 : a > b ? 1 : 0; } // CONCATENATED MODULE: ./src/timeline_group_find.js /** * Find "groups" in an ordered timeline, so that parallel * events get different (low) groups (integers starting from zero). * Events are defined by start- and end-date. The container, for * whose elements findNextFreeGroup may be called subsequentially, * must be ordered by start-date. */ class timeline_group_find_TimelineGroupFind { constructor(){ this._lastEndDates = []; this._freeGroups = new TinyQueue(); } /** * @return {int} lowest free group, starting from 0. * @param {Date} startDate start date of the next time element * @param {Date} endDate end date of the next time element */ findNextFreeGroup(startDate, endDate){ for (let i = this._lastEndDates.length - 1; i >= 0; i--) { if (startDate > this._lastEndDates[i].endTime) { this._freeGroups.push(this._lastEndDates[i].group); this._lastEndDates.splice(i, 1); } } // if we have free groups (from previous runs) use the lowest free group, // else add a new one const group = (this._freeGroups.length > 0) ? this._freeGroups.pop() : this._lastEndDates.length; this._lastEndDates.push(new _LastEndDateGroup(group, endDate)); return group; } } class _LastEndDateGroup { constructor(group, endTime){ this.group = group; this.endTime = endTime; } } // EXTERNAL MODULE: ./node_modules/async-mutex/lib/index.js var lib = __webpack_require__(0); // CONCATENATED MODULE: ./src/annotation_line_render.js /** * Render Groups of annotations on a per-line-basis. Clip annotation texts * and omit annotations as needed to fit into available space */ class annotation_line_render_AnnotationLineRender { constructor(plot) { this._annotationGroups = []; this._plot = plot; // get the width in pixel of a character this._annotationCharWidth = d3.select("#annotation_text_char").node() .getBoundingClientRect().width; // do not render an annotation which does not fit into the space. this._annotationMinWidth = this._annotationCharWidth * 2; // clip annotation-texts after that many characters this._annotationMaxNumChars = 15; this._updateMutex = new lib["Mutex"](); this._lastUpdateDummy = null; } /** * * @param {Array} group: ordered set of annotations which will be rendered * within the same line. Base class is the same as d3 annotation, however, the following * *additional* fields must be set: startX, endX, fulltext. The annotation position * (x,y) has to be set already, based on the x-values it is decided, how much of * an annotation is drawn. */ addAnnotationGroup(group) { this._annotationGroups.push(group); } async update(xScale) { this._lastUpdateDummy = {}; const currentUpdateDummy = this._lastUpdateDummy; const release = await this._updateMutex.acquire(); try { // remove and add again seems to be faster than updating this._plot.selectAll('.annotation').remove(); this._plot.selectAll('.annotationVertLine').remove(); this._plot.selectAll('.annotationHorizLine').remove(); const annotations = await this._preRenderAnnotations(xScale, currentUpdateDummy); if (annotations !== null) { this._appendAnnotations(annotations); } } finally { release(); } } setOnNoteClick(func){ this._onNoteClick = func; } // ***************** PRIVATE ******************** _compareStartX(prev, current) { return prev.startX - current.startX; } _compareEndX(prev, current) { return prev.endX - current.endX; } async _preRenderAnnotations(xScale, currentUpdateDummy ) { const annotations = []; // uniform interface for binary search, where the entrance indeces are found const dummyAnnotation = { startX: xScale.domain()[0], endX: xScale.domain()[1], }; const plotWidth = this._plot.node().getBBox().width; for(const annotationLine of this._annotationGroups) { if (annotationLine.length == 0) { continue; } // Do not render annotations outside the current view // -> find start and stop indeces in the group: // Note: one cannot simply choose 0 and length -1 after zooming // out, because panning also has to be respected. const startIdx = binarySearch(annotationLine, dummyAnnotation, this._compareStartX, true); const endIdx = binarySearch(annotationLine, dummyAnnotation, this._compareEndX, true); let displayAnnotation = annotationLine[startIdx]; displayAnnotation.x = this._calcAnnotationCenter(displayAnnotation, xScale); for (let idx = startIdx + 1; idx <= endIdx; idx++) { // this.update is run async: check if it was called in between. If that's the // case we can abort, because or xScale is outdated. if (currentUpdateDummy !== this._lastUpdateDummy) { return null; } if(idx % 30 === 0){ // avoid freezing the DOM... await sleep(5); } const annotation = annotationLine[idx]; annotation.x = this._calcAnnotationCenter(annotation, xScale); const textspace = annotation.x - displayAnnotation.x - (this._annotationCharWidth * 2); // subtract more chars to leave space to next annotation const annotationTxt = this._generateAnnotationTxt(textspace, displayAnnotation.fulltext); if (annotationTxt == null) { // do not render this annotation continue; } // always update text, we might have zoomed before! displayAnnotation.note.label = annotationTxt; annotations.push(displayAnnotation); displayAnnotation = annotation; } // still need to push the final annotation, if it fits into our plot const textspace = plotWidth - displayAnnotation.x; const annotationTxt = this._generateAnnotationTxt(textspace, displayAnnotation.fulltext); if (annotationTxt != null) { displayAnnotation.note.label = annotationTxt; annotations.push(displayAnnotation); } } return annotations; } _calcAnnotationCenter(annotation, xScale) { return (xScale(annotation.startX) + xScale(annotation.endX)) / 2.0; } /** * @param {*} textspace Available width in pixel * @param {*} txt The full text * @return {*} null, if textspace was too small, else the full or clipped text */ _generateAnnotationTxt(textspace, txt) { if (textspace < this._annotationMinWidth) { return null; } // Render only so many chars that fit into the space, but not more than // _annotationMaxNumChars; const maxCountOfRenderChars = Math.min(Math.ceil(textspace / this._annotationCharWidth) , this._annotationMaxNumChars); if (txt.length <= maxCountOfRenderChars ) { return txt; } return txt.substring(0, maxCountOfRenderChars - 1) + '.'; } /** * Append all annotations to the plot and setup mouse event handlers * @param {[annotation]} annotations */ _appendAnnotations(annotations) { const enterSelection = this._plot.selectAll(".annotation") .data(annotations) .enter(); enterSelection .append("text") .attr('class', 'annotation unselectable' ) .attr('x', (a) => { return a.x; }) .attr('y', (a) => { return a.ny; }) .text((a) => { return a.note.label; }) .attr('title', (a) => { return a.fulltext; }) .style('cursor', 'pointer') .on("click", (a) => { if (this._onNoteClick !== undefined) { // d3.event.pageX, d3.event.pageY this._onNoteClick(a.data); } }); // dynamically inserted elements -> rerun tooltip $('.annotation').tooltip({ delay: { show: 100, hide: 0 }, }); const horzLineYOffset = 2; const lineColor = 'steelblue'; enterSelection .insert("line") .attr('class', 'annotationVertLine') .attr('x1', (a) => { return a.x; }) .attr('y1', (a) => { return a.ny + horzLineYOffset; }) .attr('x2', (a) => { return a.x; }) .attr('y2', (a) => { return a.y; }) .attr("stroke-width", 0.5) .attr("stroke", lineColor); enterSelection .insert("line") .attr('class', 'annotationHorizLine') .attr('x1', (a) => { return a.x; }) .attr('y1', (a) => { return a.ny + horzLineYOffset; }) .attr('x2', (a) => { return a.x+ (a.note.label.length * this._annotationCharWidth); }) .attr('y2', (a) => { return a.ny + horzLineYOffset; }) .attr("stroke-width", 0.5) .attr("stroke", lineColor); } } // CONCATENATED MODULE: ./src/zoom_buttons.js class ZoomButtons { /** * @param {d3-element} containerDiv The plot/svg is excepted to be in that div. * Its 'position' should be 'relative', see https://stackoverflow.com/a/10487329 * so we can place the buttons in an absolute manner. * @param {d3-element} zoomArea the element used for zooming * @param {d3.zoom} d3Zoom */ constructor(containerDiv, zoomArea, d3Zoom) { const btnGroup = containerDiv.append('div'); const zoomInBtn = this._appendZoomButton(btnGroup, '+') .on("click", () => { d3Zoom.scaleBy(zoomArea.transition().duration(10), 1.2); }); const zoomInBtnWidth = parseInt(zoomInBtn.style('width'), 10); const zoomOutBtn = this._appendZoomButton(btnGroup, '-') .on("click", () => { d3Zoom.scaleBy(zoomArea.transition().duration(10), 0.8); }); const zoomOutBtnWidth = parseInt(zoomOutBtn.style('width'), 10); const zoomResetBtn = this._appendZoomButton(btnGroup, '[ ]') .on("click", () => { d3Zoom.transform(zoomArea, d3.zoomIdentity.translate(0, 0).scale(1.0)); }); const zoomResetBtnWidth = parseInt(zoomResetBtn.style('width'), 10); const zoomButtonsWidth = zoomInBtnWidth + zoomOutBtnWidth + zoomResetBtnWidth; btnGroup.style('position', 'absolute') // see https://stackoverflow.com/a/10487329 -> // parent position should be relative .style('top', 0 + 'px') .style('right', ( zoomButtonsWidth) + 'px'); } _appendZoomButton(container, text) { return container.append('button') .attr('class', 'zoomButton') .html(text); } } // CONCATENATED MODULE: ./src/session_timeline.js class session_timeline_SessionTimeline { constructor(commands, cmdFinalEndDate) { this.cmdFinalEndDate = cmdFinalEndDate; this._margin = { top: 20, right: 20, bottom: 24, left: 24, }; // get the width in pixel of a character this.annotationCharWidth = d3.select("#annotation_text_char").node() .getBoundingClientRect().width; this.annotationCharHeight = d3.select("#annotation_text_char").node() .getBoundingClientRect().height; // height of a session with no forks (parallel commands ) this.sessionBaseHeight = this.annotationCharHeight / 1.5; this.sessionPadding = this.annotationCharHeight / 5; // choose less than two, so two parallel commands // are already wider than a lonely command. this.sessionMinHeight = this.sessionBaseHeight / 1.5; // An annotation shall only be displayed, if its minimum width in pixel // is at least 5 character. Warning: do not set < 1 -> text rendering issues for annotations this.annotationMinWidth = this.annotationCharWidth * 5; // distance to the belonging command rect this.annotationDistance = this.annotationCharHeight / 3.0; this.commandRects = []; // minimum width of a cmd-rect. Let it be at least 1, otherwise very short commands // are barely visible (get another color...) this.CMD_MIN_WIDTH = 4; this.svgWidth = windowWidth() - this._margin.left - this._margin.right - 30; const plotContainer = d3.select('body').append('div') .style('position', 'relative'); // see https://stackoverflow.com/a/10487329 this.svg = plotContainer.append('svg'); this._annotationRender = new annotation_line_render_AnnotationLineRender(this.svg); const groupedSessions = this._generateCommandsPerSession(commands); this.svgHeight = Math.max(100, this._prerenderSessions(groupedSessions)); this.xScale = d3.scaleTime() .range([0, this.svgWidth]); this._yScale = d3.scaleLinear() .range([this.svgHeight, 0]); this._yScale.domain([0, this.svgHeight]); this.axisBottom = d3.axisBottom(this.xScale); this.svg.attr('width', this._margin.left + this.svgWidth + this._margin.right) .attr('height', this._margin.top + this.svgHeight + this._margin.bottom) .append('g') .attr('transform', 'translate(' + this._margin.left + ',' + this._margin.top + ')') .style('z-index', -1); const listenerRect = this.svg .append('rect') .attr('class', 'listener-rect') .attr('x', 0) .attr('y', -this._margin.top) .attr('width', this._margin.left + this.svgWidth + this._margin.right) .attr('height', this._margin.top + this.svgHeight + this._margin.bottom) .style('opacity', 0); this.xScale.domain([ // the commands are sorted by starttime... commands[0].startTime, this.cmdFinalEndDate, ]).nice(); // draw axes this.xAxisDraw = this.svg.insert('g', ':first-child') .attr('class', 'x axis') .attr('transform', 'translate(0,' + this.svgHeight + ')') .call(this.axisBottom // .ticks(d3.timeWeek, 2) // .tickFormat(d3.timeFormat('%b %d')) ); const _drawSession = (session, idx, lineIdx) => { // draw rectangles const className = 'sessionTimeSeries' + session.getSessionGroup() + idx; this.commandRects.push(this.svg.selectAll('.' + className) .data(session.getCmdsWithMeta()) .enter() .append('rect') .attr('class', className) .attr('x', (cmdWithMeta) => { return this._calcRectXPosition(cmdWithMeta.cmd, this.xScale); }) .attr('y', (cmdWithMeta) => { // rects are drawn from top to bottom, so add the height: return this._yScale(cmdWithMeta.getY() + cmdWithMeta.getHeight()); }) .attr('width', (cmdWithMeta) => { return this._calcRectWidth(cmdWithMeta.cmd, this.xScale); }) .attr('height', (cmdWithMeta) => { return cmdWithMeta.getHeight(); }) .attr('fill', (cmdWithMeta) => { // TODO: rather determine the session color in this class // on a per line-basis, so the same color appears as seldom // as possible in a given line (?). // But what about the colors in the cmd-list?... return cmdWithMeta.cmd.sessionColor; } ) .style('cursor', 'pointer') .attr('title', (cmdWithMeta) => { return cmdWithMeta.cmd.command; }) .on("click", (cmdWithMeta) => { commandList.scrollToCmd(cmdWithMeta.cmd); }) ); $('.' + className).tooltip({ delay: { show: 50, hide: 0 }, }); }; groupedSessions.forEach((sessionLine, lineIdx) => { sessionLine.forEach((session, sessionIdx) => { _drawSession(session, sessionIdx, lineIdx); }); }); this._preRenderAnnotations(groupedSessions); this._annotationRender.setOnNoteClick((cmdWithMeta) => { commandList.scrollToCmd(cmdWithMeta.cmd); }); this._annotationRender.update(this.xScale); const minTimeMilli = 20000; // do not allow zooming beyond displaying 20 seconds const maxTimeMilli = 6.3072e+11; // approx 20 years const currentWidthMilli = cmdFinalEndDate - commands[0].startTime; const minScaleFactor = currentWidthMilli / maxTimeMilli; const maxScaleFactor = currentWidthMilli / minTimeMilli; const zoom = d3.zoom() // .scaleExtent([0.001, 5000]) .scaleExtent([minScaleFactor, maxScaleFactor]) .on("zoom", () => { this._handleZoom(d3.event.transform); }); this._zoomButtons = new ZoomButtons(plotContainer, listenerRect, zoom); listenerRect.call(zoom); } getSvg(){ return this.svg; } _generateCommandsPerSession(commands) { const assignParallelCmdCounts = (commandsPerSession) => { // find out the number of parallel commands in each session and store it // in the meta-info of each cmd. The groups are already assigned, one command // is parallel to another, if there exists at least one command // between two zero-group-commands. Note that the groups of // those in-between-commands may rise and fall arbitrarily often, // so keep track of the max. commandsPerSession.forEach((session) => { // index in the sessions cmd-array, where the last group 0 was seen let lastZeroGroupIdx = 0; let lastHighestGroup = 0; // yes, <= to simplify handling the final command for (let i = 1; i <= session.getCmdsWithMeta().length; i++) { if (i >= session.getCmdsWithMeta().length || session.getCmdsWithMeta()[i].getGroup() === 0) { // a new group has started or we are at end. Assign the found number of parallel // commands to all affected commands: const countOfParallelCmds = lastHighestGroup + 1; // zero based.. for (let j = lastZeroGroupIdx; j < i; j++) { session.getCmdsWithMeta()[j].setCountOfParallelGroups(countOfParallelCmds); } // also keep track of the max number of parallel commands in this session // for later use session.setMaxCountOfParallelCommands( Math.max(session.getMaxCountOfParallelCommands(), countOfParallelCmds) ); lastZeroGroupIdx = i; lastHighestGroup = 0; } else { // keep track of the highest group lastHighestGroup = Math.max(lastHighestGroup, session.getCmdsWithMeta()[i].getGroup()); } } }); }; const commandsPerSession = new MapExtended(); commands.forEach( (cmd) => { // note: Map()' iteration order is the insert order, which is // desired here -> since the command-array is ordered by startDateTime, // the generated session map is also ordered by startDateTime const session = commandsPerSession.getDefault(cmd.sessionUuid, () => { return new session_timeline_Session(); }); session.addCmd(cmd); }); assignParallelCmdCounts(commandsPerSession); // assign a group to each session const sessionGrpFind = new timeline_group_find_TimelineGroupFind(); let maxGroup = 0; commandsPerSession.forEach((session) => { const group = sessionGrpFind.findNextFreeGroup(session.getSessionStartDate(), session.getSessionEndDate()); session.setSessionGroup(group); maxGroup = Math.max(maxGroup, group); }); // generate an array of an array of sessions, so all sessions which have // the same group are in one array (in correct order). // That way one 'line' of sessions can be // drawn easily. const groupedSessions = new Array(maxGroup + 1); for (let i = 0; i < groupedSessions.length; i++) { groupedSessions[i] = []; } commandsPerSession.forEach( (session) => { groupedSessions[session.getSessionGroup()].push(session); }); return groupedSessions; } /** * @return {int} max y offset of the plot * @param {*} groupedSessions */ _prerenderSessions(groupedSessions){ const ANNOTATION_AND_PADDING = this.annotationDistance + this.annotationCharHeight * 1.5; // * 1.5 -> give some more space const _prerenderCmd = (cmdWithMeta, currentOffset, sessionHeight) => { if(cmdWithMeta.getCountOfParallelGroups() === 1){ // non-parallel commands are aligned to session center: cmdWithMeta.setHeight(this.sessionBaseHeight); const y = currentOffset + sessionHeight/2 - this.sessionBaseHeight/2; cmdWithMeta.setY(y); return; } // parallel commands expand in equal parts over the whole sessionHeight // (separated by padding) let cmdHeight = sessionHeight / cmdWithMeta.getCountOfParallelGroups(); if(cmdHeight < this.sessionMinHeight){ cmdHeight = this.sessionMinHeight; } else { cmdHeight -= this.sessionPadding; } cmdWithMeta.setHeight(cmdHeight); const y = currentOffset + (cmdHeight + this.sessionPadding) * cmdWithMeta.getGroup(); cmdWithMeta.setY(y); }; let currentOffset = 0; groupedSessions.forEach((sessionLine, lineIdx) => { // find the max. number of parallel commands in all sessions of the current line: const maxNumberOfParallelCmds = sessionLine.reduce((prev, curr) => { return prev.getMaxCountOfParallelCommands() > curr.getMaxCountOfParallelCommands() ? prev : curr; }).getMaxCountOfParallelCommands(); const sessionHeight = maxNumberOfParallelCmds === 1 ? this.sessionBaseHeight : (this.sessionMinHeight + this.sessionPadding) * maxNumberOfParallelCmds; sessionLine.forEach((session) => { session.getCmdsWithMeta().forEach((cmdWithMeta) => { _prerenderCmd(cmdWithMeta, currentOffset, sessionHeight); }); session.setHeight(sessionHeight); session.setY(currentOffset); }); currentOffset += sessionHeight + ANNOTATION_AND_PADDING; }); return currentOffset; } _preRenderAnnotations(groupedSessions){ groupedSessions.forEach((sessionLine) => { const annotationGroup = []; sessionLine.forEach((session) => { session.getCmdsWithMeta().forEach((cmdWithMeta) => { // only create annotations for the topmost commandgroup // (in case of parallel commands) if(cmdWithMeta.getCountOfParallelGroups() === cmdWithMeta.getGroup() + 1){ annotationGroup.push(this._createAnnotation(cmdWithMeta, session.getY() + session.getHeight() + this.annotationDistance )); } }); }); this._annotationRender.addAnnotationGroup(annotationGroup); }); } _createAnnotation(cmdWithMeta, y){ return { data: cmdWithMeta, note: { align: "left", wrap: 'nowrap', // title: "Annotation title" }, dx: 0, ny: this.svgHeight - y, y: this.svgHeight - (cmdWithMeta.getY() + cmdWithMeta.getHeight()), startX: cmdWithMeta.cmd.startTime, endX: cmdWithMeta.cmd.endTime, fulltext: cmdWithMeta.cmd.command, }; } _calcRectXPosition(cmd, xScale) { let startX = xScale(cmd.startTime); const w = xScale(cmd.endTime) - startX; if (w < this.CMD_MIN_WIDTH) { // since a cmd has to have at least that width, but shall be // centered anyway: const center = startX + w / 2.0; startX = center - this.CMD_MIN_WIDTH / 2.0; } return startX; } _calcRectWidth(cmd, xScale) { const w = xScale(cmd.endTime) - xScale(cmd.startTime); if (w < this.CMD_MIN_WIDTH) { return this.CMD_MIN_WIDTH; } return w; } _handleZoom(transform) { const xScaleNew = transform.rescaleX(this.xScale); this.axisBottom.scale(xScaleNew); this.xAxisDraw.call( this.axisBottom // .ticks(d3.timeWeek, 2) // .tickFormat(d3.timeFormat('%b %d')) ); // maybe_todo: execute in parallel... this.commandRects.forEach((rectGroup) => { rectGroup.attr('x', (cmdWithMeta) => { const pos = this._calcRectXPosition(cmdWithMeta.cmd, xScaleNew); // note: pos may be less than zero which is ok, because // otherwise wide rects may disappear too soon. return pos; }) .attr('width', (cmdWithMeta) => { return this._calcRectWidth(cmdWithMeta.cmd, xScaleNew); }); }); this._annotationRender.update(xScaleNew); } } class _CommandWithMeta{ /** * * @param {Command} cmd * @param {int} group the group assigned within a session */ constructor(cmd, group){ this.cmd = cmd; this._group = group; this._countOfParallelGroups = -1; this._height = 1000; this._y = 0; this._annotation = null; } getGroup(){ return this._group; } setCountOfParallelGroups(val){ this._countOfParallelGroups = val; } getCountOfParallelGroups(){ return this._countOfParallelGroups; } setHeight(val){ this._height = val; } getHeight(){ return this._height; } setY(val){ this._y = val; } getY(){ return this._y; } setAnnotation(val){ this._annotation = val; } getAnnotation(){ return this._annotation; } } class session_timeline_Session { constructor() { this._cmdsWithMeta = []; this._finalCmdEndDate = DATE_MIN; this._groupFind = new timeline_group_find_TimelineGroupFind(); this._firstCmdStartDate = null; this._sessionGroup = null; this._maxCountOfParallelCmds = null; this._height = null; } /** * The passed commands *must* be sorted (asc) by startTime during * subsequent calls of this method. * @param {Command} cmd */ addCmd(cmd) { if(this._firstCmdStartDate === null){ // commands are sorted by startTime and we are called the first time. this._firstCmdStartDate = cmd.startTime; } // commands are sorted by startTime but the first executed cmd may well finish // last, so incrementally find the final endDate. this._finalCmdEndDate = date_max(cmd.endTime, this._finalCmdEndDate); const group = this._groupFind.findNextFreeGroup(cmd.startTime, cmd.endTime); this._cmdsWithMeta.push(new _CommandWithMeta(cmd, group)); } setMaxCountOfParallelCommands(val){ this._maxCountOfParallelCmds = val; } getMaxCountOfParallelCommands(){ return this._maxCountOfParallelCmds; } getSessionStartDate(){ return this._firstCmdStartDate; } getSessionEndDate(){ return this._finalCmdEndDate; } setSessionGroup(val){ this._sessionGroup = val; } getSessionGroup(){ return this._sessionGroup; } getCmdsWithMeta(){ return this._cmdsWithMeta; } setHeight(val){ this._height = val; } getHeight(){ return this._height; } setY(val){ this._y = val; } getY(){ return this._y; } } // CONCATENATED MODULE: ./src/d3js_util.js /** * Wrap long axis labels to mutliple lines by maximum width, splitting words by * the given *delimeter-keeping* splitStr and auto-truncating long words. * Leading and trailing whitespaces of each line are trimmed. * @param {[String]} tickTexts * @param {int} width The max. width in pixels for each label * @param {RegExp} splitStr A regular expression for the split-string, * which keeps the delimeter, e.g. /(?=\s)/. */ function wrapTextLabels(tickTexts, width, splitStr=/(?=\s)/) { tickTexts.each(function() { const text = d3.select(this); const words = text.text().split(splitStr); let line = []; let lineNumber = 0; const lineHeight = 1.1; // ems const y = text.attr("y"); const dy = parseFloat(text.attr("dy")); let tspan = text.text(null).append("tspan").attr("x", 0).attr("y", y) .attr("dy", dy + "em"); // only increment i, if the word can be for sure drawn to current line for(let i=0; i < words.length; ) { line.push(words[i]); tspan.text(line.join('')); if (tspan.node().getComputedTextLength() <= width) { ++i; continue; } if (line.length === 1) { // this single word is too long to fit into a line -> clip it _truncateAndSetLabelTxt(line[0].trim(), tspan, width); ++i; } else { // this word does not fit any more -> put it to next line // and render all others line.pop(); tspan.text(line.join('').trim()); // NO ++i -> the current word must be rendered in next line } tspan = text.append("tspan").attr("x", 0).attr("y", y) .attr("dy", `${++lineNumber * lineHeight + dy}em`).text(null); line = []; } }); } function _truncateAndSetLabelTxt(labelTxt, tspan, width) { do { labelTxt = labelTxt.slice(0, -3); tspan.text(labelTxt); } while (tspan.node().getComputedTextLength() > width); labelTxt = labelTxt.slice(0, -2); labelTxt += '..'; tspan.text(labelTxt); } // CONCATENATED MODULE: ./src/plot_simple_bar.js /** * Base class for several bar plots */ class plot_simple_bar_PlotSimpleBar { constructor() { this._margin = { top: 20, right: 20, bottom: 60, left: 40 }; this._width = 500 - this._margin.left - this._margin.right; this._height = 300 - this._margin.top - this._margin.bottom; this._maxBarWidth = 30; } generatePlot(data, siblingElement) { const plotContainer = siblingElement.append('div') .style('position', 'relative') .style('padding', '12px') .style('display', 'inherit'); this._svg = plotContainer.append("svg") .attr("width", this._width + this._margin.left + this._margin.right) .attr("height", this._height + this._margin.top + this._margin.bottom) .append("g") .attr("transform", "translate(" + this._margin.left + "," + this._margin.top + ")"); // chart title const chartTitle = this._svg.append("text") .attr("x", (this._width / 2.0)) .attr("y", -3) .attr("text-anchor", "middle") .style("font-size", "16px") .style("text-decoration", "underline") .text(this._chartTitle()); this._xScaleBand = d3.scaleBand() .range([0, this._width]) .padding(0.1); this._yScaleBand = d3.scaleLinear() // leave some space for the char title: .range([this._height, chartTitle.node().getBoundingClientRect().height * 1.2]); // In case of duplicate x-axis label values they are overridden, which should // never be desired. Instead build a range and access the respective data-array-element // by index. this._xScaleBand.domain(d3.range(data.length)); this._yScaleBand.domain(this._yScaleBandDomain()); const actualBandWidth = (this._xScaleBand.bandwidth() > this._maxBarWidth) ? this._maxBarWidth : this._xScaleBand.bandwidth(); // append the rectangles for the bar chart const dataEnterSelection = this._svg.selectAll(".bar").data(data).enter(); const bars = dataEnterSelection.append("rect") .style('fill',(d, i) => { return this._barColor(d); }) .attr("x", (d, i) => { let x = this._xScaleBand(i); const center = x + this._xScaleBand.bandwidth()/2.0; x = center - actualBandWidth/2.0; return x; }) .attr("width", actualBandWidth) .attr("y", (d) => { return this._yScaleBand(this._yValue(d)); }) .attr("height", (d) => { return this._height - this._yScaleBand(this._yValue(d)); }) .attr('data-toggle', 'tooltip') .attr('title', (d) => { return this._barTooltipTxt(d); }); this._modifyBars(bars); // add the x Axis this._svg.append("g") .attr("transform", "translate(0," + this._height + ")") .call(d3.axisBottom(this._xScaleBand).tickFormat((d,i)=> this._xValue(data[i]))) .selectAll("text") .call((tickTexts) => { const thisPlot = this; tickTexts.each(function (plainTxt, idx) { const text = d3.select(this); text.attr("title", function () { return thisPlot._xAxisTooltipTxt.call(thisPlot, data[idx]); }).attr('data-toggle', 'tooltip') .attr('data-placement', 'left'); thisPlot._modifyTickText(text, data[idx]); }); wrapTextLabels(tickTexts, this._xScaleBand.bandwidth(), this._xTxtLabelSplitStr()); }); // add the y Axis const yAxisTicks = this._yScaleBand.ticks() .filter((tick) => { return this._yAxisTicksFilter(tick); }); this._yaxis = d3.axisLeft(this._yScaleBand); const yTickFormat = this._yAxisTickFormat(); if(yTickFormat !== undefined){ this._yaxis.tickValues(yAxisTicks).tickFormat( yTickFormat ); } this._svg.append("g").call(this._yaxis); } // MUST override methods _chartTitle(){ throw new ErrorNotImplemented(); } _yScaleBandDomain(){ throw new ErrorNotImplemented(); } // Is called for each x-value _xValue(d){ throw new ErrorNotImplemented(); } // Is called for each y-value _yValue(d){ throw new ErrorNotImplemented(); } // MAY override methods _yAxisTicksFilter(tick){ return true; } _yAxisTickFormat() { return undefined; } _modifyTickText(tickTxt, data) {} _xTxtLabelSplitStr() { return /(?=\s)/; } _barTooltipTxt(dataElement){ return this._xValue(dataElement); } _xAxisTooltipTxt(dataElement){ return this._xValue(dataElement); } _barColor(dataElement){ return 'steelblue'; } // apply further modifications to the bars _modifyBars(bars){} } // CONCATENATED MODULE: ./src/plot_most_written_files.js /** * A bar plot displaying the commands which * modified the most files. */ class plot_most_written_files_PlotMostWrittenFiles extends plot_simple_bar_PlotSimpleBar { generatePlot(commands, siblingElement){ this._filteredCmds = []; mostFileMods.forEach((e) => { this._filteredCmds.push(commands[e.idx]); }); this._maxCountOfWfileEvents = this._filteredCmds[0].fileWriteEvents_length; // Be consistent with timeline and sort by date: this._filteredCmds.sort(compareStartDates); super.generatePlot(this._filteredCmds, siblingElement); } /** * @override */ _chartTitle(){ return 'Commands with most file-modifications'; } /** * @override */ _yScaleBandDomain(){ return [0, this._maxCountOfWfileEvents]; } /** * @override */ _xValue(cmd) { return humanDateFormatOnlyDate(cmd.startTime) + ": " + cmd.command; } /** * @override */ _yValue(cmd) { return cmd.fileWriteEvents_length; } /** * @override */ _yAxisTicksFilter(tick){ return Number.isInteger(tick); } /** * @override */ _yAxisTickFormat() { return d3.format('d'); } _barColor(cmd){ return cmd.sessionColor; } _modifyBars(bars){ bars .style('cursor', 'pointer') .on("click", (cmd) => { commandList.scrollToCmd(cmd); }); } _modifyTickText(tickTxt, cmd) { tickTxt .style('cursor', 'pointer') .on("click", () => { commandList.scrollToCmd(cmd); }); } } // CONCATENATED MODULE: ./src/plot_cmdcount_per_cwd.js /** * A bar plot displaying the working directories * where the most commands were executed. */ class plot_cmdcount_per_cwd_PlotCmdCountPerCwd extends plot_simple_bar_PlotSimpleBar { generatePlot(commands, siblingElement){ super.generatePlot(cwdCmdCounts, siblingElement); } /** * @override */ _chartTitle(){ return 'Working directories with most commands'; } /** * @override */ _yScaleBandDomain(){ return [0, cwdCmdCounts[0].countOfCommands]; } /** * @override */ _xValue(cwdCmdCount) { return cwdCmdCount.workingDir; } /** * @override */ _yValue(cwdCmdCount) { return cwdCmdCount.countOfCommands; } /** * @override */ _yAxisTicksFilter(tick){ return Number.isInteger(tick); } /** * @override */ _yAxisTickFormat() { return d3.format('d'); } _xTxtLabelSplitStr() { return /(?=\/)/; } } // CONCATENATED MODULE: ./src/plot_io_per_dir.js /** * A bar plot displaying directories * with most IO-activity. */ class plot_io_per_dir_PlotIoPerDir extends plot_simple_bar_PlotSimpleBar { generatePlot(commands, siblingElement){ super.generatePlot(dirIoCounts, siblingElement); } /** * @override */ _chartTitle(){ return 'Directories with most input-output-activity'; } /** * @override */ _yScaleBandDomain(){ return [0, dirIoCounts[0].readCount + dirIoCounts[0].writeCount]; } /** * @override */ _xValue(ioStat) { return ioStat.dir; } /** * @override */ _yValue(ioStat) { return ioStat.readCount + ioStat.writeCount; } /** * @override */ _yAxisTicksFilter(tick){ return Number.isInteger(tick); } /** * @override */ _yAxisTickFormat() { return d3.format('d'); } /** * @override */ _xTxtLabelSplitStr() { return /(?=\/)/; } } // CONCATENATED MODULE: ./src/plot_cmdcount_per_session.js /** * A bar plot displaying the sessions * wherein the most commands were executed. */ class plot_cmdcount_per_session_PlotCmdCountPerSession extends plot_simple_bar_PlotSimpleBar { generatePlot(commands, siblingElement) { this._sessionMostCmds = []; sessionsMostCmds.forEach((e) => { this._sessionMostCmds.push( new _SessionMostCmdsEntry(commands[e.idxFirstCmd], e.countOfCommands) ); }); this._maxCountOfCmdsInSession = this._sessionMostCmds[0].countOfCommands; // sort the sessions by start date this._sessionMostCmds.sort((s1, s2) => { return s1.firstCmd.startTime - s2.firstCmd.startTime; }); super.generatePlot(this._sessionMostCmds, siblingElement); } /** * @override */ _chartTitle(){ return 'Sessions with most commands'; } /** * @override */ _yScaleBandDomain(){ return [0, this._maxCountOfCmdsInSession]; } /** * @override */ _xValue(session) { return session.firstCmd.sessionUuid; } /** * @override */ _yValue(session) { return session.countOfCommands; } /** * @override */ _yAxisTicksFilter(tick){ return Number.isInteger(tick); } /** * @override */ _yAxisTickFormat() { return d3.format('d'); } /** * @return {int} * @param {[Command]} cmds1 * @param {[Command]} cmds2 */ _compareBySessionCmdCount(cmds1, cmds2) { return cmds1.length - cmds2.length; } _barColor(session){ return session.firstCmd.sessionColor; } _modifyBars(bars){ bars .style('cursor', 'pointer') .on("click", (session) => { commandList.scrollToCmd(session.firstCmd); }); } _modifyTickText(tickTxt, session) { tickTxt .style('cursor', 'pointer') .on("click", () => { commandList.scrollToCmd(session.firstCmd); }); } } class _SessionMostCmdsEntry { constructor(firstCmd, countOfCommands){ this.firstCmd = firstCmd; this.countOfCommands = countOfCommands; } } // CONCATENATED MODULE: ./src/stats.js async function generateMiscStats() { const body = d3.select('body'); if (typeof commands[0].fileWriteEvents === 'undefined') { // when generating from shournal, command-data (like fileWriteEvents) // is loaded later for performance reasons await timedForEach(commands, (cmd, idx) => { const cmdDataTag = d3.select('#commandDataJSON' + idx); const cmdData = JSON.parse(cmdDataTag.html()); Object.assign(cmd, cmdData); cmdDataTag.remove(); }); } if (mostFileMods.length === 0 && sessionsMostCmds.length === 0 && cwdCmdCounts.length === 0 && dirIoCounts.length === 0) { // No stats to display... return; } body.append('h3') .html('Miscellaneous statistics') .style('padding-top', '1em'); const miscStatElement = body.append('div') .style('padding-top', '20px') .style('display', 'inline-block'); if (mostFileMods.length > 0) { const plotMostWrittenFiles = new plot_most_written_files_PlotMostWrittenFiles(); plotMostWrittenFiles.generatePlot(commands, miscStatElement); } if (sessionsMostCmds.length > 0) { const plotCmdCountPerSession = new plot_cmdcount_per_session_PlotCmdCountPerSession(); plotCmdCountPerSession.generatePlot(commands, miscStatElement); } if(cwdCmdCounts.length > 0){ const plotCmdCountPerCwd = new plot_cmdcount_per_cwd_PlotCmdCountPerCwd(); plotCmdCountPerCwd.generatePlot(commands, miscStatElement); } if (dirIoCounts.length > 0) { const plotIoPerDir = new plot_io_per_dir_PlotIoPerDir(); plotIoPerDir.generatePlot(commands, miscStatElement); } $('[data-toggle="tooltip"]').tooltip({ delay: { show: 300, hide: 0 }, }); } // CONCATENATED MODULE: ./src/index.js function displayErrorAtTop(msg){ // vanilla js, since loading of libraries might have failed const errEl = document.getElementById('topError'); errEl.style["visibility"] = "visible"; errEl.innerHTML = msg; } function main() { if (scriptLoadError) { console.log(scriptLoadError); displayErrorAtTop(scriptLoadError); return; } init(); assert(commands.length > 0, 'commands.length > 0'); const queryDate = d3TimeParseIsoWithMil(ORIGINAL_QUERY_DATE_STR); const body = d3.select('body'); body.append('button') .attr('class', 'btn btn-primary') .style('position', 'absolute') .style('right', '0px') .style('top', '0px') .html("Report Metadata") .on("click", () => { textDialog.show("Report Metadata", `Commandline-query (executed on ` + `${humanDateFormat(queryDate)}): ${ORIGINAL_QUERY}`); }); prepareCommands(commands); { let lastStart = commands[0].startTime; for(let i=1; i < commands.length; i++){ assert(commands[i].startTime >= lastStart); lastStart = commands[i].startTime; } } const cmdFinalEndDate = d3TimeParseIsoWithMil(CMD_FINAL_ENDDATE_STR); // Do not change order -> commandList.size computed based on sessionTimeLine.size. sessionTimeline = new session_timeline_SessionTimeline(commands, cmdFinalEndDate); commandList = new command_list_CommandList(commands); d3.select('#initialSpinner').remove(); $(document).ready(generateMiscStats); } try { main(); } catch (error) { console.log(error); displayErrorAtTop(error); } /***/ }) /******/ ]); ================================================ FILE: html-export/dist/main.licenses.txt ================================================ async-mutex MIT The MIT License (MIT) Copyright (c) 2016 Christian Speckner Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. tinyqueue ISC ISC License Copyright (c) 2017, Vladimir Agafonkin Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ================================================ FILE: html-export/package.json ================================================ { "name": "shournal-html-stats", "version": "1.0.0", "description": "interactively browse shournal's command history", "private": true, "scripts": { "build": "webpack" }, "keywords": [], "author": "Tycho Kirchner", "license": "GPL-3.0", "devDependencies": { "eslint": "^6.5.1", "eslint-config-google": "^0.14.0", "license-webpack-plugin": "^2.1.3", "webpack": "^4.41.1", "webpack-cli": "^3.3.9" }, "dependencies": { "async-mutex": "^0.1.4", "tinyqueue": "^2.0.3" } } ================================================ FILE: html-export/src/annotation_line_render.js ================================================ import * as util from './util'; import {sleep} from './util'; import {Mutex} from 'async-mutex'; /** * Render Groups of annotations on a per-line-basis. Clip annotation texts * and omit annotations as needed to fit into available space */ export default class AnnotationLineRender { constructor(plot) { this._annotationGroups = []; this._plot = plot; // get the width in pixel of a character this._annotationCharWidth = d3.select("#annotation_text_char").node() .getBoundingClientRect().width; // do not render an annotation which does not fit into the space. this._annotationMinWidth = this._annotationCharWidth * 2; // clip annotation-texts after that many characters this._annotationMaxNumChars = 15; this._updateMutex = new Mutex(); this._lastUpdateDummy = null; } /** * * @param {Array} group: ordered set of annotations which will be rendered * within the same line. Base class is the same as d3 annotation, however, the following * *additional* fields must be set: startX, endX, fulltext. The annotation position * (x,y) has to be set already, based on the x-values it is decided, how much of * an annotation is drawn. */ addAnnotationGroup(group) { this._annotationGroups.push(group); } async update(xScale) { this._lastUpdateDummy = {}; const currentUpdateDummy = this._lastUpdateDummy; const release = await this._updateMutex.acquire(); try { // remove and add again seems to be faster than updating this._plot.selectAll('.annotation').remove(); this._plot.selectAll('.annotationVertLine').remove(); this._plot.selectAll('.annotationHorizLine').remove(); const annotations = await this._preRenderAnnotations(xScale, currentUpdateDummy); if (annotations !== null) { this._appendAnnotations(annotations); } } finally { release(); } } setOnNoteClick(func){ this._onNoteClick = func; } // ***************** PRIVATE ******************** _compareStartX(prev, current) { return prev.startX - current.startX; } _compareEndX(prev, current) { return prev.endX - current.endX; } async _preRenderAnnotations(xScale, currentUpdateDummy ) { const annotations = []; // uniform interface for binary search, where the entrance indeces are found const dummyAnnotation = { startX: xScale.domain()[0], endX: xScale.domain()[1], }; const plotWidth = this._plot.node().getBBox().width; for(const annotationLine of this._annotationGroups) { if (annotationLine.length == 0) { continue; } // Do not render annotations outside the current view // -> find start and stop indeces in the group: // Note: one cannot simply choose 0 and length -1 after zooming // out, because panning also has to be respected. const startIdx = util.binarySearch(annotationLine, dummyAnnotation, this._compareStartX, true); const endIdx = util.binarySearch(annotationLine, dummyAnnotation, this._compareEndX, true); let displayAnnotation = annotationLine[startIdx]; displayAnnotation.x = this._calcAnnotationCenter(displayAnnotation, xScale); for (let idx = startIdx + 1; idx <= endIdx; idx++) { // this.update is run async: check if it was called in between. If that's the // case we can abort, because or xScale is outdated. if (currentUpdateDummy !== this._lastUpdateDummy) { return null; } if(idx % 30 === 0){ // avoid freezing the DOM... await sleep(5); } const annotation = annotationLine[idx]; annotation.x = this._calcAnnotationCenter(annotation, xScale); const textspace = annotation.x - displayAnnotation.x - (this._annotationCharWidth * 2); // subtract more chars to leave space to next annotation const annotationTxt = this._generateAnnotationTxt(textspace, displayAnnotation.fulltext); if (annotationTxt == null) { // do not render this annotation continue; } // always update text, we might have zoomed before! displayAnnotation.note.label = annotationTxt; annotations.push(displayAnnotation); displayAnnotation = annotation; } // still need to push the final annotation, if it fits into our plot const textspace = plotWidth - displayAnnotation.x; const annotationTxt = this._generateAnnotationTxt(textspace, displayAnnotation.fulltext); if (annotationTxt != null) { displayAnnotation.note.label = annotationTxt; annotations.push(displayAnnotation); } } return annotations; } _calcAnnotationCenter(annotation, xScale) { return (xScale(annotation.startX) + xScale(annotation.endX)) / 2.0; } /** * @param {*} textspace Available width in pixel * @param {*} txt The full text * @return {*} null, if textspace was too small, else the full or clipped text */ _generateAnnotationTxt(textspace, txt) { if (textspace < this._annotationMinWidth) { return null; } // Render only so many chars that fit into the space, but not more than // _annotationMaxNumChars; const maxCountOfRenderChars = Math.min(Math.ceil(textspace / this._annotationCharWidth) , this._annotationMaxNumChars); if (txt.length <= maxCountOfRenderChars ) { return txt; } return txt.substring(0, maxCountOfRenderChars - 1) + '.'; } /** * Append all annotations to the plot and setup mouse event handlers * @param {[annotation]} annotations */ _appendAnnotations(annotations) { const enterSelection = this._plot.selectAll(".annotation") .data(annotations) .enter(); enterSelection .append("text") .attr('class', 'annotation unselectable' ) .attr('x', (a) => { return a.x; }) .attr('y', (a) => { return a.ny; }) .text((a) => { return a.note.label; }) .attr('title', (a) => { return a.fulltext; }) .style('cursor', 'pointer') .on("click", (a) => { if (this._onNoteClick !== undefined) { // d3.event.pageX, d3.event.pageY this._onNoteClick(a.data); } }); // dynamically inserted elements -> rerun tooltip $('.annotation').tooltip({ delay: { show: 100, hide: 0 }, }); const horzLineYOffset = 2; const lineColor = 'steelblue'; enterSelection .insert("line") .attr('class', 'annotationVertLine') .attr('x1', (a) => { return a.x; }) .attr('y1', (a) => { return a.ny + horzLineYOffset; }) .attr('x2', (a) => { return a.x; }) .attr('y2', (a) => { return a.y; }) .attr("stroke-width", 0.5) .attr("stroke", lineColor); enterSelection .insert("line") .attr('class', 'annotationHorizLine') .attr('x1', (a) => { return a.x; }) .attr('y1', (a) => { return a.ny + horzLineYOffset; }) .attr('x2', (a) => { return a.x+ (a.note.label.length * this._annotationCharWidth); }) .attr('y2', (a) => { return a.ny + horzLineYOffset; }) .attr("stroke-width", 0.5) .attr("stroke", lineColor); } } ================================================ FILE: html-export/src/command_list.js ================================================ import * as html_util from './html_util'; import * as util from './util'; import * as globals from './globals'; import * as conversions from './conversions'; export default class CommandList { constructor(commands) { this._CMDLISTPADDING = 18; this._CMDLISTBG = '#777'; const cmdListHeight = (() => { const boundClient = globals.sessionTimeline.getSvg().node().getBoundingClientRect(); let h = util.windowHeight() - (boundClient.y + boundClient.height) - 30; // why minus 30? if (h < 200) { // screen too small (or too many command groups): allow for scrolling h = 300; } return h; })(); const cmdListScroll = d3.select('body').append('div') .attr('id', 'cmdListScroll') .style('height', cmdListHeight + 'px'); cmdListScroll.selectAll('.collapsibleCmd') .data(commands) .enter() .append('button') .attr('class', 'collapsibleCmd') .attr('id', (cmd) => { return 'cmdListEntry' + cmd.id; }) .html((cmd) => { // only display year,month,day of endTime if different from start const actualEndFormat = (cmd.startTime.getFullYear() == cmd.endTime.getFullYear() && cmd.startTime.getMonth() == cmd.endTime.getMonth() && cmd.startTime.getDay() == cmd.endTime.getDay() ) ? globals.humanDateFormatOnlyTime : globals.humanDateFormat; return globals.humanDateFormat(cmd.startTime) + ' - ' + actualEndFormat(cmd.endTime) + ': ' + cmd.command; }) .style('padding', this._CMDLISTPADDING + 'px') .style('background', (cmd) => { return this._computeCmdBackground(cmd); }) .on("click", (cmd, idx) => { if (document.readyState !== "complete"){ // silently ignore clicks, until everything loaded... return; } this._handleClickOnCmd(cmd, idx); }); } /** * @param {*} cmd command-object to scroll to */ scrollToCmd(cmd) { const cmdElement = this._selectCmdEntry(cmd); const scroll = document.getElementById('cmdListScroll'); scroll.scrollTop = cmdElement.node().offsetTop; cmdElement .transition() .duration(1300) // miliseconds .style("background", "red") .on("end", () => { cmdElement.style('background', (cmd) => { return this._computeCmdBackground(cmd); }); }); } _selectCmdEntry(cmd){ return d3.select(`#cmdListEntry${cmd.id}`); } _computeCmdBackground(cmd){ return `linear-gradient(to right, ${cmd.sessionColor} 0px, ${cmd.sessionColor} ${this._CMDLISTPADDING - 1}px, ${this._CMDLISTBG} ${this._CMDLISTPADDING - 1}px, ${this._CMDLISTBG} 100%)`; } _handleClickOnCmd(cmd, idx){ let contentDiv = d3.select(`#cmdcontent${cmd.id}`); if (! contentDiv.empty()) { contentDiv.remove(); return; } contentDiv = d3.select('body').append('div') .attr('id', `cmdcontent${cmd.id}`) .attr('class', 'collapsibleCmdContent') .html(`Working directory: ${cmd.workingDir}
` + `Command exit status: ${cmd.returnValue}
` + `Session uuid: ${cmd.sessionUuid}
` + `Command id: ${cmd.id}
` + `Hostname: ${cmd.hostname}
`); const alternatingColor = '#D9D9D9'; if (cmd.fileWriteEvents_length > 0) { contentDiv.append('span') .html(cmd.fileWriteEvents_length + ' written files') .style('color', 'red') .style('display', 'block'); contentDiv.selectAll('.nonexistentClass') .data(cmd.fileWriteEvents) .enter() .append('span') .style('display', 'block') .style('background-color', (e, idx) => { return (idx % 2 === 0) ? 'transparent' : alternatingColor; }) .text((e) => { return `${e.path} (${conversions.bytesToHuman(e.size)}), Hash: ${e.hash}`; }); if (cmd.fileWriteEvents.length !== cmd.fileWriteEvents_length) { contentDiv.append('span').html( `... and ` + `${cmd.fileWriteEvents_length - cmd.fileWriteEvents.length}` + ` more (see shournal's query help to increase limits)
`); } } if (cmd.fileReadEvents_length > 0) { if(cmd.fileWriteEvents_length > 0){ contentDiv.append('span').html('
'); } contentDiv.append('span') .html(cmd.fileReadEvents_length + ' read files') .style('color', 'red') .style('display', 'block'); } contentDiv.selectAll('.nonexistentClass') .data(cmd.fileReadEvents) .enter() .append('span') .style('background-color', (e, idx) => { return (idx % 2 === 0) ? 'transparent' : alternatingColor; }) .style('color', (readFile) => { return (readFile.isStoredToDisk) ? 'blue' : 'black'; }) .style('cursor', (readFile) => { return (readFile.isStoredToDisk) ? 'pointer' : 'default'; }) .style('display', 'block') // only one read file per line .text((e) => { return `${e.path} (${conversions.bytesToHuman(e.size)}), Hash: ${e.hash}`; }) .on("click", (readFile) => { if (readFile.isStoredToDisk) { const mtimeHuman = globals.humanDateFormat(d3.isoParse(readFile.mtime)); const title = `Read file ${readFile.path}
` + `mtime: ${mtimeHuman}
` + `size: ${conversions.bytesToHuman(readFile.size)}
` + `hash: ${readFile.hash}
`; const readFileContent = atob(readFileContentMap.get(readFile.id)); globals.textDialog.show(title, readFileContent); } }); if (cmd.fileReadEvents.length !== cmd.fileReadEvents_length) { contentDiv.append('span').html( `... and ` + `${cmd.fileReadEvents_length - cmd.fileReadEvents.length}` + ` more (see shournal's query help to increase limits)
` ); } const cmdElement = this._selectCmdEntry(cmd); html_util.insertAfter(contentDiv.node(), cmdElement.node()); const cmdListScroll = document.getElementById('cmdListScroll'); if(! html_util.isScrolledIntoView(contentDiv.node(), cmdListScroll, true)){ // scroll down one element, so at least the beginning of content is visible: cmdListScroll.scrollTop += cmdElement.node().clientHeight; } } } ================================================ FILE: html-export/src/command_manipulation.js ================================================ import * as globals from './globals'; /** * Parse the command-date into d3's date and assign session colors * @param {[Command]} commands */ export function prepareCommands(commands){ commands.forEach(function(cmd) { cmd.startTime = globals.d3TimeParseIsoWithMil(cmd.startTime); cmd.endTime = globals.d3TimeParseIsoWithMil(cmd.endTime); }); _fillCommandSessionColors(commands); } /** * Can be passed to array.sort or similar functions. * @param {*} cmd1 * @param {*} cmd2 * @return {int} */ export function compareStartDates(cmd1, cmd2) { return cmd1.startTime - cmd2.startTime; } /** * Can be passed to array.sort or similar functions. * @param {*} cmd1 * @param {*} cmd2 * @return {int} */ export function compareEndDates(cmd1, cmd2) { return cmd1.endTime - cmd2.endTime; } /** * Assign session-colors to the given commands. * Each session gets a specific color, after n sessions occurred, colors * start from beginning again. * @param {[Command]} commands */ function _fillCommandSessionColors(commands){ const DISTINCT_COLORS = [ '#e6194b', '#3cb44b', '#ffe119', '#4363d8', '#f58231', '#911eb4', '#46f0f0', '#f032e6', '#bcf60c', '#fabebe', '#008080', '#e6beff', '#9a6324', '#fffac8', '#800000', '#aaffc3', '#808000', '#ffd8b1', '#000075', '#808080', ]; let lastColorIdx = 0; const sessionColorMap = new Map(); commands.forEach(function(cmd) { if(cmd.sessionUuid === null){ cmd.sessionColor = '#000000'; } else { let color = sessionColorMap.get(cmd.sessionUuid); if(color === undefined){ color = DISTINCT_COLORS[lastColorIdx]; sessionColorMap.set(cmd.sessionUuid, color); lastColorIdx++; if(lastColorIdx >= DISTINCT_COLORS.length){ lastColorIdx = 0; } } cmd.sessionColor = color; } }); } ================================================ FILE: html-export/src/command_timeline.js ================================================ import * as util from './util'; import Tooltip from './tooltip'; import AnnotationLineRender from './annotation_line_render'; export default class CommandTimeline { constructor(commands, countOfCmdGroups, cmdFinalEndDate) { this.commands = commands; this.countOfCmdGroups = countOfCmdGroups; this.cmdFinalEndDate = cmdFinalEndDate; this._margin = { top: 20, right: 20, bottom: 24, left: 24, }; this.commandsPerGroup = this._generateCommandsPerGroup(); // get the width in pixel of a character this.annotationCharWidth = d3.select("#annotation_text_char").node() .getBoundingClientRect().width; this.annotationCharHeight = d3.select("#annotation_text_char").node() .getBoundingClientRect().height; // An annotation shall only be displayed, if its minimum width in pixel // is at least 5 character. Warning: do not set < 1 -> text rendering issues for annotations // distance to the belonging command rect this.ANNOTATION_DISTANCE = 15; // minimum width of a cmd-rect. Let it be at least 1, otherwise very short commands // are barely visible (get another color...) this.CMD_MIN_WIDTH = 4; this.TOTAL_CMD_GROUP_HEIGHT = _CmdRectHeights.VERY_MANY_MOD + this.ANNOTATION_DISTANCE + this.annotationCharHeight * 2; // *2 to give some more space this.cmdGroupOffsets = this._generateCommandGroupOffsets(); this.svgWidth = util.windowWidth() - this._margin.left - this._margin.right - 30; // min. height, might be increased below this.svgHeight = 100; // If too many command-groups, increase plot size if (this.svgHeight < this.cmdGroupOffsets[this.cmdGroupOffsets.length - 1] + this.TOTAL_CMD_GROUP_HEIGHT) { this.svgHeight = this.cmdGroupOffsets[this.cmdGroupOffsets.length - 1] + this.TOTAL_CMD_GROUP_HEIGHT; } this.xScale = d3.scaleTime() .range([0, this.svgWidth]); this.axisBottom = d3.axisBottom(this.xScale); this.svg = d3.select('body').append('svg') .attr('width', this._margin.left + this.svgWidth + this._margin.right) .attr('height', this._margin.top + this.svgHeight + this._margin.bottom) .append('g') .attr('transform', 'translate(' + this._margin.left + ',' + this._margin.top + ')') .style('z-index', -1); this._annotationRender = new AnnotationLineRender(this.svg); const listenerRect = this.svg .append('rect') .attr('class', 'listener-rect') .attr('x', 0) .attr('y', -this._margin.top) .attr('width', this._margin.left + this.svgWidth + this._margin.right) .attr('height', this._margin.top + this.svgHeight + this._margin.bottom) .style('opacity', 0); this.xScale.domain([ // the commands are sorted by starttime... this.commands[0].startTime, this.cmdFinalEndDate, ]).nice(); // draw axes this.xAxisDraw = this.svg.insert('g', ':first-child') .attr('class', 'x axis') .attr('transform', 'translate(0,' + this.svgHeight + ')') .call(this.axisBottom // .ticks(d3.timeWeek, 2) // .tickFormat(d3.timeFormat('%b %d')) ); this.tooltip = new Tooltip(); // draw rectangles this.commandRects = this.svg.selectAll('rect') .data(commands) .enter() .append('rect') .attr('x', (cmd) => { return this._calcRectXPosition(cmd, this.xScale); }) .attr('y', (cmd) => { return this.svgHeight - this.cmdGroupOffsets[cmd.vertOffsetGroup]; }) .attr('width', (cmd) => { return this._calcRectWidth(cmd, this.xScale); }) .attr('height', (cmd) => { if(cmd.fileWriteEvents.length === 0) return _CmdRectHeights.NO_MOD; if(cmd.fileWriteEvents.length < 5) return _CmdRectHeights.FEW_MOD; if(cmd.fileWriteEvents.length < 15) return _CmdRectHeights.MANY_MOD; return _CmdRectHeights.VERY_MANY_MOD; }) .attr('fill', (cmd, i) => { return cmd.sessionColor; // maybe_todo: mark 'sessionEnd' with a color? // const p = 0.1 * 100; // const grad = defs.append("linearGradient") // .attr("id", "grad_" + i); // // const color1 = "orange"; // const color2 = "steelblue"; // // grad.append("stop") // .attr("offset", "0%") // .attr("stop-color", color1); // grad.append("stop") // .attr("offset", (p) + "%") // .attr("stop-color", color1); // grad.append("stop") // .attr("offset", (p) + "%") // .attr("stop-color", color2); // grad.append("stop") // .attr("offset", "100%") // .attr("stop-color", color2); // // return "url(#grad_" + i + ")"; }) .on("mouseover", (cmd) => { this.tooltip.show(cmd.command, d3.event.pageX, d3.event.pageY); }) .on("mouseout", () => { this.tooltip.hide(); }) .on("click", (cmd) => { this._commandList.scrollToCmd(cmd); }); this._setupAnnotations(); const zoom = d3.zoom() .scaleExtent([0.001, 5000]) .on("zoom", () => { this._handleZoom(d3.event.transform); }); listenerRect.call(zoom); } getSvg(){ return this.svg; } setCommandList(commandList){ this._commandList = commandList; } _setupAnnotations(){ this.commandsPerGroup.forEach((cmdGroup) => { const annotationGroup = []; cmdGroup.forEach((cmd) => { const annotation = { data: cmd, note: { align: "left", wrap: 'nowrap', // title: "Annotation title" }, y: this.svgHeight - this.cmdGroupOffsets[cmd.vertOffsetGroup], // TODO: use cmdGroupIdx?? dx: 0, dy: - this.ANNOTATION_DISTANCE, startX: cmd.startTime, endX: cmd.endTime, fulltext: cmd.command, }; annotationGroup.push(annotation); }); this._annotationRender.addAnnotationGroup(annotationGroup); }); this._annotationRender.setOnNoteOver((cmd) => { this.tooltip.show(cmd.command, d3.event.pageX, d3.event.pageY); }); this._annotationRender.setOnNoteOut(() => { this.tooltip.hide(); }); this._annotationRender.setOnNoteClick((cmd) => { this._commandList.scrollToCmd(cmd); }); this._annotationRender.update(this.xScale); } _generateCommandsPerGroup() { // put each cmd-groups into separate arrays: const commandsPerGroup = new Array(this.countOfCmdGroups); for (let i = 0; i < commandsPerGroup.length; i++) { commandsPerGroup[i] = []; } this.commands.forEach( (cmd) => { commandsPerGroup[cmd.vertOffsetGroup].push(cmd); }); return commandsPerGroup; } _generateCommandGroupOffsets() { const offsets = []; // dont start directly on the x-axis, but a little higher let currentOffset = _CmdRectHeights.VERY_MANY_MOD; for (let i = 0; i < this.countOfCmdGroups; i++) { offsets.push(currentOffset); currentOffset += this.TOTAL_CMD_GROUP_HEIGHT; } return offsets; } _calcRectXPosition(cmd, xScale) { let startX = xScale(cmd.startTime); const w = xScale(cmd.endTime) - startX; if (w < this.CMD_MIN_WIDTH) { // since a cmd has to have at least that width, but shall be // centered anyway: const center = startX + w / 2.0; startX = center - this.CMD_MIN_WIDTH / 2.0; } return startX; } _calcRectWidth(cmd, xScale) { const w = xScale(cmd.endTime) - xScale(cmd.startTime); if (w < this.CMD_MIN_WIDTH) { return this.CMD_MIN_WIDTH; } return w; } _handleZoom(transform) { const xScaleNew = transform.rescaleX(this.xScale); this.axisBottom.scale(xScaleNew); this.xAxisDraw.call( this.axisBottom // .ticks(d3.timeWeek, 2) // .tickFormat(d3.timeFormat('%b %d')) ); this.commandRects .attr('x', (cmd) => { const pos = this._calcRectXPosition(cmd, xScaleNew); // note: pos may be less than zero which is ok, because // otherwise wide rects may disappear too soon. return pos; }) .attr('width', (cmd) => { return this._calcRectWidth(cmd, xScaleNew); }); this._annotationRender.update(xScaleNew); } } // TODO: document it class _CmdRectHeights { static get NO_MOD() { return 7; } static get FEW_MOD() { return 14; } static get MANY_MOD() { return 20; } static get VERY_MANY_MOD() { return 24; } } ================================================ FILE: html-export/src/conversions.js ================================================ /** * @return {String} human readble byte-size-string * @param {int} bytes * @param {boolean} si if true: use 1000 as base (kB), else 1024 (KiB) */ export function bytesToHuman(bytes, si = false) { const thresh = si ? 1000 : 1024; if (Math.abs(bytes) < thresh) { return bytes + ' B'; } const units = si ? ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'] : ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']; let u = -1; do { bytes /= thresh; ++u; } while (Math.abs(bytes) >= thresh && u < units.length - 1); return bytes.toFixed(1) + ' ' + units[u]; } ================================================ FILE: html-export/src/d3js_util.js ================================================ /** * Wrap long axis labels to mutliple lines by maximum width, splitting words by * the given *delimeter-keeping* splitStr and auto-truncating long words. * Leading and trailing whitespaces of each line are trimmed. * @param {[String]} tickTexts * @param {int} width The max. width in pixels for each label * @param {RegExp} splitStr A regular expression for the split-string, * which keeps the delimeter, e.g. /(?=\s)/. */ export function wrapTextLabels(tickTexts, width, splitStr=/(?=\s)/) { tickTexts.each(function() { const text = d3.select(this); const words = text.text().split(splitStr); let line = []; let lineNumber = 0; const lineHeight = 1.1; // ems const y = text.attr("y"); const dy = parseFloat(text.attr("dy")); let tspan = text.text(null).append("tspan").attr("x", 0).attr("y", y) .attr("dy", dy + "em"); // only increment i, if the word can be for sure drawn to current line for(let i=0; i < words.length; ) { line.push(words[i]); tspan.text(line.join('')); if (tspan.node().getComputedTextLength() <= width) { ++i; continue; } if (line.length === 1) { // this single word is too long to fit into a line -> clip it _truncateAndSetLabelTxt(line[0].trim(), tspan, width); ++i; } else { // this word does not fit any more -> put it to next line // and render all others line.pop(); tspan.text(line.join('').trim()); // NO ++i -> the current word must be rendered in next line } tspan = text.append("tspan").attr("x", 0).attr("y", y) .attr("dy", `${++lineNumber * lineHeight + dy}em`).text(null); line = []; } }); } function _truncateAndSetLabelTxt(labelTxt, tspan, width) { do { labelTxt = labelTxt.slice(0, -3); tspan.text(labelTxt); } while (tspan.node().getComputedTextLength() > width); labelTxt = labelTxt.slice(0, -2); labelTxt += '..'; tspan.text(labelTxt); } ================================================ FILE: html-export/src/generic_text_dialog.js ================================================ export default class GenericTextDialog { constructor() { } show(title, content){ $("#genericModalTitle").html(title); $("#genericModalBody").html(content); $("#genericModal").modal('toggle'); } } ================================================ FILE: html-export/src/globals.js ================================================ import GenericTextDialog from './generic_text_dialog'; export let humanDateFormat; export let humanDateFormatOnlyDate; export let humanDateFormatOnlyTime; export let d3TimeParseIsoWithMil; export let textDialog; export let commandList; export let sessionTimeline; export function init(){ humanDateFormat = d3.timeFormat("%Y-%m-%d %H:%M"); humanDateFormatOnlyDate = d3.timeFormat("%Y-%m-%d"); humanDateFormatOnlyTime = d3.timeFormat("%H:%M"); d3TimeParseIsoWithMil = d3.timeParse("%Y-%m-%dT%H:%M:%S.%L"); textDialog = new GenericTextDialog(); } ================================================ FILE: html-export/src/html_util.js ================================================ export function insertAfter(newNode, referenceNode) { referenceNode.parentNode.insertBefore(newNode, referenceNode.nextSibling); } /** * Check if element is visible inside container - also partially at your wish. * @return {boolean} * @param {Element} element * @param {Element} container * @param {boolean} partial if true, return true, if not completely but partially * visible */ export function isScrolledIntoView(element, container, partial) { // Get container properties const cTop = container.scrollTop; const cBottom = cTop + container.clientHeight; // Get element properties const eTop = element.offsetTop; const eBottom = eTop + element.clientHeight; // Check if in view const isTotal = (eTop >= cTop && eBottom <= cBottom); const isPartial = partial && ( (eTop < cTop && eBottom > cTop) || (eBottom > cBottom && eTop < cBottom) ); return (isTotal || isPartial); } ================================================ FILE: html-export/src/index.js ================================================ import * as command_manipulation from './command_manipulation'; import CommandList from './command_list'; import {assert} from './util'; import * as globals from './globals'; import SessionTimeline from './session_timeline'; import * as stats from './stats'; function displayErrorAtTop(msg){ // vanilla js, since loading of libraries might have failed const errEl = document.getElementById('topError'); errEl.style["visibility"] = "visible"; errEl.innerHTML = msg; } function main() { if (scriptLoadError) { console.log(scriptLoadError); displayErrorAtTop(scriptLoadError); return; } globals.init(); assert(commands.length > 0, 'commands.length > 0'); const queryDate = globals.d3TimeParseIsoWithMil(ORIGINAL_QUERY_DATE_STR); const body = d3.select('body'); body.append('button') .attr('class', 'btn btn-primary') .style('position', 'absolute') .style('right', '0px') .style('top', '0px') .html("Report Metadata") .on("click", () => { globals.textDialog.show("Report Metadata", `Commandline-query (executed on ` + `${globals.humanDateFormat(queryDate)}): ${ORIGINAL_QUERY}`); }); command_manipulation.prepareCommands(commands); { let lastStart = commands[0].startTime; for(let i=1; i < commands.length; i++){ assert(commands[i].startTime >= lastStart); lastStart = commands[i].startTime; } } const cmdFinalEndDate = globals.d3TimeParseIsoWithMil(CMD_FINAL_ENDDATE_STR); // Do not change order -> commandList.size computed based on sessionTimeLine.size. globals.sessionTimeline = new SessionTimeline(commands, cmdFinalEndDate); globals.commandList = new CommandList(commands); d3.select('#initialSpinner').remove(); $(document).ready(stats.generateMiscStats); } try { main(); } catch (error) { console.log(error); displayErrorAtTop(error); } ================================================ FILE: html-export/src/limited_queue.js ================================================ import TinyQueue from 'tinyqueue'; /** * Allow for a max. length of the queue. * Add further convenience functions */ export default class LimitedQueue extends TinyQueue { setMaxLength(l){ this._maxLength = l; } /** * @override */ push(item) { super.push(item); if(this._maxLength !== undefined && this.length > this._maxLength){ this.pop(); } } popAll(){ const items = []; while (this.length > 0) { items.push(this.pop()); } return items; } } ================================================ FILE: html-export/src/map_extended.js ================================================ export default class MapExtended extends Map { /** * Like get() but insert and return a default, if the key * does not exist * @return {*} * @param {*} key * @param {Function} defaultFactory A parameterless function whose return value * is used as default. */ getDefault(key, defaultFactory) { if(defaultFactory === undefined){ throw Error('defaultValue must not be undefined'); } let val = this.get(key); if(val === undefined){ val = defaultFactory(); this.set(key, val); } return val; } } ================================================ FILE: html-export/src/plot_cmdcount_per_cwd.js ================================================ import PlotSimpleBar from './plot_simple_bar'; /** * A bar plot displaying the working directories * where the most commands were executed. */ export default class PlotCmdCountPerCwd extends PlotSimpleBar { generatePlot(commands, siblingElement){ super.generatePlot(cwdCmdCounts, siblingElement); } /** * @override */ _chartTitle(){ return 'Working directories with most commands'; } /** * @override */ _yScaleBandDomain(){ return [0, cwdCmdCounts[0].countOfCommands]; } /** * @override */ _xValue(cwdCmdCount) { return cwdCmdCount.workingDir; } /** * @override */ _yValue(cwdCmdCount) { return cwdCmdCount.countOfCommands; } /** * @override */ _yAxisTicksFilter(tick){ return Number.isInteger(tick); } /** * @override */ _yAxisTickFormat() { return d3.format('d'); } _xTxtLabelSplitStr() { return /(?=\/)/; } } ================================================ FILE: html-export/src/plot_cmdcount_per_session.js ================================================ import PlotSimpleBar from './plot_simple_bar'; import * as globals from './globals'; /** * A bar plot displaying the sessions * wherein the most commands were executed. */ export default class PlotCmdCountPerSession extends PlotSimpleBar { generatePlot(commands, siblingElement) { this._sessionMostCmds = []; sessionsMostCmds.forEach((e) => { this._sessionMostCmds.push( new _SessionMostCmdsEntry(commands[e.idxFirstCmd], e.countOfCommands) ); }); this._maxCountOfCmdsInSession = this._sessionMostCmds[0].countOfCommands; // sort the sessions by start date this._sessionMostCmds.sort((s1, s2) => { return s1.firstCmd.startTime - s2.firstCmd.startTime; }); super.generatePlot(this._sessionMostCmds, siblingElement); } /** * @override */ _chartTitle(){ return 'Sessions with most commands'; } /** * @override */ _yScaleBandDomain(){ return [0, this._maxCountOfCmdsInSession]; } /** * @override */ _xValue(session) { return session.firstCmd.sessionUuid; } /** * @override */ _yValue(session) { return session.countOfCommands; } /** * @override */ _yAxisTicksFilter(tick){ return Number.isInteger(tick); } /** * @override */ _yAxisTickFormat() { return d3.format('d'); } /** * @return {int} * @param {[Command]} cmds1 * @param {[Command]} cmds2 */ _compareBySessionCmdCount(cmds1, cmds2) { return cmds1.length - cmds2.length; } _barColor(session){ return session.firstCmd.sessionColor; } _modifyBars(bars){ bars .style('cursor', 'pointer') .on("click", (session) => { globals.commandList.scrollToCmd(session.firstCmd); }); } _modifyTickText(tickTxt, session) { tickTxt .style('cursor', 'pointer') .on("click", () => { globals.commandList.scrollToCmd(session.firstCmd); }); } } class _SessionMostCmdsEntry { constructor(firstCmd, countOfCommands){ this.firstCmd = firstCmd; this.countOfCommands = countOfCommands; } } ================================================ FILE: html-export/src/plot_io_per_dir.js ================================================ import PlotSimpleBar from './plot_simple_bar'; /** * A bar plot displaying directories * with most IO-activity. */ export default class PlotIoPerDir extends PlotSimpleBar { generatePlot(commands, siblingElement){ super.generatePlot(dirIoCounts, siblingElement); } /** * @override */ _chartTitle(){ return 'Directories with most input-output-activity'; } /** * @override */ _yScaleBandDomain(){ return [0, dirIoCounts[0].readCount + dirIoCounts[0].writeCount]; } /** * @override */ _xValue(ioStat) { return ioStat.dir; } /** * @override */ _yValue(ioStat) { return ioStat.readCount + ioStat.writeCount; } /** * @override */ _yAxisTicksFilter(tick){ return Number.isInteger(tick); } /** * @override */ _yAxisTickFormat() { return d3.format('d'); } /** * @override */ _xTxtLabelSplitStr() { return /(?=\/)/; } } ================================================ FILE: html-export/src/plot_most_written_files.js ================================================ import PlotSimpleBar from './plot_simple_bar'; import * as command_manipulation from './command_manipulation'; import * as globals from './globals'; /** * A bar plot displaying the commands which * modified the most files. */ export default class PlotMostWrittenFiles extends PlotSimpleBar { generatePlot(commands, siblingElement){ this._filteredCmds = []; mostFileMods.forEach((e) => { this._filteredCmds.push(commands[e.idx]); }); this._maxCountOfWfileEvents = this._filteredCmds[0].fileWriteEvents_length; // Be consistent with timeline and sort by date: this._filteredCmds.sort(command_manipulation.compareStartDates); super.generatePlot(this._filteredCmds, siblingElement); } /** * @override */ _chartTitle(){ return 'Commands with most file-modifications'; } /** * @override */ _yScaleBandDomain(){ return [0, this._maxCountOfWfileEvents]; } /** * @override */ _xValue(cmd) { return globals.humanDateFormatOnlyDate(cmd.startTime) + ": " + cmd.command; } /** * @override */ _yValue(cmd) { return cmd.fileWriteEvents_length; } /** * @override */ _yAxisTicksFilter(tick){ return Number.isInteger(tick); } /** * @override */ _yAxisTickFormat() { return d3.format('d'); } _barColor(cmd){ return cmd.sessionColor; } _modifyBars(bars){ bars .style('cursor', 'pointer') .on("click", (cmd) => { globals.commandList.scrollToCmd(cmd); }); } _modifyTickText(tickTxt, cmd) { tickTxt .style('cursor', 'pointer') .on("click", () => { globals.commandList.scrollToCmd(cmd); }); } } ================================================ FILE: html-export/src/plot_simple_bar.js ================================================ import * as d3js_util from './d3js_util'; import {ErrorNotImplemented} from './util'; /** * Base class for several bar plots */ export default class PlotSimpleBar { constructor() { this._margin = { top: 20, right: 20, bottom: 60, left: 40 }; this._width = 500 - this._margin.left - this._margin.right; this._height = 300 - this._margin.top - this._margin.bottom; this._maxBarWidth = 30; } generatePlot(data, siblingElement) { const plotContainer = siblingElement.append('div') .style('position', 'relative') .style('padding', '12px') .style('display', 'inherit'); this._svg = plotContainer.append("svg") .attr("width", this._width + this._margin.left + this._margin.right) .attr("height", this._height + this._margin.top + this._margin.bottom) .append("g") .attr("transform", "translate(" + this._margin.left + "," + this._margin.top + ")"); // chart title const chartTitle = this._svg.append("text") .attr("x", (this._width / 2.0)) .attr("y", -3) .attr("text-anchor", "middle") .style("font-size", "16px") .style("text-decoration", "underline") .text(this._chartTitle()); this._xScaleBand = d3.scaleBand() .range([0, this._width]) .padding(0.1); this._yScaleBand = d3.scaleLinear() // leave some space for the char title: .range([this._height, chartTitle.node().getBoundingClientRect().height * 1.2]); // In case of duplicate x-axis label values they are overridden, which should // never be desired. Instead build a range and access the respective data-array-element // by index. this._xScaleBand.domain(d3.range(data.length)); this._yScaleBand.domain(this._yScaleBandDomain()); const actualBandWidth = (this._xScaleBand.bandwidth() > this._maxBarWidth) ? this._maxBarWidth : this._xScaleBand.bandwidth(); // append the rectangles for the bar chart const dataEnterSelection = this._svg.selectAll(".bar").data(data).enter(); const bars = dataEnterSelection.append("rect") .style('fill',(d, i) => { return this._barColor(d); }) .attr("x", (d, i) => { let x = this._xScaleBand(i); const center = x + this._xScaleBand.bandwidth()/2.0; x = center - actualBandWidth/2.0; return x; }) .attr("width", actualBandWidth) .attr("y", (d) => { return this._yScaleBand(this._yValue(d)); }) .attr("height", (d) => { return this._height - this._yScaleBand(this._yValue(d)); }) .attr('data-toggle', 'tooltip') .attr('title', (d) => { return this._barTooltipTxt(d); }); this._modifyBars(bars); // add the x Axis this._svg.append("g") .attr("transform", "translate(0," + this._height + ")") .call(d3.axisBottom(this._xScaleBand).tickFormat((d,i)=> this._xValue(data[i]))) .selectAll("text") .call((tickTexts) => { const thisPlot = this; tickTexts.each(function (plainTxt, idx) { const text = d3.select(this); text.attr("title", function () { return thisPlot._xAxisTooltipTxt.call(thisPlot, data[idx]); }).attr('data-toggle', 'tooltip') .attr('data-placement', 'left'); thisPlot._modifyTickText(text, data[idx]); }); d3js_util.wrapTextLabels(tickTexts, this._xScaleBand.bandwidth(), this._xTxtLabelSplitStr()); }); // add the y Axis const yAxisTicks = this._yScaleBand.ticks() .filter((tick) => { return this._yAxisTicksFilter(tick); }); this._yaxis = d3.axisLeft(this._yScaleBand); const yTickFormat = this._yAxisTickFormat(); if(yTickFormat !== undefined){ this._yaxis.tickValues(yAxisTicks).tickFormat( yTickFormat ); } this._svg.append("g").call(this._yaxis); } // MUST override methods _chartTitle(){ throw new ErrorNotImplemented(); } _yScaleBandDomain(){ throw new ErrorNotImplemented(); } // Is called for each x-value _xValue(d){ throw new ErrorNotImplemented(); } // Is called for each y-value _yValue(d){ throw new ErrorNotImplemented(); } // MAY override methods _yAxisTicksFilter(tick){ return true; } _yAxisTickFormat() { return undefined; } _modifyTickText(tickTxt, data) {} _xTxtLabelSplitStr() { return /(?=\s)/; } _barTooltipTxt(dataElement){ return this._xValue(dataElement); } _xAxisTooltipTxt(dataElement){ return this._xValue(dataElement); } _barColor(dataElement){ return 'steelblue'; } // apply further modifications to the bars _modifyBars(bars){} } ================================================ FILE: html-export/src/session_timeline.js ================================================ import * as util from './util'; import MapExtended from './map_extended'; import TimelineGroupFind from './timeline_group_find'; import AnnotationLineRender from './annotation_line_render'; import ZoomButtons from './zoom_buttons'; import * as globals from './globals'; export default class SessionTimeline { constructor(commands, cmdFinalEndDate) { this.cmdFinalEndDate = cmdFinalEndDate; this._margin = { top: 20, right: 20, bottom: 24, left: 24, }; // get the width in pixel of a character this.annotationCharWidth = d3.select("#annotation_text_char").node() .getBoundingClientRect().width; this.annotationCharHeight = d3.select("#annotation_text_char").node() .getBoundingClientRect().height; // height of a session with no forks (parallel commands ) this.sessionBaseHeight = this.annotationCharHeight / 1.5; this.sessionPadding = this.annotationCharHeight / 5; // choose less than two, so two parallel commands // are already wider than a lonely command. this.sessionMinHeight = this.sessionBaseHeight / 1.5; // An annotation shall only be displayed, if its minimum width in pixel // is at least 5 character. Warning: do not set < 1 -> text rendering issues for annotations this.annotationMinWidth = this.annotationCharWidth * 5; // distance to the belonging command rect this.annotationDistance = this.annotationCharHeight / 3.0; this.commandRects = []; // minimum width of a cmd-rect. Let it be at least 1, otherwise very short commands // are barely visible (get another color...) this.CMD_MIN_WIDTH = 4; this.svgWidth = util.windowWidth() - this._margin.left - this._margin.right - 30; const plotContainer = d3.select('body').append('div') .style('position', 'relative'); // see https://stackoverflow.com/a/10487329 this.svg = plotContainer.append('svg'); this._annotationRender = new AnnotationLineRender(this.svg); const groupedSessions = this._generateCommandsPerSession(commands); this.svgHeight = Math.max(100, this._prerenderSessions(groupedSessions)); this.xScale = d3.scaleTime() .range([0, this.svgWidth]); this._yScale = d3.scaleLinear() .range([this.svgHeight, 0]); this._yScale.domain([0, this.svgHeight]); this.axisBottom = d3.axisBottom(this.xScale); this.svg.attr('width', this._margin.left + this.svgWidth + this._margin.right) .attr('height', this._margin.top + this.svgHeight + this._margin.bottom) .append('g') .attr('transform', 'translate(' + this._margin.left + ',' + this._margin.top + ')') .style('z-index', -1); const listenerRect = this.svg .append('rect') .attr('class', 'listener-rect') .attr('x', 0) .attr('y', -this._margin.top) .attr('width', this._margin.left + this.svgWidth + this._margin.right) .attr('height', this._margin.top + this.svgHeight + this._margin.bottom) .style('opacity', 0); this.xScale.domain([ // the commands are sorted by starttime... commands[0].startTime, this.cmdFinalEndDate, ]).nice(); // draw axes this.xAxisDraw = this.svg.insert('g', ':first-child') .attr('class', 'x axis') .attr('transform', 'translate(0,' + this.svgHeight + ')') .call(this.axisBottom // .ticks(d3.timeWeek, 2) // .tickFormat(d3.timeFormat('%b %d')) ); const _drawSession = (session, idx, lineIdx) => { // draw rectangles const className = 'sessionTimeSeries' + session.getSessionGroup() + idx; this.commandRects.push(this.svg.selectAll('.' + className) .data(session.getCmdsWithMeta()) .enter() .append('rect') .attr('class', className) .attr('x', (cmdWithMeta) => { return this._calcRectXPosition(cmdWithMeta.cmd, this.xScale); }) .attr('y', (cmdWithMeta) => { // rects are drawn from top to bottom, so add the height: return this._yScale(cmdWithMeta.getY() + cmdWithMeta.getHeight()); }) .attr('width', (cmdWithMeta) => { return this._calcRectWidth(cmdWithMeta.cmd, this.xScale); }) .attr('height', (cmdWithMeta) => { return cmdWithMeta.getHeight(); }) .attr('fill', (cmdWithMeta) => { // TODO: rather determine the session color in this class // on a per line-basis, so the same color appears as seldom // as possible in a given line (?). // But what about the colors in the cmd-list?... return cmdWithMeta.cmd.sessionColor; } ) .style('cursor', 'pointer') .attr('title', (cmdWithMeta) => { return cmdWithMeta.cmd.command; }) .on("click", (cmdWithMeta) => { globals.commandList.scrollToCmd(cmdWithMeta.cmd); }) ); $('.' + className).tooltip({ delay: { show: 50, hide: 0 }, }); }; groupedSessions.forEach((sessionLine, lineIdx) => { sessionLine.forEach((session, sessionIdx) => { _drawSession(session, sessionIdx, lineIdx); }); }); this._preRenderAnnotations(groupedSessions); this._annotationRender.setOnNoteClick((cmdWithMeta) => { globals.commandList.scrollToCmd(cmdWithMeta.cmd); }); this._annotationRender.update(this.xScale); const minTimeMilli = 20000; // do not allow zooming beyond displaying 20 seconds const maxTimeMilli = 6.3072e+11; // approx 20 years const currentWidthMilli = cmdFinalEndDate - commands[0].startTime; const minScaleFactor = currentWidthMilli / maxTimeMilli; const maxScaleFactor = currentWidthMilli / minTimeMilli; const zoom = d3.zoom() // .scaleExtent([0.001, 5000]) .scaleExtent([minScaleFactor, maxScaleFactor]) .on("zoom", () => { this._handleZoom(d3.event.transform); }); this._zoomButtons = new ZoomButtons(plotContainer, listenerRect, zoom); listenerRect.call(zoom); } getSvg(){ return this.svg; } _generateCommandsPerSession(commands) { const assignParallelCmdCounts = (commandsPerSession) => { // find out the number of parallel commands in each session and store it // in the meta-info of each cmd. The groups are already assigned, one command // is parallel to another, if there exists at least one command // between two zero-group-commands. Note that the groups of // those in-between-commands may rise and fall arbitrarily often, // so keep track of the max. commandsPerSession.forEach((session) => { // index in the sessions cmd-array, where the last group 0 was seen let lastZeroGroupIdx = 0; let lastHighestGroup = 0; // yes, <= to simplify handling the final command for (let i = 1; i <= session.getCmdsWithMeta().length; i++) { if (i >= session.getCmdsWithMeta().length || session.getCmdsWithMeta()[i].getGroup() === 0) { // a new group has started or we are at end. Assign the found number of parallel // commands to all affected commands: const countOfParallelCmds = lastHighestGroup + 1; // zero based.. for (let j = lastZeroGroupIdx; j < i; j++) { session.getCmdsWithMeta()[j].setCountOfParallelGroups(countOfParallelCmds); } // also keep track of the max number of parallel commands in this session // for later use session.setMaxCountOfParallelCommands( Math.max(session.getMaxCountOfParallelCommands(), countOfParallelCmds) ); lastZeroGroupIdx = i; lastHighestGroup = 0; } else { // keep track of the highest group lastHighestGroup = Math.max(lastHighestGroup, session.getCmdsWithMeta()[i].getGroup()); } } }); }; const commandsPerSession = new MapExtended(); commands.forEach( (cmd) => { // note: Map()' iteration order is the insert order, which is // desired here -> since the command-array is ordered by startDateTime, // the generated session map is also ordered by startDateTime const session = commandsPerSession.getDefault(cmd.sessionUuid, () => { return new _Session(); }); session.addCmd(cmd); }); assignParallelCmdCounts(commandsPerSession); // assign a group to each session const sessionGrpFind = new TimelineGroupFind(); let maxGroup = 0; commandsPerSession.forEach((session) => { const group = sessionGrpFind.findNextFreeGroup(session.getSessionStartDate(), session.getSessionEndDate()); session.setSessionGroup(group); maxGroup = Math.max(maxGroup, group); }); // generate an array of an array of sessions, so all sessions which have // the same group are in one array (in correct order). // That way one 'line' of sessions can be // drawn easily. const groupedSessions = new Array(maxGroup + 1); for (let i = 0; i < groupedSessions.length; i++) { groupedSessions[i] = []; } commandsPerSession.forEach( (session) => { groupedSessions[session.getSessionGroup()].push(session); }); return groupedSessions; } /** * @return {int} max y offset of the plot * @param {*} groupedSessions */ _prerenderSessions(groupedSessions){ const ANNOTATION_AND_PADDING = this.annotationDistance + this.annotationCharHeight * 1.5; // * 1.5 -> give some more space const _prerenderCmd = (cmdWithMeta, currentOffset, sessionHeight) => { if(cmdWithMeta.getCountOfParallelGroups() === 1){ // non-parallel commands are aligned to session center: cmdWithMeta.setHeight(this.sessionBaseHeight); const y = currentOffset + sessionHeight/2 - this.sessionBaseHeight/2; cmdWithMeta.setY(y); return; } // parallel commands expand in equal parts over the whole sessionHeight // (separated by padding) let cmdHeight = sessionHeight / cmdWithMeta.getCountOfParallelGroups(); if(cmdHeight < this.sessionMinHeight){ cmdHeight = this.sessionMinHeight; } else { cmdHeight -= this.sessionPadding; } cmdWithMeta.setHeight(cmdHeight); const y = currentOffset + (cmdHeight + this.sessionPadding) * cmdWithMeta.getGroup(); cmdWithMeta.setY(y); }; let currentOffset = 0; groupedSessions.forEach((sessionLine, lineIdx) => { // find the max. number of parallel commands in all sessions of the current line: const maxNumberOfParallelCmds = sessionLine.reduce((prev, curr) => { return prev.getMaxCountOfParallelCommands() > curr.getMaxCountOfParallelCommands() ? prev : curr; }).getMaxCountOfParallelCommands(); const sessionHeight = maxNumberOfParallelCmds === 1 ? this.sessionBaseHeight : (this.sessionMinHeight + this.sessionPadding) * maxNumberOfParallelCmds; sessionLine.forEach((session) => { session.getCmdsWithMeta().forEach((cmdWithMeta) => { _prerenderCmd(cmdWithMeta, currentOffset, sessionHeight); }); session.setHeight(sessionHeight); session.setY(currentOffset); }); currentOffset += sessionHeight + ANNOTATION_AND_PADDING; }); return currentOffset; } _preRenderAnnotations(groupedSessions){ groupedSessions.forEach((sessionLine) => { const annotationGroup = []; sessionLine.forEach((session) => { session.getCmdsWithMeta().forEach((cmdWithMeta) => { // only create annotations for the topmost commandgroup // (in case of parallel commands) if(cmdWithMeta.getCountOfParallelGroups() === cmdWithMeta.getGroup() + 1){ annotationGroup.push(this._createAnnotation(cmdWithMeta, session.getY() + session.getHeight() + this.annotationDistance )); } }); }); this._annotationRender.addAnnotationGroup(annotationGroup); }); } _createAnnotation(cmdWithMeta, y){ return { data: cmdWithMeta, note: { align: "left", wrap: 'nowrap', // title: "Annotation title" }, dx: 0, ny: this.svgHeight - y, y: this.svgHeight - (cmdWithMeta.getY() + cmdWithMeta.getHeight()), startX: cmdWithMeta.cmd.startTime, endX: cmdWithMeta.cmd.endTime, fulltext: cmdWithMeta.cmd.command, }; } _calcRectXPosition(cmd, xScale) { let startX = xScale(cmd.startTime); const w = xScale(cmd.endTime) - startX; if (w < this.CMD_MIN_WIDTH) { // since a cmd has to have at least that width, but shall be // centered anyway: const center = startX + w / 2.0; startX = center - this.CMD_MIN_WIDTH / 2.0; } return startX; } _calcRectWidth(cmd, xScale) { const w = xScale(cmd.endTime) - xScale(cmd.startTime); if (w < this.CMD_MIN_WIDTH) { return this.CMD_MIN_WIDTH; } return w; } _handleZoom(transform) { const xScaleNew = transform.rescaleX(this.xScale); this.axisBottom.scale(xScaleNew); this.xAxisDraw.call( this.axisBottom // .ticks(d3.timeWeek, 2) // .tickFormat(d3.timeFormat('%b %d')) ); // maybe_todo: execute in parallel... this.commandRects.forEach((rectGroup) => { rectGroup.attr('x', (cmdWithMeta) => { const pos = this._calcRectXPosition(cmdWithMeta.cmd, xScaleNew); // note: pos may be less than zero which is ok, because // otherwise wide rects may disappear too soon. return pos; }) .attr('width', (cmdWithMeta) => { return this._calcRectWidth(cmdWithMeta.cmd, xScaleNew); }); }); this._annotationRender.update(xScaleNew); } } class _CommandWithMeta{ /** * * @param {Command} cmd * @param {int} group the group assigned within a session */ constructor(cmd, group){ this.cmd = cmd; this._group = group; this._countOfParallelGroups = -1; this._height = 1000; this._y = 0; this._annotation = null; } getGroup(){ return this._group; } setCountOfParallelGroups(val){ this._countOfParallelGroups = val; } getCountOfParallelGroups(){ return this._countOfParallelGroups; } setHeight(val){ this._height = val; } getHeight(){ return this._height; } setY(val){ this._y = val; } getY(){ return this._y; } setAnnotation(val){ this._annotation = val; } getAnnotation(){ return this._annotation; } } class _Session { constructor() { this._cmdsWithMeta = []; this._finalCmdEndDate = util.DATE_MIN; this._groupFind = new TimelineGroupFind(); this._firstCmdStartDate = null; this._sessionGroup = null; this._maxCountOfParallelCmds = null; this._height = null; } /** * The passed commands *must* be sorted (asc) by startTime during * subsequent calls of this method. * @param {Command} cmd */ addCmd(cmd) { if(this._firstCmdStartDate === null){ // commands are sorted by startTime and we are called the first time. this._firstCmdStartDate = cmd.startTime; } // commands are sorted by startTime but the first executed cmd may well finish // last, so incrementally find the final endDate. this._finalCmdEndDate = util.date_max(cmd.endTime, this._finalCmdEndDate); const group = this._groupFind.findNextFreeGroup(cmd.startTime, cmd.endTime); this._cmdsWithMeta.push(new _CommandWithMeta(cmd, group)); } setMaxCountOfParallelCommands(val){ this._maxCountOfParallelCmds = val; } getMaxCountOfParallelCommands(){ return this._maxCountOfParallelCmds; } getSessionStartDate(){ return this._firstCmdStartDate; } getSessionEndDate(){ return this._finalCmdEndDate; } setSessionGroup(val){ this._sessionGroup = val; } getSessionGroup(){ return this._sessionGroup; } getCmdsWithMeta(){ return this._cmdsWithMeta; } setHeight(val){ this._height = val; } getHeight(){ return this._height; } setY(val){ this._y = val; } getY(){ return this._y; } } ================================================ FILE: html-export/src/stats.js ================================================ import PlotMostWrittenFiles from './plot_most_written_files'; import PlotCmdCountPerCwd from './plot_cmdcount_per_cwd'; import PlotIoPerDir from './plot_io_per_dir'; import PlotCmdCountPerSession from './plot_cmdcount_per_session'; import { timedForEach } from './util'; export async function generateMiscStats() { const body = d3.select('body'); if (typeof commands[0].fileWriteEvents === 'undefined') { // when generating from shournal, command-data (like fileWriteEvents) // is loaded later for performance reasons await timedForEach(commands, (cmd, idx) => { const cmdDataTag = d3.select('#commandDataJSON' + idx); const cmdData = JSON.parse(cmdDataTag.html()); Object.assign(cmd, cmdData); cmdDataTag.remove(); }); } if (mostFileMods.length === 0 && sessionsMostCmds.length === 0 && cwdCmdCounts.length === 0 && dirIoCounts.length === 0) { // No stats to display... return; } body.append('h3') .html('Miscellaneous statistics') .style('padding-top', '1em'); const miscStatElement = body.append('div') .style('padding-top', '20px') .style('display', 'inline-block'); if (mostFileMods.length > 0) { const plotMostWrittenFiles = new PlotMostWrittenFiles(); plotMostWrittenFiles.generatePlot(commands, miscStatElement); } if (sessionsMostCmds.length > 0) { const plotCmdCountPerSession = new PlotCmdCountPerSession(); plotCmdCountPerSession.generatePlot(commands, miscStatElement); } if(cwdCmdCounts.length > 0){ const plotCmdCountPerCwd = new PlotCmdCountPerCwd(); plotCmdCountPerCwd.generatePlot(commands, miscStatElement); } if (dirIoCounts.length > 0) { const plotIoPerDir = new PlotIoPerDir(); plotIoPerDir.generatePlot(commands, miscStatElement); } $('[data-toggle="tooltip"]').tooltip({ delay: { show: 300, hide: 0 }, }); } ================================================ FILE: html-export/src/timeline_group_find.js ================================================ import TinyQueue from 'tinyqueue'; /** * Find "groups" in an ordered timeline, so that parallel * events get different (low) groups (integers starting from zero). * Events are defined by start- and end-date. The container, for * whose elements findNextFreeGroup may be called subsequentially, * must be ordered by start-date. */ export default class TimelineGroupFind { constructor(){ this._lastEndDates = []; this._freeGroups = new TinyQueue(); } /** * @return {int} lowest free group, starting from 0. * @param {Date} startDate start date of the next time element * @param {Date} endDate end date of the next time element */ findNextFreeGroup(startDate, endDate){ for (let i = this._lastEndDates.length - 1; i >= 0; i--) { if (startDate > this._lastEndDates[i].endTime) { this._freeGroups.push(this._lastEndDates[i].group); this._lastEndDates.splice(i, 1); } } // if we have free groups (from previous runs) use the lowest free group, // else add a new one const group = (this._freeGroups.length > 0) ? this._freeGroups.pop() : this._lastEndDates.length; this._lastEndDates.push(new _LastEndDateGroup(group, endDate)); return group; } } class _LastEndDateGroup { constructor(group, endTime){ this.group = group; this.endTime = endTime; } } ================================================ FILE: html-export/src/tooltip.js ================================================ export default class Tooltip { constructor(){ this._tooltipDiv = d3.select('body') .append('div') .style("position", "absolute") .style("visibility", 'hidden') .style("background-color", "white") .style("border", "solid") .style("border-width", "2px") .style("border-radius", "5px") .style("padding", "5px") .style("z-index", "1000") .style("pointer-events", "none"); // no flickering in chromium... } show(txt, x, y) { // maybe_todo: if tooltip is too much on the right, it gets clipped. Maybe use solution from // https://stackoverflow.com/a/51066294/7015849 this._tooltipDiv .style("left", x + "px") .style("top", y + "px") .style('visibility', 'visible') .html(txt); } hide() { this._tooltipDiv.style('visibility', 'hidden'); } } ================================================ FILE: html-export/src/util.js ================================================ export class ErrorNotImplemented extends Error { constructor() { super('Required method not implemented'); } } export function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } export function getTime() { return new Date().getTime(); } export function date_max(d1, d2){ return d1 > d2 ? d1 : d2; } export function date_min(d1, d2){ return d1 < d2 ? d1 : d2; } export function windowWidth() { return window.innerWidth || document.documentElement.clientWidth || document.body.clientWidth; } export function windowHeight() { return window.innerHeight || document.documentElement.clientHeight || document.body.clientHeight; } export function assert(condition, message) { if (!condition){ throw Error('Assert failed: ' + (message || '')); } } export const DATE_MIN = new Date(-8640000000000000); /** * non-blocking .foreach array loop. * @param {*} array * @param {*} func */ export async function timedForEach(array, func) { const maxTimePerChunk = 200; // max 200ms until next sleep function getTime() { return new Date().getTime(); } let lastStart = getTime(); for (let i=0; i < array.length; i++) { func(array[i], i, array); const now = getTime(); if(now - lastStart > maxTimePerChunk){ // enough computation time used await sleep(5); lastStart = now; } } } /** * Binary search. * @param {[]} ar sorted array, may contain duplicate elements. * If there are more than one equal elements in the array, * the returned value can be the index of any one of the equal elements. * @param {*} el element to search for * @param {function} compareFn A comparator function. The function takes two arguments: (a, b) and returns: * a negative number if a is less than b; * 0 if a is equal to b; * a positive number of a is greater than b. * @param {boolean} clipIdx see @return: * @return {int} if clipIdx is false: index of of the element in a sorted array or (-n-1) where n * is the insertion point for the new element. * If clipIdx is true: return an index within the array element bounds, independent of * wheter the element exists or not (the best matching existing index is returned). */ export function binarySearch(ar, el, compareFn, clipIdx=false) { const clipIdxIfOn = (idx) => { if(! clipIdx){ return idx; } if (idx < 0) { idx = -(idx + 1); } if (idx >= ar.length) { return ar.length - 1; } return idx; }; let m = 0; let n = ar.length - 1; while (m <= n) { const k = (n + m) >> 1; const cmp = compareFn(el, ar[k]); if (cmp > 0) { m = k + 1; } else if(cmp < 0) { n = k - 1; } else { return clipIdxIfOn(k); } } return clipIdxIfOn(-m - 1); } /** * Get the directry of a unix path, e.g. the path /home/user/foo * would return /home/user. * @return {String} * @param {String} path */ export function getDirFromAbsPath(path){ return path.substring(0,path.lastIndexOf("/")); } ================================================ FILE: html-export/src/zoom_buttons.js ================================================ export default class ZoomButtons { /** * @param {d3-element} containerDiv The plot/svg is excepted to be in that div. * Its 'position' should be 'relative', see https://stackoverflow.com/a/10487329 * so we can place the buttons in an absolute manner. * @param {d3-element} zoomArea the element used for zooming * @param {d3.zoom} d3Zoom */ constructor(containerDiv, zoomArea, d3Zoom) { const btnGroup = containerDiv.append('div'); const zoomInBtn = this._appendZoomButton(btnGroup, '+') .on("click", () => { d3Zoom.scaleBy(zoomArea.transition().duration(10), 1.2); }); const zoomInBtnWidth = parseInt(zoomInBtn.style('width'), 10); const zoomOutBtn = this._appendZoomButton(btnGroup, '-') .on("click", () => { d3Zoom.scaleBy(zoomArea.transition().duration(10), 0.8); }); const zoomOutBtnWidth = parseInt(zoomOutBtn.style('width'), 10); const zoomResetBtn = this._appendZoomButton(btnGroup, '[ ]') .on("click", () => { d3Zoom.transform(zoomArea, d3.zoomIdentity.translate(0, 0).scale(1.0)); }); const zoomResetBtnWidth = parseInt(zoomResetBtn.style('width'), 10); const zoomButtonsWidth = zoomInBtnWidth + zoomOutBtnWidth + zoomResetBtnWidth; btnGroup.style('position', 'absolute') // see https://stackoverflow.com/a/10487329 -> // parent position should be relative .style('top', 0 + 'px') .style('right', ( zoomButtonsWidth) + 'px'); } _appendZoomButton(container, text) { return container.append('button') .attr('class', 'zoomButton') .html(text); } } ================================================ FILE: html-export/webpack.config.js ================================================ const LicenseWebpackPlugin = require('license-webpack-plugin').LicenseWebpackPlugin; module.exports = { optimization:{ minimize: false, // maybe_todo: set to true for release }, plugins: [ new LicenseWebpackPlugin() ], mode:'production', } ================================================ FILE: install/90-shournaladd.rules.in ================================================ ACTION=="add", KERNEL=="shournalk_ctrl", RUN="/bin/sh -c 'test -f /etc/shournal.d/kgroup && read -r ___kgrp < /etc/shournal.d/kgroup || ___kgrp=${GROUPNAME_SHOURNALK}; chgrp $$___kgrp /sys%p/mark'" ================================================ FILE: install/CMakeLists.txt ================================================ # The files here are only required for installation and # have no direct relation to source-code. configure_file( postinst.in "${CMAKE_BINARY_DIR}/debian/postinst" ) configure_file( prerm.in "${CMAKE_BINARY_DIR}/debian/prerm" ) # For the kernelmodule: configure_file( "90-shournaladd.rules.in" "90-shournaladd.rules") configure_file( shournalk-load.conf shournalk.conf ) configure_file( postinst-dkms.in postinst-dkms ) configure_file( prerm-dkms.in prerm-dkms ) if(${SHOURNAL_EDITION} MATCHES "full|ko") install(FILES "${CMAKE_CURRENT_BINARY_DIR}/90-shournaladd.rules" DESTINATION "/lib/udev/rules.d" ) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/shournalk.conf" DESTINATION "/usr/lib/modules-load.d" ) endif() # ${SHOURNAL_EDITION} MATCHES "full" ================================================ FILE: install/postinst-dkms.in ================================================ # No shebang, we are appended as needed! # Copyright (C) 2002-2005 Flavio Stanchina # Copyright (C) 2005-2006 Aric Cyr # Copyright (C) 2007 Mario Limonciello # Copyright (C) 2009 Alberto Milone # Copyright (C) 2021 Tycho Kirchner: modified to fit shournal's needs DKMS_NAME=shournalk # name of the kernel module DKMS_PACKAGE_NAME=shournal # we bundle the dkms package inside shournal for easy installation. DKMS_VERSION="${shournal_version}" postinst_found=0 case "$1" in configure) for DKMS_POSTINST in /usr/lib/dkms/common.postinst /usr/share/$DKMS_PACKAGE_NAME/postinst; do if [ -f $DKMS_POSTINST ]; then $DKMS_POSTINST $DKMS_NAME $DKMS_VERSION /usr/share/$DKMS_PACKAGE_NAME "" $2 postinst_found=1 break fi done if [ "$postinst_found" -eq 1 ]; then # Don't modprobe -r shournalk - we don't want to disturb running processes. modprobe shournalk || : else echo "ERROR: DKMS version is too old and $DKMS_PACKAGE_NAME was not" echo "built with legacy DKMS support." echo "You must either rebuild $DKMS_PACKAGE_NAME with legacy postinst" echo "support or upgrade DKMS to a more current version." exit 1 fi ;; esac exit 0 ================================================ FILE: install/postinst.in ================================================ #!/bin/sh set -e getent group ${MSENTER_GROUPNAME} || groupadd ${MSENTER_GROUPNAME} getent group ${GROUPNAME_SHOURNALK} || groupadd ${GROUPNAME_SHOURNALK} # do not call exit, this file might be appended to.. ================================================ FILE: install/prerm-dkms.in ================================================ # No shebang, we are appended as needed! set -e DKMS_NAME=shournalk DKMS_VERSION="${shournal_version}" case "$1" in remove|upgrade|deconfigure) if [ "$(dkms status -m $DKMS_NAME -v $DKMS_VERSION)" ]; then dkms remove -m $DKMS_NAME -v $DKMS_VERSION --all fi ;; esac exit 0 ================================================ FILE: install/prerm.in ================================================ #!/bin/sh set -e # This file is intentionally left blank for consistency with # postinst-dkms # do not call exit, this file might be appended to.. ================================================ FILE: install/shournalk-load.conf ================================================ shournalk ================================================ FILE: kernel/CMakeLists.txt ================================================ # Find kernel headers list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") find_package(KernelHeaders REQUIRED) set(common_prefix "../src/common") include_directories( ${KERNELHEADERS_INCLUDE_DIRS} "${common_prefix}" ) # Add a dummy-library to satisfy IDE-intellisense - # plain 'make' does the real work (see below) and file Kbuild. # Also used for copying files to build and dkms-dir, so add all source-files here. file(GLOB kernel_src "*.h" "*.c" "${common_prefix}/xxhash_common.h" "${common_prefix}/xxhash_common.c" "${common_prefix}/user_kernerl.h" ) add_library(lib_shournalk_dummy EXCLUDE_FROM_ALL ${kernel_src} ) set_target_properties(lib_shournalk_dummy PROPERTIES LANGUAGE C) # avoid MODULE_LICENSE-warnings. target_compile_definitions(lib_shournalk_dummy PRIVATE -D__KERNEL__ -DMODULE -DCONFIG_MEMCG ) if(CMAKE_BUILD_TYPE MATCHES Debug) set(k_extra_cflags "-DDEBUG") elseif(CMAKE_BUILD_TYPE MATCHES RelWithDebInfo) # profile set(k_extra_cflags "-O3 -DPROFILE") else() # release set(k_extra_cflags "-O3") endif() # The kernel module is compiled in-tree, after we # copied the source-files there. # This has the advantage that the Kbuild file # can be generated and that we compile # the source code in the same way as # a later user from the installed /usr/src/shournalk*. configure_file(Kbuild "${CMAKE_CURRENT_BINARY_DIR}/" @ONLY) add_custom_target(shournalk ALL COMMAND $(MAKE) --file=Kbuild EXTRA_CFLAGS=${k_extra_cflags} shournal_cmake_build=true WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" VERBATIM DEPENDS shournalk_dep_file ) # Better be safe and copy the files on cmake and make. file(COPY ${kernel_src} DESTINATION "${CMAKE_CURRENT_BINARY_DIR}" ) add_custom_command(OUTPUT shournalk_dep_file PRE_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${kernel_src} "${CMAKE_CURRENT_BINARY_DIR}") # Also copy src-files for dkms set(dkms_dir "${CMAKE_BINARY_DIR}/dkms") add_custom_command(TARGET shournalk POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${kernel_src} "${dkms_dir}") configure_file(Kbuild "${dkms_dir}/Makefile" @ONLY) configure_file("dkms.conf.in" "${dkms_dir}/dkms.conf" @ONLY) install( DIRECTORY "${dkms_dir}/" DESTINATION "${shournal_install_dir_shournalk_src}" ) ================================================ FILE: kernel/Kbuild ================================================ $(info building kernel module shournalk version @shournal_version@) obj-m := shournalk.o # shournal_version filled by cmake (see Makefile in generated dkms dir) CFLAGS_MODULE += "-DSHOURNAL_VERSION=\"@shournal_version@\"" shournalk-y += shournalk_main.o event_handler.o shournalk_sysfs.o \ tracepoint_helper.o event_target.o kutil.o event_queue.o \ event_consumer.o shournal_kio.o xxhash_shournalk.o \ kpathtree.o shournalk_test.o shournalk_global.o \ hash_table_str.o kfileextensions.o \ event_consumer_cache.o \ xxhash_common.o \ PWD := $(shell pwd) KVER ?= $(shell uname -r) KBASE ?= /lib/modules/$(KVER) KBUILD_DIR ?= $(KBASE)/build ifeq ($(shournal_cmake_build), true) $(info building kernel module from cmake) all: @$(MAKE) -C $(KBUILD_DIR) M=$(PWD) modules else # usually called by dkms but can also be used standalone no_strip ?= false common_make_args = @$(MAKE) -C $(KBUILD_DIR) M=$(PWD) EXTRA_CFLAGS="-O3" # strip module by default ifndef INSTALL_MOD_STRIP ifneq ($(no_strip), true) $(info stripping is ON by default, use no_strip=true or define \ INSTALL_MOD_STRIP if this is not desired.) common_make_args += INSTALL_MOD_STRIP=1 endif endif all: $(common_make_args) modules install: $(common_make_args) modules_install depmod -A endif clean: @rm -rf *~ *.o *.mod *.mod.c .*.cmd .tmp_versions ================================================ FILE: kernel/LICENSE ================================================ GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Lesser General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. ================================================ FILE: kernel/cmake/FindKernelHeaders.cmake ================================================ # get kernel release execute_process( COMMAND uname -r OUTPUT_VARIABLE KERNEL_RELEASE OUTPUT_STRIP_TRAILING_WHITESPACE ) string(REGEX REPLACE "-[^-]+$" "" KERNEL_RELEASE_NO_ARCH ${KERNEL_RELEASE}) string(REGEX REPLACE "^([0-9]+\.[0-9]+).*$" "\\1" KERNEL_RELEASE_HWE ${KERNEL_RELEASE}) # Find the headers foreach(header_path /usr/src/linux-headers-${KERNEL_RELEASE_NO_ARCH}-common # Debian /usr/src/linux-${KERNEL_RELEASE_NO_ARCH}/include # Opensuse /usr/src/linux-headers-${KERNEL_RELEASE_NO_ARCH} # Ubuntu /usr/src/linux-hwe-${KERNEL_RELEASE_HWE}-headers-${KERNEL_RELEASE_NO_ARCH} # Ubuntu HWE ) if(EXISTS "${header_path}") set(KERNELHEADERS_DIR "${header_path}") break() endif() endforeach() if(NOT (KERNELHEADERS_DIR)) # Red Hat (?) find_path(KERNELHEADERS_DIR include/linux/user.h PATHS /usr/src/kernels/${KERNEL_RELEASE}) endif() message(STATUS "Kernel release: ${KERNEL_RELEASE}") if (KERNELHEADERS_DIR) set(KERNELHEADERS_INCLUDE_DIRS ${KERNELHEADERS_DIR}/include ${KERNELHEADERS_DIR}/arch/x86/include) message(STATUS "Kernel headers: ${KERNELHEADERS_INCLUDE_DIRS}") else() message(WARNING "Unable to find kernel headers!") endif() ================================================ FILE: kernel/dkms.conf.in ================================================ PACKAGE_NAME="shournalk" PACKAGE_VERSION="@shournal_version@" CLEAN="make clean" MAKE[0]="make all KVER=$kernelver" BUILT_MODULE_NAME[0]="shournalk" DEST_MODULE_LOCATION[0]="/updates/dkms" AUTOINSTALL="yes" ================================================ FILE: kernel/event_consumer.c ================================================ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "event_consumer.h" #include "event_consumer_cache.h" #include "event_queue.h" #include "event_target.h" #include "kutil.h" #include "shournal_kio.h" #include "shournalk_user.h" #include "kpathtree.h" #include "xxhash_common.h" #define CONSUMER_CIRC_BUFSIZE (1 << 15) static inline bool __path_is_hidden(const char* pathname, int path_len){ return strnstr(pathname, "/.", path_len) != NULL; } static inline struct file * __reopen_file_silent(const struct path *path, const struct cred * cred){ return dentry_open(path, O_RDONLY | O_NOATIME | FMODE_NONOTIFY, cred); } #ifdef DEBUG static void __dbg_print_event(struct event_target* t __attribute__ ((unused)), struct close_event* ev, const char* msg){ const char* pathname; char* buf = (char*)(__get_free_page(GFP_KERNEL)); if(! buf){ pr_devel("__get_free_page failed!\n"); return; } pathname = d_path(&ev->path, buf, PATH_MAX); if(IS_ERR(pathname)){ pr_devel("%s failed to resolve pathname..\n", msg); } else { pr_devel("%s %s\n", msg, pathname); } free_page((ulong)buf); } #else static void __dbg_print_event(struct event_target* t __attribute__ ((unused)), struct close_event* ev __attribute__ ((unused)), const char* msg __attribute__ ((unused))){} #endif static bool __write_to_target_file_safe(struct event_target* event_target, const void *buf, size_t count){ const char* msg; ssize_t ret = shournal_kio_write(event_target->file, buf, count); if(likely(ret == (ssize_t)count)){ return true; } WRITE_ONCE(event_target->ERROR, true); if(ret >= 0) { ret = EIO; msg = "not all bytes written"; } else { ret = -ret; msg = "errno"; } pr_debug("Failed to write to event target with parent pid %d - %s: %ld\n", event_target->caller_tsk->pid, msg, ret); event_target_write_result_to_user_ONCE(event_target, (int)ret); return false; } /// xxhash the passed file static void __do_hash_file(struct partial_xxhash* part_hash, struct file* file, loff_t file_size, const struct qstr* filename, struct shournalk_close_event* user_event){ struct partial_xxhash_result hash_result; long ret; kutil_WARN_DBG(file_size == 0, "file_size == 0"); kutil_WARN_DBG(part_hash->chunksize == 0, "part_hash->chunksize == 0"); part_hash->seekstep = file_size / part_hash->max_count_of_reads; if(unlikely(ret = partial_xxh_digest_file(file, part_hash, &hash_result))){ pr_devel("failed to partial_hash file with %ld - %s\n", ret, filename->name); goto invalidate_hash; } if(unlikely(hash_result.count_of_bytes == 0)){ // zero bytes read - file became empty in between? pr_devel("zero bytes read for previously non-empty file"); goto invalidate_hash; } user_event->hash_is_null = false; user_event->hash = hash_result.hash; return; invalidate_hash: user_event->hash = 0; user_event->hash_is_null = true; } /// rewrite user_event after having corrected the file content /// size with the actual number of bytes written (should /// happen rarely) static bool __correct_file_event_at_pos(struct event_target* t, loff_t pos, struct shournalk_close_event* user_event){ ssize_t ret; struct file* dest = t->file->__file; if( (ret = kutil_kernel_write(dest, user_event, sizeof(struct shournalk_close_event), &pos )) != sizeof(struct shournalk_close_event)){ WRITE_ONCE(t->ERROR, true); pr_debug("Failed to correct file content size for target %s, returned %ld\n", t->file_init_path, ret); event_target_write_result_to_user_ONCE(t, (int)-ret); return false; } return true; } /// write size bytes from src to /// our target file static bool __write_file_content(struct event_target* t, struct file* src, loff_t size, struct shournalk_close_event* user_event){ ssize_t ret; struct file* dest = t->file->__file; loff_t src_pos = 0; loff_t old_dst_pos; loff_t written_size; if(unlikely(! event_consumer_flush_target_file_safe(t))){ return false; } file_start_write(dest); old_dst_pos = dest->f_pos; ret = do_splice_direct(src, &src_pos, dest, &dest->f_pos, size, 0); written_size = dest->f_pos - old_dst_pos; file_end_write(dest); if(unlikely(written_size != size)){ // before having written the file content, the // close event was written, which we overwrite now. // seek back and correct the written bytes loff_t correct_pos = old_dst_pos - sizeof(struct shournalk_close_event); user_event->bytes = written_size; pr_debug("Only %lld of %lld bytes written - attempting " "to correct this...\n", written_size, size); return __correct_file_event_at_pos(t, correct_pos, user_event); } return true; } static bool __do_log_file_event(struct event_target* t, struct close_event* close_ev, int event_flags, struct qstr* filename, bool store_whole_file, struct consumer_cache_entry* directory, struct path* last_directory){ struct shournalk_close_event user_event; struct file* file = NULL; const struct inode* inode = close_ev->path.dentry->d_inode; if(current->mm && unlikely(current->mm->owner != t->caller_tsk)){ WRITE_ONCE(t->ERROR, true); // See comment in event_consumer_thread_setup // for the rationale. pr_debug("mm->owner does not belong to pid %d " "any more - most likely the caller died. " "Event-logging was stopped.\n", t->caller_tsk->pid); event_target_write_result_to_user_ONCE(t, EREMCHG); return false; } user_event.flags = event_flags; user_event.mtime = kutil_get_mtime_sec(inode); user_event.size = inode->i_size; user_event.mode = inode->i_mode; user_event.hash_is_null = t->partial_hash.chunksize == 0 || unlikely(user_event.size == 0); if(! user_event.hash_is_null || store_whole_file){ file = __reopen_file_silent(&close_ev->path, t->cred); if( unlikely(IS_ERR_OR_NULL(file))) { pr_devel("failed to reopen file %s\n", filename->name); user_event.hash_is_null = true; store_whole_file = false; } else { long ret; // maybe_todo: only set to random, if ! store_whole_file? ret = vfs_fadvise(file, 0,0, POSIX_FADV_RANDOM); if(ret){ pr_devel("vfs_fadvise failed with %ld\n", ret); } } } user_event.bytes = (store_whole_file) ? user_event.size : 0; if(! user_event.hash_is_null){ __do_hash_file(&t->partial_hash, file, user_event.size, filename, &user_event); } if(unlikely(! __write_to_target_file_safe( t, &user_event, sizeof (struct shournalk_close_event))) ){ return false; } if(unlikely(store_whole_file)){ if(__write_file_content(t, file, user_event.bytes, &user_event)){ t->stored_files_count++; } } if(! IS_ERR_OR_NULL(file)){ fput(file); } // Only write directory path, if not written before if(! path_equal(last_directory, &directory->dir)){ __write_to_target_file_safe(t, directory->dirname, directory->dirname_len); __write_to_target_file_safe(t, "/", 1); *last_directory = directory->dir; } __write_to_target_file_safe(t, filename->name, filename->len + 1); return true; } static void __handle_read_event(struct event_target* t, struct close_event* close_ev, bool may_write){ const struct shounalk_settings* sets = &t->settings; bool general_discard; bool store_discard; struct kutil_name_snapshot name_snapshot; struct qstr* filename; const struct inode* inode = close_ev->path.dentry->d_inode; int path_is_hidden = -1; struct path* path = &close_ev->path; struct consumer_cache_entry* d_ent; bool cache_entry_existed; t->r_examined_count++; kutil_WARN_DBG(! t->r_enable, "! t->r_enable"); if(inode->i_nlink == 0){ t->r_deleted_count++; pr_devel("ignore deleted file\n"); return; } // Where possible, we check conditions in ascending order // of the expected computational overhead general_discard = t->r_includes.n_paths == 0 || (sets->r_only_writable && ! may_write); store_discard = t->script_includes.n_paths == 0 || (sets->r_store_only_writable && ! may_write ) || inode->i_size > sets->r_store_max_size || t->stored_files_count >= sets->r_store_max_count_of_files; if(general_discard && store_discard){ // maybe_todo: put this early-discard code directly into the // __fput-handler, to avoid the ringbuffer altogether. // __dbg_print_event(t, close_ev, "early discard"); return; } d_ent = consumer_cache_find( t->event_consumer.r_cache, path->mnt, READ_ONCE(path->dentry->d_parent), &cache_entry_existed); if(IS_ERR(d_ent)){ return; } if(cache_entry_existed){ t->_dircache_hits++; general_discard |= d_ent->flags & DIRCACHE_R_OFF; store_discard |= d_ent->flags & DIRCACHE_SCRIPT_OFF; } else { general_discard |= !kpathtree_is_subpath(&t->r_includes,d_ent->dirname,d_ent->dirname_len,true) || kpathtree_is_subpath(&t->r_excludes,d_ent->dirname,d_ent->dirname_len,true) || (sets->r_exclude_hidden && (path_is_hidden = __path_is_hidden(d_ent->dirname,d_ent->dirname_len))); store_discard |= !kpathtree_is_subpath(&t->script_includes,d_ent->dirname,d_ent->dirname_len,true) || kpathtree_is_subpath(&t->script_excludes,d_ent->dirname,d_ent->dirname_len,true); if(! store_discard && sets->r_store_exclude_hidden){ // use hidden result from above, if possible store_discard = (path_is_hidden != -1) ? path_is_hidden : __path_is_hidden(d_ent->dirname,d_ent->dirname_len); } d_ent->flags = 0; if(general_discard){ d_ent->flags |= DIRCACHE_R_OFF; } if(store_discard){ d_ent->flags |= DIRCACHE_SCRIPT_OFF; } } if(general_discard && store_discard){ return; } // file only settings kutil_take_name_snapshot(&name_snapshot, path->dentry); filename = &name_snapshot.name; general_discard |= (sets->r_exclude_hidden && filename->name[0] == '.'); store_discard |= (sets->r_store_exclude_hidden && filename->name[0] == '.') || (t->script_ext.n_ext && ! file_extensions_contain(&t->script_ext, (const char*)filename->name, filename->len)); if(general_discard && store_discard){ // pr_devel("discarding %s\n", filename); goto out_release; } // Capture store-events regardless of r_max_event_count if(store_discard && t->r_event_count >= sets->r_max_event_count){ t->r_dropped_count++; goto out_release; } // user really wants this event if(likely(__do_log_file_event(t, close_ev, O_RDONLY, filename, !store_discard, d_ent, &t->event_consumer.r_last_written_path ))){ t->r_event_count++; } out_release: kutil_release_name_snapshot(&name_snapshot); } static void __handle_write_event(struct event_target* t, struct close_event* close_ev, bool may_write){ const struct shounalk_settings* sets = &t->settings; struct kutil_name_snapshot name_snapshot; const struct inode* inode = close_ev->path.dentry->d_inode; struct path* path = &close_ev->path; struct consumer_cache_entry* d_ent; bool cache_entry_existed; t->w_examined_count++; // __dbg_print_event(t, close_ev, "processing wevent"); if(inode->i_nlink == 0){ t->w_deleted_count++; pr_devel("ignore deleted file\n"); return; } if(unlikely(! may_write)){ pr_devel("ignore not writable file\n"); return; } d_ent = consumer_cache_find( t->event_consumer.w_cache, path->mnt, READ_ONCE(path->dentry->d_parent), &cache_entry_existed); if(IS_ERR(d_ent)){ return; } // Check if we have seen and accepted our d_parent-dir before. if(cache_entry_existed){ t->_dircache_hits++; if(d_ent->flags & DIRCACHE_W_OFF){ return; } } else { if(!kpathtree_is_subpath(&t->w_includes,d_ent->dirname,d_ent->dirname_len,true) || kpathtree_is_subpath(&t->w_excludes,d_ent->dirname,d_ent->dirname_len,true) || (sets->w_exclude_hidden && __path_is_hidden(d_ent->dirname,d_ent->dirname_len))) { d_ent->flags = DIRCACHE_W_OFF; return; } d_ent->flags = 0; } // directory was accepted - check file: kutil_take_name_snapshot(&name_snapshot, path->dentry); if(sets->w_exclude_hidden && name_snapshot.name.name[0] == '.'){ goto out_release; } if(t->w_event_count >= sets->w_max_event_count){ t->w_dropped_count++; goto out_release; } // user really wants this event if(likely(__do_log_file_event(t, close_ev, O_WRONLY, &name_snapshot.name, false, d_ent, &t->event_consumer.w_last_written_path))){ t->w_event_count++; } out_release: kutil_release_name_snapshot(&name_snapshot); } //////////////////////////////////////////////////////////////////////////// long event_consumer_init(struct event_consumer* consumer){ memset(consumer, 0, sizeof (struct event_consumer)); // To avoid alignment of struct close_event to buffer size, // we simply allocate a little more space, so we do not // overflow right before the ring-buffer wrap-around. consumer->circ_buf.buf = kvzalloc(CONSUMER_CIRC_BUFSIZE + sizeof (struct close_event), SHOURNALK_GFP | __GFP_RETRY_MAYFAIL); if(! consumer->circ_buf.buf) return -ENOMEM; consumer->w_cache = kvzalloc(sizeof (struct consumer_cache), SHOURNALK_GFP | __GFP_RETRY_MAYFAIL); if(! consumer->w_cache) goto err1; consumer->r_cache = kvzalloc(sizeof (struct consumer_cache), SHOURNALK_GFP | __GFP_RETRY_MAYFAIL); if(! consumer->r_cache) goto err2; consumer->circ_buf_size = CONSUMER_CIRC_BUFSIZE; spin_lock_init(&consumer->queue_lock); sema_init(&consumer->start_sema, 0); consumer_cache_init(consumer->w_cache); consumer_cache_init(consumer->r_cache); return 0; err2: kvfree(consumer->w_cache); err1: kvfree(consumer->circ_buf.buf); return -ENOMEM; } void event_consumer_cleanup(struct event_consumer* c){ if(! IS_ERR_OR_NULL(c->consume_task)){ put_task_struct(c->consume_task); } kvfree(c->r_cache); kvfree(c->w_cache); kvfree(c->circ_buf.buf); } long event_consumer_thread_create(struct event_target* event_target, const char* thread_name){ struct event_consumer* consumer = &event_target->event_consumer; consumer->consume_task = kthread_create(event_queue_consume_thread, event_target, "%s", thread_name); if(IS_ERR(consumer->consume_task)){ pr_warn("Failed to create consume thread %s - %ld\n", thread_name, PTR_ERR(consumer->consume_task)); return PTR_ERR(consumer->consume_task); } get_task_struct(consumer->consume_task); wake_up_process(consumer->consume_task); // see documentation of kthread_stop (linux 4.19): // if kthread is stopped very early, // the threadfn might *never* be called. Here this might happen // during the observation of short-lived processes. // We must however make sure, it runs at least once, as // events might be pending. So wait, until our thread calls "up". down(&consumer->start_sema); return 0; } void event_consumer_thread_setup(struct event_target* event_target){ struct event_consumer* consumer = &event_target->event_consumer; if(event_target->mm) { #ifdef USE_MM_SET_FS_OFF consumer->consume_tsk_oldfs = get_fs(); set_fs(USER_DS); #endif kutil_use_mm(event_target->mm); } // use_mm() -> We want this kthread's page-cache memory allocations to account to // the callers memcg. At least on linux 4.19 and ext4 using the mm of the // caller should suffice. See also below stacktrace, which shows how // exactly the memcg is used. Note however that mm->owner might be set to null, // in case our parent process exits or execs, so we will only keep on logging file // events, if current->mm->owner == event_target->caller_tsk. See // also exit.c:mm_update_next_owner // // # ext4 mem_cgroup charging // First the mem_cgroup is associated with a page: // (some parts of the stacktrace were omitted for better readability). // // ext4_da_write_begin // grab_cache_page_write_begin // pagecache_get_page // __alloc_pages_nodemask // add_to_page_cache_lru // __add_to_page_cache_locked // mem_cgroup_try_charge(current->mm) <-- ! // get_mem_cgroup_from_mm <-- owner != null for correct accounting // try_charge // memcgroup_commit_charge: page->mem_cgroup = memcg; <-- ! // // Then the mem_cgroup is taken later // // ext4_block_write_begin // create_empty_buffers // alloc_page_buffers ( in fs/buffer.c, uses __GFP_ACCOUNT!) // get_mem_cgroup_from_page: memcg = page->mem_cgroup; <-- ! // alloc_buffer_head // kmem_cache_alloc // set_user_nice(current, 1); // 2? 10? MAX_NICE? // Only affects reads. See also: https://unix.stackexchange.com/a/480863/288001 set_task_ioprio(current, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 6)); // process events with user credentials consumer->consume_task_orig_cred = override_creds(event_target->cred); } void event_consumer_thread_cleanup(struct event_target* event_target){ struct event_consumer* consumer = &event_target->event_consumer; kutil_WARN_ONCE_IFN_DBG(current != consumer->consume_task, "current != consumer->consume_task"); revert_creds(consumer->consume_task_orig_cred); if(event_target->mm){ kutil_unuse_mm(event_target->mm); #ifdef USE_MM_SET_FS_OFF set_fs(consumer->consume_tsk_oldfs); #endif } } void event_consumer_thread_stop(struct event_consumer* consumer){ int ret; if( (ret = kthread_stop(consumer->consume_task))){ kutil_WARN_ONCE_IFN_DBG(1, "event-consume-thread returned %d\n", ret); } } bool event_consumer_flush_target_file_safe(struct event_target *t) { ssize_t ret; if(unlikely((ret = shournal_kio_flush(t->file)) < 0)){ WRITE_ONCE(t->ERROR, true); pr_debug("Failed to flush event target file %s, returned %ld\n", t->file_init_path, ret); event_target_write_result_to_user_ONCE(t, (int)-ret); return false; } return true; } void close_event_consume(struct event_target* event_target, struct close_event* close_ev){ bool may_read; bool may_write; if(unlikely(event_target->ERROR)){ goto out; } if(unlikely(! close_ev->path.dentry->d_inode)){ pr_devel("ignore event, inode is NULL.\n"); goto out; } may_read = kutil_inode_permission(&close_ev->path, MAY_READ) == 0; if(unlikely(! may_read)){ // maybe the file event came from a setuid-program? Otherwise, the file // might be writable, but not readable. We ignore this special // case here, because we cannot (securely) hash it anyway. __dbg_print_event(event_target, close_ev, "may_read is false"); goto out; } // __dbg_print_event(event_target, close_event, "test"); may_write = kutil_inode_permission(&close_ev->path, MAY_WRITE) == 0; // Just as fanotify does, we consider O_RDWR only // as write-event. // maybe_todo: differentiate? if(close_ev->f_mode & FMODE_WRITE){ __handle_write_event(event_target, close_ev, may_write); } else { __handle_read_event(event_target, close_ev, may_write); } out: close_event_cleanup(close_ev); } void close_event_cleanup(struct close_event* event){ dput(event->path.dentry); mntput(event->path.mnt); } ================================================ FILE: kernel/event_consumer.h ================================================ #pragma once #include "shournalk_global.h" #include #include #include #include #include #include #include #include #include #include "kutil.h" struct event_target; struct consumer_cache; struct close_event { struct path path; fmode_t f_mode; }; struct event_consumer { struct circ_buf circ_buf; struct spinlock queue_lock; int circ_buf_size; bool woken_up; struct task_struct* consume_task; const struct cred* consume_task_orig_cred; struct consumer_cache* w_cache; struct path w_last_written_path; /* last logged full path */ struct consumer_cache* r_cache; struct path r_last_written_path; struct semaphore start_sema; #ifdef USE_MM_SET_FS_OFF mm_segment_t consume_tsk_oldfs; #endif }; long event_consumer_init(struct event_consumer*); void event_consumer_cleanup(struct event_consumer*); long event_consumer_thread_create(struct event_target* event_target, const char* thread_name); void event_consumer_thread_setup(struct event_target* event_target); void event_consumer_thread_cleanup(struct event_target* event_target); void event_consumer_thread_stop(struct event_consumer* consumer); bool event_consumer_flush_target_file_safe(struct event_target*); void close_event_consume(struct event_target*, struct close_event*); void close_event_cleanup(struct close_event* event); ================================================ FILE: kernel/event_consumer_cache.c ================================================ #include #include #include "event_consumer_cache.h" #include "kutil.h" // stolen from fs/proc/base.c:do_proc_readlink static inline size_t d_path_len(const char* buf, size_t buflen, const char* pathname){ return buf + buflen - 1 - pathname; } static void __cache_entry_init(struct consumer_cache_entry* e){ e->dir.mnt = NULL; e->dir.dentry = NULL; e->dirname = NULL; e->dirname_len = 0; e->flags = 0; e->__cache_invalid_jiffy = 0; } static bool __cache_entry_hit(const struct consumer_cache_entry*e, const struct vfsmount *mnt, const struct dentry *dentry){ return dentry == e->dir.dentry && mnt == e->dir.mnt && time_is_after_jiffies(e->__cache_invalid_jiffy); } static bool __append_dname_to_parent(struct consumer_cache_entry* parent, struct qstr* dname ){ if(parent->dirname_len + dname->len >= sizeof(parent->__dirname_buf)){ pr_devel("path-buffer too small for %s/%s", parent->dirname, dname->name); return false; } if(parent->dirname != parent->__dirname_buf){ // d_path prepends backwards, to make sure we have the full buffer-length // move to the front of our buffer memmove(parent->__dirname_buf, parent->dirname, parent->dirname_len + 1); parent->dirname = parent->__dirname_buf; } if(parent->dirname_len > 1){ // not the root node parent->dirname[parent->dirname_len] = '/'; parent->dirname_len++; } memcpy(parent->dirname + parent->dirname_len, (const char*)dname->name, dname->len + 1); parent->dirname_len += dname->len; return true; } void consumer_cache_init(struct consumer_cache* c){ __cache_entry_init(&c->_last_entry); } /// Try to find cached meta-data for the given directory /// @param existed: set to true, if existed /// @return the found or new entry or an ERROR_PTR on err. Note that /// in rare cases the corresponding directory-path may be *wrong*, because /// currently no reference on struct path is held! struct consumer_cache_entry* consumer_cache_find( struct consumer_cache* c, struct vfsmount *mnt, struct dentry *dentry, bool* existed){ // dentry is initialized NULL, so on first call we never return true struct consumer_cache_entry* e = &c->_last_entry; struct dentry* dparent; if(__cache_entry_hit(e, mnt, dentry)){ *existed = true; return e; } dparent = READ_ONCE(dentry->d_parent); if(__cache_entry_hit(e, mnt, dparent)){ bool append_success; struct kutil_name_snapshot name_snapshot; kutil_take_name_snapshot(&name_snapshot, dentry); append_success = __append_dname_to_parent(e, &name_snapshot.name); kutil_release_name_snapshot(&name_snapshot); if(unlikely(! append_success)){ return ERR_PTR(-EDOM); } e->dir.dentry = dentry; // for now, set existed to false, because we don't know // whether child is e.g. an exclude-dir, if parent was so. *existed = false; return e; } *existed = false; // maybe_todo: hold a path_get reference for correctness (implications?)? e->dir.mnt = mnt; e->dir.dentry = dentry; e->dirname = d_path(&e->dir, e->__dirname_buf, PATH_MAX); if (IS_ERR(e->dirname)) { e->dir.dentry = NULL; pr_devel("failed to resolve pathname\n"); // Dbg: print raw path in case d_path fail (why?) // pathname = dentry_path_raw(e->file->f_path.dentry, g_tmp_path, PATH_MAX); return (struct consumer_cache_entry*)e->dirname; } e->dirname_len = (int)(d_path_len(e->__dirname_buf, PATH_MAX, e->dirname)); e->__cache_invalid_jiffy = jiffies + msecs_to_jiffies(5000); return e; } ================================================ FILE: kernel/event_consumer_cache.h ================================================ /* Cache d_names and settings (in flags) for * the given struct path. Note that in rare cases * wrong (older) paths may be returned: * 1. Currently we hold no referecne on the struct path. If the * memory adress is reused, we return the old path. * 2. On a hit, we do not resolve the path again, which might * have changed meanwhile. * However, the cache is invalidated after a short time and * at least the filename is always correct. * */ #pragma once #include "shournalk_global.h" #include #include // For consumer_cache_entry.flags enum { DIRCACHE_W_OFF = 1 << 0, DIRCACHE_R_OFF = 1 << 1, DIRCACHE_SCRIPT_OFF = 1 << 2, }; struct consumer_cache_entry { struct path dir; /* WARNING - do not dereference */ char* dirname; int dirname_len; int flags; // e.g. DIRCACHE_W_OFF unsigned long __cache_invalid_jiffy; char __dirname_buf[PATH_MAX]; }; struct consumer_cache { struct consumer_cache_entry _last_entry; }; void consumer_cache_init(struct consumer_cache*); struct consumer_cache_entry* consumer_cache_find( struct consumer_cache*, struct vfsmount*, struct dentry*, bool* existed); ================================================ FILE: kernel/event_handler.c ================================================ #include #include #include #include #include #include #include #include #include "event_handler.h" #include "event_target.h" #include "event_queue.h" #include "kutil.h" #include "tracepoint_helper.h" struct task_entry { struct task_struct* tsk; struct event_target* event_target; // here file events are written into struct hlist_node node ; struct rcu_work destroy_rwork; } ; static DEFINE_HASHTABLE(task_table, 16); static struct kmem_cache * __task_entry_cache; static DEFINE_SPINLOCK(task_table_lock); static struct workqueue_struct* del_taskentries_wq = NULL; static struct task_entry* __task_entry_alloc(void){ return kmem_cache_alloc(__task_entry_cache, GFP_NOWAIT | __GFP_ACCOUNT | __GFP_NOWARN); } /// Warning: May sleep! static void __task_entry_destroy(struct task_entry* e){ event_target_put(e->event_target); kmem_cache_free(__task_entry_cache ,e); } static void __task_entry_destroy_work(struct work_struct *work){ struct task_entry* el = container_of(to_rcu_work(work), struct task_entry, destroy_rwork); __task_entry_destroy(el); } static inline u32 __task_hash(struct task_struct* task) { return (u32)(long)task; } static inline struct task_entry* __find_task_entry(struct task_struct* task, u32 task_hash){ struct task_entry* el; hash_for_each_possible_rcu(task_table, el, node, task_hash) { if(el->tsk == task){ return el; } } return NULL; } /// find and get a reference from task table under rcu_lock static inline __attribute__((__warn_unused_result__)) struct event_target* __find_get_event_target_safe(struct task_struct* task){ struct task_entry* el; struct event_target* event_target; u32 t_hash; t_hash = __task_hash(task); rcu_read_lock(); if((el = __find_task_entry(task, t_hash)) == NULL){ rcu_read_unlock(); return NULL; } event_target = event_target_get(el->event_target); rcu_read_unlock(); return event_target; } /// Called when we stop observing the task set in the event_target's /// exit_tsk, either because it exited or it was unmarked for observation. static inline void __handle_exit_tsk_remove(struct task_struct *task, bool in_exit, struct event_target* event_target){ if(in_exit){ // see kernel/exit.c: the lower 8 bits are shifted. // do_exit((error_code&0xff)<<8); // Undo that: unsigned lower_exit_code = (task->exit_code >> 8) & 0xff; smp_store_mb(event_target->exit_code, lower_exit_code); pr_devel("event_target caller %d: parent task %d exited with %d\n", event_target->caller_tsk->pid, task->pid, lower_exit_code); } else { WRITE_ONCE(event_target->exit_tsk, NULL); pr_debug("exit_tsk pid %d unset for " "caller %d\n", task->pid, event_target->caller_tsk->pid); } } /// Insert the given task into the table. If the task exists /// and param update_if_exist is true, the event_target is updated, else /// -EEXIST is returned. On success, the target's ref-counter is incremeneted. static long __insert_task_into_table_safe(struct task_struct *task, struct event_target* target, bool update_if_exist){ struct task_entry* el; u32 t_hash; long ret = 0; struct event_target* old_target; t_hash = __task_hash(task); rcu_read_lock(); // create-if-not-exist in same lock! spin_lock(&task_table_lock); if( likely((el=__find_task_entry(task, t_hash)) == NULL)) { // Whoever is interested in the events, pays for the allocation. struct mem_cgroup * oldcg; oldcg = kutil_set_active_memcg(target->memcg); el = __task_entry_alloc(); kutil_set_active_memcg(oldcg); if(! el){ ret = -ENOMEM; goto out_unlock; } el->tsk = task; el->event_target = event_target_get(target); hash_add_rcu(task_table, &el->node, t_hash); goto out_unlock; } // target exists. Fail, if update not allowed if(! update_if_exist){ ret = -EEXIST; goto out_unlock; } old_target = el->event_target; // first increment, then decrement ref-counter // in case old_target == new_target. el->event_target = event_target_get(target); if(old_target != target){ pr_debug("pid %d: event_target " "caller changed from %d to %d\n", task->pid, old_target->caller_tsk->pid, target->caller_tsk->pid); if(unlikely(old_target->exit_tsk == task)){ __handle_exit_tsk_remove(task, false, old_target); } } spin_unlock(&task_table_lock); rcu_read_unlock(); // might_sleep, so put outside of lock event_target_put(old_target); return ret; out_unlock: spin_unlock(&task_table_lock); rcu_read_unlock(); return ret; } static bool __remove_task_from_table_safe(struct task_struct *task, bool in_exit){ struct task_entry* el; bool removed = false; u32 t_hash; t_hash = __task_hash(task); rcu_read_lock(); if((el = __find_task_entry(task, t_hash)) != NULL){ spin_lock(&task_table_lock); hash_del_rcu(&el->node); // Maybe it is safe to concurrently INIT_RCU_WORK outside the spinlock. // But better safe than sorry. INIT_RCU_WORK(&el->destroy_rwork, __task_entry_destroy_work); spin_unlock(&task_table_lock); if(unlikely(el->event_target->exit_tsk == task)){ __handle_exit_tsk_remove(task, in_exit, el->event_target); } // free later queue_rcu_work(del_taskentries_wq, &el->destroy_rwork); removed = true; // pr_devel("stop observing pid %d, init event path %s, caller %d\n", // task->pid, el->event_target->file_init_path, // el->event_target->caller_pid); } rcu_read_unlock(); return removed; } static inline bool __fput_is_interesting(const struct file* file, const struct event_target* t){ bool w_enable; bool r_enable; w_enable = READ_ONCE(t->w_enable); r_enable = READ_ONCE(t->r_enable); return (w_enable && file->f_mode & FMODE_WRITE) || (r_enable && file->f_mode & FMODE_READ); } /// Check if @param task has same owner as current process /// stolen from kernel/sched/core.c static bool __task_check_same_owner(struct task_struct *task) { const struct cred *cred_me = current_cred(); const struct cred *cred_task; bool match; rcu_read_lock(); cred_task = __task_cred(task); match = (uid_eq(cred_me->euid, cred_task->euid) || uid_eq(cred_me->euid, cred_task->uid)); rcu_read_unlock(); return match; } static struct task_struct* __get_task_if_allowed(pid_t pid){ struct task_struct* tsk; rcu_read_lock(); tsk = get_pid_task(find_vpid(pid), PIDTYPE_PID); rcu_read_unlock(); if( IS_ERR_OR_NULL(tsk)){ // no such process in current pid-namespace pr_devel("pid %d does not exist in current pid namespace", pid); return ERR_PTR(-ESRCH); } if(! __task_check_same_owner(tsk)){ pr_devel("pid %d has different owner", tsk->pid); put_task_struct(tsk); return ERR_PTR(-EPERM); } return tsk; } int event_handler_constructor(void) { del_taskentries_wq = system_long_wq; hash_init(task_table); __task_entry_cache = KMEM_CACHE(task_entry, 0); if(! __task_entry_cache) return -ENOMEM; return 0; } void event_handler_destructor(void) { u32 bucket; struct task_entry* el; struct hlist_node *temp_node; // First wait for all call_rcu() (called in queue_rcu_work) to // complete using rcu_barrier(), then flush the used workqueue. // Note that we are the only thread with access to the task_table, // since sysfs and tracepoints were already disabled. When this // function returns, all event_targets should have been freed. // See also Documentation/RCU/rcubarrier.txt: synchronize_rcu() is // *not* sufficent! We have to wait "for all outstanding RCU // callbacks to complete". rcu_barrier(); flush_workqueue(del_taskentries_wq); hash_for_each_safe(task_table, bucket, temp_node, el, node) { hash_del(&el->node); __task_entry_destroy(el); } kmem_cache_destroy(__task_entry_cache); } struct event_target* get_event_target_from_pid(pid_t pid){ struct task_struct* task; struct event_target* event_target; task = __get_task_if_allowed(pid); if(IS_ERR(task)){ return (struct event_target*)task; } event_target = __find_get_event_target_safe(task); if(unlikely(event_target == NULL)){ event_target = ERR_PTR(-ENXIO); } put_task_struct(task); return event_target; } /// Register param event_target as target for file events for the /// given pid. The respective task must be running and /// the caller must have the necessary capabilities. If the process is /// already observed by another event_target we silently replace it /// with the new one. /// @param collect_exitcode: if set to true, set this task as the /// "exit_tsk" for which to collect the exit code long event_handler_add_pid(struct event_target* event_target, pid_t pid, bool collect_exitcode){ struct task_struct* task; long ret = 0; task = __get_task_if_allowed(pid); if(IS_ERR(task)){ return PTR_ERR(task); } ret = __insert_task_into_table_safe(task, event_target, true); // We may have been asked to trace a task which is just about to exit and there is a // small timeslot, where the task is still there but has already called our traced // function cgroup_exit. Is this case, we have just created a stale event_target // reference. Look at kernel/exit.c::do_exit. do_exit sets the PF_EXITING flag before // calling cgroup_exit, so below code is fine. if (unlikely(READ_ONCE(task->flags) & PF_EXITING)) { pr_debug("just marked an exiting task. Removing it again"); __remove_task_from_table_safe(task, false); } else if(collect_exitcode && ret == 0){ WRITE_ONCE(event_target->exit_tsk, task); } put_task_struct(task); return ret; } long event_handler_remove_pid(pid_t pid){ struct task_struct* task; long ret = 0; task = __get_task_if_allowed(pid); if(IS_ERR(task)){ return PTR_ERR(task); } if(! __remove_task_from_table_safe(task, false)){ ret = -ESRCH; } put_task_struct(task); return ret; } /// If the current task shall be observed, /// enqueue the file event for later processing. /// Endless recursion is avoided by /// using the flag FMODE_NONOTIFY and by observing /// only regular files (so the target pipe does no harm /// as well). void event_handler_fput(unsigned long ip __attribute__ ((unused)), unsigned long parent_ip __attribute__ ((unused)), struct ftrace_ops *op __attribute__ ((unused)), struct pt_regs *regs) { struct event_target* event_target; struct file* file; if(unlikely(current->flags & PF_KTHREAD)) return; file = (struct file*)(kutil_get_first_arg_from_reg(tracepoint_helper_get_ftrace_regs(regs))); // Ideally we would ftrace fsnotify_close which is, however, inlined // (thus cannot be traced). // Below code is partially duplicated from there. if (file->f_mode & FMODE_NONOTIFY || // maybe_todo: check file_inode(file) == NULL ifndef FMODE_OPENED !S_ISREG(file_inode(file)->i_mode) ) return; // ftrace doc recommends to check this, however, __fput() calls dput() which // does rcu_read_lock() itself, so we should be safe. // if(! rcu_is_watching()) return; kutil_WARN_DBG(atomic_read(&file_inode(file)->i_count) < 1, "file_inode(file)->i_count < 1"); if((event_target = __find_get_event_target_safe(current)) == NULL ){ return; } if( unlikely(! __fput_is_interesting(file, event_target))) goto out_put; // event_target ownership transferred to queue! event_queue_add(event_target, file); return; out_put: // Might sleep! event_target_put(event_target); } void event_handler_process_exit(struct task_struct *task) { if(unlikely(current->flags & PF_KTHREAD)) return; if (unlikely(!rcu_is_watching())){ kutil_WARN_DBG(1, "called without rcu"); return; } __remove_task_from_table_safe(task, true); } /// If the parent task is observed and the child task is in /// the same pid namespace, also add the child task to our /// task_table void event_handler_process_fork(struct task_struct *parent, struct task_struct *child){ struct event_target* target; long ret; if(unlikely(current->flags & PF_KTHREAD)) return; if (unlikely(!rcu_is_watching())){ kutil_WARN_DBG(1, "called without rcu"); return; } if((target = __find_get_event_target_safe(parent)) == NULL ){ // parent not observed -> ignore child too return; } if(unlikely(READ_ONCE(target->ERROR))){ pr_devel("Ignore fork of pid %d. Error flag " "of event target is set.", parent->pid); goto put_out; } if(unlikely(task_active_pid_ns(child) != target->pid_ns)){ pr_devel("pid namespace does not match " "to parent with pid %d. Ignore.\n", parent->pid); goto put_out; } if(unlikely((ret = __insert_task_into_table_safe(child, target, false)))){ // fixme: set some flag in this event_target. pr_debug("failed to observe child process: %ld", ret); goto put_out; } put_out: event_target_put(target); } ================================================ FILE: kernel/event_handler.h ================================================ #pragma once #include "shournalk_global.h" #include struct event_target; struct ftrace_ops; struct pt_regs; int event_handler_constructor(void); void event_handler_destructor(void); struct event_target* get_event_target_from_pid(pid_t pid); long event_handler_add_pid(struct event_target*, pid_t, bool collect_exitcode); long event_handler_remove_pid(pid_t pid); noinline notrace void event_handler_fput(unsigned long, unsigned long, struct ftrace_ops*, struct pt_regs*); void event_handler_process_exit(struct task_struct *task); void event_handler_process_fork(struct task_struct *parent, struct task_struct *child); ================================================ FILE: kernel/event_queue.c ================================================ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "event_queue.h" #include "event_consumer.h" #include "shournal_kio.h" #include "kutil.h" #define __CONSUMER_JIFFY_OFFSET 200 /// "Consumes" the ringbuffer (writes tail) /// @return the number of consumed bytes (*not* events). static int __consume_close_events(struct event_target* event_target){ int bytes; int bytes_total; int head, tail; struct circ_buf* circ_buf = &event_target->event_consumer.circ_buf; const int cir_buf_size = event_target->event_consumer.circ_buf_size; struct close_event* e; unsigned long next_sched_jiffy; head = smp_load_acquire(&circ_buf->head); tail = READ_ONCE(circ_buf->tail); bytes_total = CIRC_CNT(head, tail, cir_buf_size); next_sched_jiffy = jiffies + msecs_to_jiffies(__CONSUMER_JIFFY_OFFSET); for(bytes=0; bytes < bytes_total; ){ e = (struct close_event*)&circ_buf->buf[tail]; tail = (tail + sizeof (struct close_event)) & (cir_buf_size - 1); bytes += sizeof(struct close_event); close_event_consume(event_target, e); if(time_is_before_jiffies(next_sched_jiffy)){ smp_store_release(&circ_buf->tail, tail); kutil_kthread_be_nice(); next_sched_jiffy = jiffies + msecs_to_jiffies(__CONSUMER_JIFFY_OFFSET); } } // maybe_todo: move into loop to avoid event-overflow? smp_store_release(&circ_buf->tail, tail); if( bytes_total > 0){ // bulk refcount-decrement.. int event_count = bytes_total/sizeof(struct close_event); event_target->consumed_event_count += event_count; if(kuref_sub_and_test(event_count, &event_target->_f_count)){ __event_target_put(event_target); } } return bytes_total; } /// For each event_target one thread is created, which /// consumes the events of the target's ringbuffer. int event_queue_consume_thread(void* data){ struct event_target* event_target = (struct event_target*)data; struct event_consumer* consumer = &event_target->event_consumer; struct kbuffered_file* target_file = event_target->file; uint64_t schedcount = 0; int sleep_counter = 0; up(&consumer->start_sema); event_consumer_thread_setup(event_target); // lost wake-up problem. set_current_state(TASK_INTERRUPTIBLE); // Calling wake_up_process is costly, so we want the producer // to do so rarely. Therefore: // First loop until nothing can be consumed. // Then sleep and check again for new events, before scheduling // without timeout. while(!kthread_should_stop()){ int consumed_bytes = __consume_close_events(event_target); if(consumed_bytes){ sleep_counter = 0; } else { // maybe a good time to flush? if(target_file->__pos > target_file->__bufsize/4){ event_consumer_flush_target_file_safe(event_target); // this might have taken a while, so.. kutil_kthread_be_nice(); continue; } if(sleep_counter > 2){ smp_store_mb(event_target->event_consumer.woken_up, false); // By checking again for events we allow a // harmless race in the consumer. __consume_close_events(event_target); schedcount++; schedule(); } else { schedule_timeout(1); sleep_counter++; } set_current_state(TASK_INTERRUPTIBLE); } } // consume final remaining events. Note in case *this* kthread // puts the final event_target-ref, we never get here. __consume_close_events(event_target); event_consumer_thread_cleanup(event_target); kutil_kthread_exit(NULL, 0); return 0; } ================================================ FILE: kernel/event_queue.h ================================================ /* File events are stored into a ringbuffer * and consumed in a per-event_target-thread. */ #pragma once #include "shournalk_global.h" #include #include "event_target.h" #include "event_consumer.h" int event_queue_consume_thread(void *data); /// Threadsafe enqueue the close event and wake up /// the consumer. This function consumes one event_target-reference (passes /// it to the consumer or puts it in case of overflow)! static inline void event_queue_add(struct event_target* event_target, struct file* file){ int head; int tail; int remaining_bytes; struct close_event* close_ev; struct event_consumer* consumer = &event_target->event_consumer; struct circ_buf* circ_buf = &consumer->circ_buf; // Be optimistic, that we have space in the ringbuf. We // *must* get the refs before enqueuing, otherwise // the consumer might put the last ones before us! mntget(file->f_path.mnt); dget(file->f_path.dentry); // No need to ihold(dentry->d_inode) // "as long as a counted reference is held to a dentry, // a non-NULL ->d_inode value will never be changed." // See also: kernel.org/doc/html/latest/filesystems/path-lookup.html spin_lock(&consumer->queue_lock); head = READ_ONCE(circ_buf->head); tail = READ_ONCE(circ_buf->tail); remaining_bytes = CIRC_SPACE(head ,tail ,consumer->circ_buf_size); if (unlikely(remaining_bytes < (int)sizeof (struct close_event))) { unsigned long long lostcount = READ_ONCE(event_target->lost_event_count); ++lostcount; // Event is lost, consumer was too slow. pr_devel("too many file events - skipping some (now lost: %lld)\n", lostcount); WRITE_ONCE(event_target->lost_event_count, lostcount); goto overflow_out; } close_ev = (struct close_event*)&circ_buf->buf[head]; close_ev->f_mode = file->f_mode; close_ev->path = file->f_path; // write new head *after* having written content: head = (head + sizeof (struct close_event)) & (consumer->circ_buf_size - 1); smp_store_release(&circ_buf->head, head); spin_unlock(&consumer->queue_lock); // We could simply call wake_up_process all the time, but this // slows down things significantly when many events occur. // Therefore try to only wake the consumer up, if // necessary. // The consumer sets woken_up to false and // afterwards checks for remaining events. Due to this race // it may (rarely) happen that we wake the consumer up // with nothing to do, *but* it can *never* happen // that we produce something which is never consumed. if(READ_ONCE(consumer->woken_up)) return; smp_store_mb(consumer->woken_up, true); wake_up_process(consumer->consume_task); return; overflow_out: spin_unlock(&consumer->queue_lock); dput(file->f_path.dentry); mntput(file->f_path.mnt); event_target_put(event_target); } ================================================ FILE: kernel/event_target.c ================================================ #include #include #include #include #include #include #include #include #include #include #include #include #include "event_target.h" #include "kutil.h" #include "shournal_kio.h" #include "shournalk_user.h" #include "xxhash_shournalk.h" static struct file* __get_check_target_file(int fd){ struct file* file; int error_nb = 0; file = fget(fd); if (!file){ pr_devel("fget failed on target file\n"); error_nb = -EBADF; goto err_cleanup_ret; } // We'll write to this file, so make sure we're allowed to if (!(file->f_mode & FMODE_WRITE)) { pr_debug("target file not writable\n"); error_nb = -EPERM; goto err_cleanup_ret; } if (! S_ISREG(file_inode(file)->i_mode) ) { pr_debug("target file not a regular file\n"); error_nb = -EBADFD; goto err_cleanup_ret; } return file; err_cleanup_ret: if(! IS_ERR_OR_NULL(file)){ fput(file); } return ERR_PTR(error_nb); } /// Get and reopen the pipe passed from userspace. /// That way we are independent of the user-space /// file status flags and always write with O_NONBLOCK /// (a malicious user-space process might otherwise /// block us indefinitely). static struct file* __get_check_pipe(int pipe_fd){ struct file* orig_pipe = NULL; struct file* new_pipe = NULL; long error_nb = 0; orig_pipe = fget(pipe_fd); if (!orig_pipe){ pr_devel("fget failed on pipe-fd\n"); error_nb = -EBADF; goto err_cleanup_ret; } if (! S_ISFIFO(file_inode(orig_pipe)->i_mode)) { pr_debug("passed fd not a FIFO\n"); error_nb = -ENOTTY; goto err_cleanup_ret; } if (!(orig_pipe->f_mode & FMODE_WRITE)) { pr_debug("passed FIFO descriptor is not the write end\n"); error_nb = -EPERM; goto err_cleanup_ret; } // With CONFIG_PROVE_LOCKING, kernel v5.10.191 a spurious "BUG: Invalid wait context" // occurred. Apparently, during dentry_open, a mutex is locked, thus previous code // calling dentry_open inside spin_lock(¤t->files->file_lock) was buggy. Let's // remember this by calling: might_sleep(); // reopen in nonblocking mode new_pipe = dentry_open(&orig_pipe->f_path, O_WRONLY | O_NONBLOCK, current->cred); if(!new_pipe){ error_nb = -EXDEV; goto err_cleanup_ret; } if(IS_ERR(new_pipe)){ error_nb = PTR_ERR(new_pipe); goto err_cleanup_ret; } fput(orig_pipe); return new_pipe; err_cleanup_ret: if(! IS_ERR_OR_NULL(orig_pipe)) fput(orig_pipe); if(! IS_ERR_OR_NULL(new_pipe)) fput(new_pipe); return ERR_PTR(error_nb); } static struct event_target* __event_target_create(struct file* target_file, struct file* pipe_w, const struct shournalk_mark_struct * mark_struct){ struct event_target* t = NULL; struct kbuffered_file* target_file_buffered = NULL; struct mem_cgroup* memcg = NULL; struct pid_namespace *pid_ns = task_active_pid_ns(current); char* path_tmp; long error = -ENOSYS; struct mm_struct * mm = NULL; if(!pid_ns){ WARN(1, "pid_ns == NULL"); return ERR_PTR(-ENXIO); } memcg = get_mem_cgroup_from_mm(current->mm); if(! memcg) { WARN(1, "memcg == NULL"); return ERR_PTR(-ENXIO); } mm = get_task_mm(current); if(mm) { mmgrab(mm); } else { pr_debug("mm == NULL"); // does no real harm though } t = kvzalloc(sizeof (struct event_target), SHOURNALK_GFP | __GFP_RETRY_MAYFAIL); if(!t) { error = -ENOMEM; goto error_out; } t->partial_hash.bufsize = PAGE_SIZE; t->partial_hash.buf = kmalloc(t->partial_hash.bufsize, SHOURNALK_GFP); if(! t->partial_hash.buf){ error = -ENOMEM; goto error_out; } t->partial_hash.xxh_state = kmalloc(sizeof (struct xxh64_state), SHOURNALK_GFP); if(!t->partial_hash.xxh_state){ error = -ENOMEM; goto error_out; } path_tmp = d_path(&target_file->f_path, t->file_init_path, sizeof (t->file_init_path)); if (IS_ERR(path_tmp)) { pr_debug("failed to resolve target file pathname\n"); error = PTR_ERR(path_tmp); goto error_out; } memmove(t->file_init_path, path_tmp, strlen(path_tmp) + 1); target_file_buffered = shournal_kio_from_file(target_file, TARGET_FILE_BUFSIZE); if(IS_ERR(target_file_buffered)){ error = PTR_ERR(target_file_buffered); goto error_out; } if((error = event_consumer_init(&t->event_consumer)) ) goto error_out; t->exit_code = SHOURNALK_INVALID_EXIT_CODE; t->pid_ns = pid_ns; t->user_ns = current_user_ns(); t->memcg = memcg; t->mm = mm; t->file = target_file_buffered; t->pipe_w = pipe_w; t->caller_tsk = current; atomic_set(&t->_written_to_user_pipe, 0); t->ERROR = false; kuref_set(&t->_f_count, 1); t->cred = current_cred(); t->lost_event_count = 0; t->stored_files_count = 0; t->partial_hash.chunksize = mark_struct->settings.hash_chunksize; t->partial_hash.max_count_of_reads = mark_struct->settings.hash_max_count_reads; mutex_init(&t->lock); t->settings = mark_struct->settings; kpathtree_init(&t->w_includes); kpathtree_init(&t->w_excludes); kpathtree_init(&t->r_includes); kpathtree_init(&t->r_excludes); kpathtree_init(&t->script_includes); kpathtree_init(&t->script_excludes); file_extensions_init(&t->script_ext); // do not fail from here on, otherwise references would need // to be dropped again get_pid_ns(pid_ns); get_cred(current_cred()); get_task_struct(current); get_user_ns(t->user_ns); return t; error_out: if(! IS_ERR_OR_NULL(target_file_buffered)) kfree(target_file_buffered); if(! IS_ERR_OR_NULL(t)) { kfree(t->partial_hash.xxh_state); kvfree(t->partial_hash.buf); kvfree(t); } if(memcg) mem_cgroup_put(memcg); if(mm) { mmdrop(mm); mmput(mm); } return ERR_PTR(error); } static void __event_target_free(struct event_target* t){ event_consumer_cleanup(&t->event_consumer); file_extensions_cleanup(&t->script_ext); kpathtree_cleanup(&t->w_includes); kpathtree_cleanup(&t->w_excludes); kpathtree_cleanup(&t->r_includes); kpathtree_cleanup(&t->r_excludes); kpathtree_cleanup(&t->script_includes); kpathtree_cleanup(&t->script_excludes); put_user_ns(t->user_ns); put_pid_ns(t->pid_ns); mem_cgroup_put(t->memcg); if(t->mm){ mmdrop(t->mm); mmput(t->mm); } put_task_struct(t->caller_tsk); put_cred(t->cred); shournal_kio_close(t->file); fput(t->pipe_w); kvfree(t->partial_hash.buf); kfree(t->partial_hash.xxh_state); kvfree(t); } static void __envent_target_destroy_work(struct work_struct *work){ struct event_target* t = container_of(to_rcu_work(work), struct event_target, destroy_rwork); __event_target_free(t); } ////////////////////////////// public //////////////////////////////////// // TODO: limit listeners per user to 128, like fanotify does. /// event_target's are found by the struct file /// where metadata about the file events are written to. /// That way, multiple pid's can be marked for observation. /// If no entry for the given target_fd exists, create a new one /// (and store it in the hash-table). Once all processes finished /// (or were unmarked again), notify userspace by writing into /// the passed pipe. struct event_target* event_target_create(const struct shournalk_mark_struct * mark_struct) { struct event_target* event_target = NULL; struct file* target_file = NULL; struct file* pipe_w = NULL; long error = -ENOSYS; target_file = __get_check_target_file(mark_struct->target_fd); if(IS_ERR(target_file)){ return (void*)target_file; } pipe_w = __get_check_pipe(mark_struct->pipe_fd); if(IS_ERR(pipe_w)){ error = PTR_ERR(pipe_w); goto err_put_unlock; } event_target = __event_target_create(target_file, pipe_w, mark_struct); if(IS_ERR(event_target) ){ error = PTR_ERR(event_target); goto err_put_unlock; } // maybe_todo: add caller-pid to threadname? if((error = event_consumer_thread_create(event_target, "shournalk_consumer"))){ goto err_put_unlock; } return event_target; err_put_unlock: if(! IS_ERR_OR_NULL(event_target)){ // ownership of pipe and target_file already transferred __event_target_free(event_target); } else { if(! IS_ERR_OR_NULL(pipe_w)) fput(pipe_w); if(! IS_ERR_OR_NULL(target_file)) fput(target_file); } return ERR_PTR(error); } // no events are registered before target is commited long event_target_commit(struct event_target* t){ WARN(! mutex_is_locked(&t->lock), "commit called without target lock\n"); barrier(); if( unlikely(event_target_is_commited(t))){ pr_debug("event target already commited"); return -EBUSY; } if(t->w_includes.n_paths){ WRITE_ONCE(t->w_enable, true); } if(t->r_includes.n_paths || t->script_includes.n_paths){ WRITE_ONCE(t->r_enable, true); } if( unlikely( ! event_target_is_commited(t))){ // nothing marked - user did not specify any include-paths pr_debug("cannot commit - no include-paths registered"); return -ENOTDIR; } return 0; } bool event_target_is_commited(const struct event_target* t){ return READ_ONCE(t->w_enable) || READ_ONCE(t->r_enable); } /// Final put void __event_target_put(struct event_target* event_target){ long user_ret; int pending_bytes; struct event_consumer* consumer = &event_target->event_consumer; struct circ_buf* circ_buf = &consumer->circ_buf; bool we_are_consume_thread; might_sleep(); #ifdef DEBUG if(event_target->__dbg_flags){ pr_info("event_target has dbg-flags set!\n"); dump_stack(); } #endif we_are_consume_thread = current == consumer->consume_task; if(we_are_consume_thread){ event_consumer_thread_cleanup(event_target); } else { // stopping the consumer thread also flushes the event buffer // one last time event_consumer_thread_stop(consumer); } pr_devel("Event processing done. Caller pid: %d - init target file path %s\n", event_target->caller_tsk->pid, event_target->file_init_path); pr_devel("consumed count: %lld, examined files: w: %lld, r: %lld\n", event_target->consumed_event_count, event_target->w_examined_count, event_target->r_examined_count); user_ret = shournal_kio_flush(event_target->file); if(user_ret >= 0){ user_ret = 0; } else { pr_debug("final target-file flush failed with %ld\n", user_ret); user_ret = -user_ret; } pending_bytes = CIRC_CNT(READ_ONCE(circ_buf->head), READ_ONCE(circ_buf->tail), event_target->event_consumer.circ_buf_size); kutil_WARN_ONCE_IFN_DBG(pending_bytes != 0, "pending bytes not 0 but %d", pending_bytes); event_target_write_result_to_user_ONCE(event_target, user_ret); // pr_info("dircache-hits: %lld, pathwrite_hits: %lld\n", event_target->_dircache_hits, // event_target->_pathwrite_hits); if(current_work() == NULL){ INIT_RCU_WORK(&event_target->destroy_rwork, __envent_target_destroy_work); // system_long_wq is flushed in event_handler_destructor, so do not change! queue_rcu_work(system_long_wq, &event_target->destroy_rwork); } else { __event_target_free(event_target); } if(we_are_consume_thread){ // we just released the final reference - that's it. kutil_kthread_exit(NULL, 0); } } void event_target_write_result_to_user_ONCE(struct event_target* event_target, long error_nb){ loff_t pos; ssize_t write_ret; struct shournalk_run_result result = { .error_nb = (int)error_nb, .w_event_count = event_target->w_event_count, .r_event_count = event_target->r_event_count, .lost_event_count = event_target->lost_event_count, .stored_event_count = event_target->stored_files_count, .selected_exitcode = event_target->exit_code }; if(atomic_xchg(&event_target->_written_to_user_pipe, 1)){ pr_devel("already written result (probably a previous error occurred"); return; } pos = 0; write_ret = kutil_kernel_write( event_target->pipe_w, &result, sizeof(result), &pos); if(write_ret != sizeof(result)){ pr_debug("Failed to write to user pipe - returned: %ld", write_ret); } } ================================================ FILE: kernel/event_target.h ================================================ #pragma once #include "shournalk_global.h" #include "xxhash_common.h" #include #include #include "kpathtree.h" #include "kfileextensions.h" #include "shournalk_user.h" #include "event_consumer.h" #include "kutil.h" // somewhat arbitrary, maybe raise? #define PART_HASH_MAX_CHUNKSIZE 4096*16 #define TARGET_FILE_BUFSIZE (1 << 15) struct cred; struct file; struct kbuffered_file; struct pid_namespace; struct user_namespace; struct shournalk_mark_struct; struct dentry; struct event_target { kuref_t _f_count; /* refcount - do not edit */ bool w_enable; /* record write events */ bool r_enable; /* record read events */ bool ERROR; /* lazy-release references in case of an error */ uint64_t lost_event_count; struct task_struct* exit_tsk; /* task for which to collect the exit code */ int exit_code; /* see exit_tsk */ struct event_consumer event_consumer; uint64_t consumed_event_count; struct file* pipe_w; /* write end of pipe. Id and bridge to user space group */ struct kbuffered_file* file; /* write events in here from kernel space */ const struct cred *cred; /* of the owner of the event target */ uint64_t w_event_count; /* # logged write events */ uint64_t w_dropped_count; /* # dropped exceeding max_event_count */ uint64_t w_deleted_count; /* # file was deleted */ uint64_t w_examined_count; /* events taken a closer look at */ uint64_t r_event_count; /* # logged read events */ uint64_t r_dropped_count; /* # dropped exceeding max_event_count */ uint64_t r_deleted_count; /* # file was deleted */ uint64_t r_examined_count; /* events taken a closer look at */ unsigned stored_files_count; struct pid_namespace *pid_ns; /* we only follow forks in same pid ns */ struct user_namespace* user_ns; /* of caller */ struct mem_cgroup* memcg; /* of caller */ struct mm_struct* mm; /* of caller */ struct task_struct* caller_tsk; /* the caller interested in events. */ struct shounalk_settings settings; struct partial_xxhash partial_hash; struct mutex lock; /* protects adding paths before committed */ atomic_t _written_to_user_pipe; /* we write to user pipe only once */ uint64_t _dircache_hits; uint64_t _pathwrite_hits; struct file_extensions script_ext; struct kpathtree w_includes; struct kpathtree w_excludes; struct kpathtree r_includes; struct kpathtree r_excludes; struct kpathtree script_includes; struct kpathtree script_excludes; char file_init_path[PATH_MAX]; struct rcu_work destroy_rwork; int __dbg_flags; }; struct event_target* event_target_create(const struct shournalk_mark_struct*); long event_target_commit(struct event_target*); bool event_target_is_commited(const struct event_target*); static inline __attribute__((__warn_unused_result__)) struct event_target* event_target_get(struct event_target* event){ if(likely(kuref_inc_not_zero(&event->_f_count))){ return event; } return NULL; } void __event_target_put(struct event_target* event_target); static inline void event_target_put(struct event_target* event_target){ #ifdef DEBUG might_sleep(); #endif if(unlikely( kuref_dec_and_test(&event_target->_f_count) )){ __event_target_put(event_target); } } void event_target_write_result_to_user_ONCE(struct event_target*, long error_nb); ================================================ FILE: kernel/hash_table_str.c ================================================ #include "hash_table_str.h" #include #include static const int HASH_GFP_FLAGS = SHOURNALK_GFP | __GFP_RETRY_MAYFAIL; /// creates a copy of the passed string struct hash_entry_str* hash_entry_str_create(const char* str, size_t str_len){ struct hash_entry_str* str_entry = kmalloc(sizeof (struct hash_entry_str), HASH_GFP_FLAGS); if(!str_entry){ return ERR_PTR(-ENOMEM); } str_entry->str = kmalloc(str_len, HASH_GFP_FLAGS); if(!str_entry->str){ kfree(str_entry); return ERR_PTR(-ENOMEM); } memcpy(str_entry->str, str, str_len); str_entry->str_len = str_len; return str_entry; } void hash_entry_str_free(struct hash_entry_str* entry){ kfree(entry->str); kfree(entry); } ================================================ FILE: kernel/hash_table_str.h ================================================ #pragma once #include "shournalk_global.h" #include "kutil.h" #include #include "xxhash_shournalk.h" static inline u32 __hash_table_str_do_hash(const char* path, size_t path_len) { return xxh32(path, path_len, 0); } struct hash_entry_str { char* str; size_t str_len; struct hlist_node node ; }; struct hash_entry_str* hash_entry_str_create(const char* str, size_t str_len); void hash_entry_str_free(struct hash_entry_str* entry); /// @param _obj_ must be passed as null, result is stored there if any #define hash_table_str_find(_name_, _obj_, _str_, _str_len_) \ do { \ struct hash_entry_str* ____tmp; \ u32 ____str_hash; \ ____str_hash = __hash_table_str_do_hash(_str_, _str_len_); \ kutil_WARN_DBG((_obj_) != NULL, "(_obj_) != NULL"); \ hash_for_each_possible(_name_, ____tmp, node, ____str_hash) \ if(____tmp->str_len == (_str_len_) && \ memcmp(____tmp->str, (_str_), (_str_len_)) == 0){ \ (_obj_) = ____tmp; \ break; \ } \ } while (0) #define hash_table_str_add(_name_, _obj_) \ hash_add(_name_, &(_obj_)->node, \ __hash_table_str_do_hash((_obj_)->str, (_obj_)->str_len)) #define hash_table_str_cleanup(_name_) \ do { \ u32 ____bucket; \ struct hash_entry_str* ____el; \ struct hlist_node *____temp_node; \ hash_for_each_safe((_name_), ____bucket, ____temp_node, ____el, node) { \ hash_del(&____el->node); \ hash_entry_str_free(____el); \ } \ } while (0) ================================================ FILE: kernel/kfileextensions.c ================================================ #include "kfileextensions.h" #include "hash_table_str.h" void file_extensions_init(struct file_extensions* extensions){ hash_init(extensions->table); extensions->n_ext = 0; } void file_extensions_cleanup(struct file_extensions* extensions){ hash_table_str_cleanup(extensions->table); } long file_extensions_add(struct file_extensions* extensions, const char* ext, size_t ext_len){ struct hash_entry_str* entry; // we store file extensions in the table without // leading dot. kutil_WARN_DBG(ext_len == 0, "ext_len == 0"); kutil_WARN_DBG(ext[0] == '.', "ext[0] == '.'"); kutil_WARN_DBG(strnstr(ext, "/", ext_len) != NULL, "strnstr(ext, /, ext_len)"); entry = hash_entry_str_create(ext, ext_len); if(IS_ERR(entry)){ return PTR_ERR(entry); } hash_table_str_add(extensions->table, entry); extensions->n_ext++; return 0; } long file_extensions_add_multiple(struct file_extensions* extensions, const char* ext_strs, size_t str_len){ long ret; const char* end = ext_strs + str_len; const char* s; for(s = ext_strs; s < end; s++) { if(*s == '/'){ size_t s_len = s - ext_strs; if(unlikely(s_len < 1)){ pr_debug("empty extension passed\n"); return -EINVAL; } if(unlikely((ret=file_extensions_add(extensions, ext_strs, s_len)))){ return ret; } ext_strs = s + 1; } } if(unlikely(s != ext_strs)){ pr_debug("extensions-string did not have trailing /\n"); return -EINVAL; } return 0; } /// Check if the file-extension (if any) of the given canonical path /// is contained within the struct file_extensions bool file_extensions_contain(struct file_extensions* extensions, const char* path, size_t path_len){ const char* str; const char* const end = path + path_len - 1; // empty paths are not allowed here kutil_WARN_ON_DBG(path_len == 0); if(*end == '.') return false; // loop backwards through the string until the first slash (no extension) // or dot is found for(str = end - 1; str >= path; str-- ){ if(*str == '/'){ // nothing found break; } if(*str == '.'){ struct hash_entry_str* entry = NULL; const char* ext_start = str + 1; size_t ext_len = end - ext_start + 1; hash_table_str_find(extensions->table, entry, ext_start, ext_len); return entry != NULL; } } return false; } ================================================ FILE: kernel/kfileextensions.h ================================================ #pragma once #include "shournalk_global.h" #include #define KFILEEXT_BITS 6 struct file_extensions { DECLARE_HASHTABLE(table, KFILEEXT_BITS); size_t n_ext; /* number of extensions within the table */ }; void file_extensions_init(struct file_extensions*); void file_extensions_cleanup(struct file_extensions*); long file_extensions_add(struct file_extensions* extensions, const char* ext, size_t ext_len); long file_extensions_add_multiple(struct file_extensions* extensions, const char* ext_strs, size_t str_len); bool file_extensions_contain(struct file_extensions* extensions, const char* path, size_t path_len); ================================================ FILE: kernel/kpathtree.c ================================================ #include "kpathtree.h" #include #include #include #include #include "kutil.h" #include "xxhash_shournalk.h" #include "hash_table_str.h" static int __compare_ints(const void *lhs, const void *rhs) { int lhs_integer = *(const int *)(lhs); int rhs_integer = *(const int *)(rhs); if (lhs_integer < rhs_integer) return -1; if (lhs_integer > rhs_integer) return 1; return 0; } static bool __path_len_exists(struct kpathtree* pathtree, int path_len){ int i; // maybe_todo: the path sizes are sorted, // so this coul be improved. However, KPATHTREE_MAX_SIZE // is small and adding a path not performance-critical.. for(i=0; i < pathtree->__n_path_sizes; i++){ if(pathtree->__path_sizes[i] == path_len){ return true; } } return false; } //////////////////////////////////////////////////////////////////// struct kpathtree* kpathtree_create(void){ struct kpathtree* pathtree = kzalloc(sizeof (struct kpathtree), SHOURNALK_GFP); if(!pathtree){ return ERR_PTR(-ENOMEM); } kpathtree_init(pathtree); return pathtree; } void kpathtree_free(struct kpathtree* pathtree){ kpathtree_cleanup(pathtree); kfree(pathtree); } /// pathtree must have been nulled before void kpathtree_init(struct kpathtree* pathtree){ WARN(pathtree->__is_init, "pathtree already initialized!"); pathtree->n_paths = 0; pathtree->__n_path_sizes = 0; hash_init(pathtree->path_table); mutex_init(&pathtree->lock); pathtree->__is_init = true; } void kpathtree_cleanup(struct kpathtree* pathtree){ if(! pathtree->__is_init){ WARN(1, "pathtree not initialized!"); return; } hash_table_str_cleanup(pathtree->path_table); pathtree->__is_init = false; } long kpathtree_add(struct kpathtree* pathtree, const char* path, int path_len){ struct hash_entry_str* entry; if(pathtree->n_paths >= KPATHTREE_MAX_SIZE){ return -ENOSPC; } entry = hash_entry_str_create(path, path_len); if(IS_ERR(entry)){ return PTR_ERR(entry); } hash_table_str_add(pathtree->path_table, entry); pathtree->n_paths++; if(! __path_len_exists(pathtree, path_len)){ pathtree->__path_sizes[pathtree->__n_path_sizes] = path_len; pathtree->__n_path_sizes++; sort(pathtree->__path_sizes, pathtree->__n_path_sizes, sizeof(int), &__compare_ints, NULL); } return 0; } bool kpathtree_is_subpath(struct kpathtree* pathtree, const char* path, int path_len, bool allow_equals){ int i; struct hash_entry_str* entry = NULL; if(pathtree->n_paths == 0){ return false; } if(pathtree->__path_sizes[0] == 1){ // We contain the root node (if input is valid - else we don't care). // As this function is only intended for file-paths, just: return true; } for(i=0; i < pathtree->__n_path_sizes; i++){ int s = pathtree->__path_sizes[i]; if(s < path_len){ // If we didn't have a / at the next position, we would cut the // path at a wrong position -> continue if(path[s] != '/'){ continue; } // A candiate path with the same size exists. // maybe_todo: incremental hash (but xxhash_update also // has overhead..) hash_table_str_find(pathtree->path_table, entry, path, (size_t)s); if(entry != NULL){ return true; } // keep going } else if(s > path_len) { // __path_sizes is ordered ascending -> the // next paths will be even longer: return false; } else { // s == path.size // The next m_orderedPathlength will be greater, so we can only // be a 'sub'-path, if allow_equals is true. if(allow_equals){ hash_table_str_find(pathtree->path_table, entry, path, (size_t)s); return entry != NULL; } return false; } } return false; } ================================================ FILE: kernel/kpathtree.h ================================================ #pragma once #include "shournalk_global.h" #include #include #define KPATHTREE_BITS 6 #define KPATHTREE_MAX_SIZE (1 << KPATHTREE_BITS) #define __KPATHTREE_INITIALIZER(treename) \ { .n_paths = 0 \ , .__n_path_sizes = 0 \ , .__is_init = true \ , .lock = __MUTEX_INITIALIZER(treename.lock) \ , .path_table = { [0 ... ((1 << (KPATHTREE_BITS)) - 1)] = HLIST_HEAD_INIT } } struct kpathtree { DECLARE_HASHTABLE(path_table, KPATHTREE_BITS); struct mutex lock; int n_paths; /* number of paths alreay added */ int __path_sizes[KPATHTREE_MAX_SIZE]; int __n_path_sizes; bool __is_init; }; struct kpathtree* kpathtree_create(void); void kpathtree_free(struct kpathtree* pathtree); void kpathtree_init(struct kpathtree* pathtree); void kpathtree_cleanup(struct kpathtree* pathtree); long kpathtree_add(struct kpathtree* pathtree, const char* path, int path_len); bool kpathtree_is_subpath(struct kpathtree* pathtree, const char* path, int path_len, bool allow_equals); ================================================ FILE: kernel/kutil.c ================================================ #include #include #include #include #include #include #include #include #include #include #include "kutil.h" #ifdef KVMALLOC_BACKPORT #include void *_kvmalloc_node_backport(size_t size, gfp_t flags, int node) { gfp_t kmalloc_flags = flags; void *ret; /* * vmalloc uses GFP_KERNEL for some internal allocations (e.g page tables) * so the given set of flags has to be compatible. */ WARN_ON_ONCE((flags & GFP_KERNEL) != GFP_KERNEL); /* * Make sure that larger requests are not too disruptive - no OOM * killer and no allocation failure warnings as we have a fallback */ if (size > PAGE_SIZE) kmalloc_flags |= __GFP_NORETRY | __GFP_NOWARN; ret = kmalloc_node(size, kmalloc_flags, node); /* * It doesn't really make sense to fallback to vmalloc for sub page * requests */ if (ret || size <= PAGE_SIZE) return ret; return __vmalloc(size, flags | __GFP_HIGHMEM, PAGE_KERNEL); } #endif /// Resolve the pathname of a regular, *not* deleted file. /// @param file The file to resolve the pathname of /// @param resolved_pathname a buffer of at least PATH_MAX size. char* resolve_reg_filepath(struct files_struct *files, struct file *file, char * buf){ // see also proc_fd_link and // https://stackoverflow.com/a/8250940/7015849 struct path *path; char* pathname; spin_lock(&files->file_lock); if (!file || !S_ISREG(file_inode(file)->i_mode) || file_inode(file)->i_nlink == 0) { spin_unlock(&files->file_lock); return NULL; } path = &file->f_path; path_get(path); spin_unlock(&files->file_lock); pathname = d_path(path, buf, PATH_MAX); path_put(path); if (IS_ERR(pathname)) { return NULL; } return pathname; } ssize_t kutil_kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos){ ssize_t ret; #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) ret = kernel_write(file, buf, count, pos); #else mm_segment_t fs_save; fs_save = get_fs(); set_fs(get_ds()); ret = vfs_write(file, buf, count, pos); set_fs(fs_save); #endif return ret; } ssize_t kutil_kernel_write_locked(struct file * file, const void *buf, size_t count) { ssize_t ret; mutex_lock(&file->f_pos_lock); ret = kutil_kernel_write(file, buf, count, &file->f_pos); mutex_unlock(&file->f_pos_lock); return ret; } ssize_t kutil_kernel_read_locked(struct file *file, void *buf, size_t count){ ssize_t ret; mutex_lock(&file->f_pos_lock); ret = kutil_kernel_read(file, buf, count, &file->f_pos); mutex_unlock(&file->f_pos_lock); return ret; } /// Try to read from file without disturbing the page cache. /// Eventually we should use mmap and something like /// MADV_FREE instead (zap_page_range)? /// Or re-suggest a similar approach Jens Axboe proposed in Dec 2019, e.g. /// [PATCH 1/5] fs: add read support for RWF_UNCACHED /// https://lwn.net/Articles/807519/ /// Imaginable is e.g. a preadv2-flag RWF_CACHEFRIENDLY which does not /// call mark_page_accessed() when a page is read. /// /// Anyhow, here we trick mm/filemap.c:filemap_read() (v5.12-rc5-3-g1e43c377a79f) /// into *not* calling mark_page_accessed by assigning ra->prev_pos to the current /// pos(ition): /// if (iocb->ki_pos >> PAGE_SHIFT != /// ra->prev_pos >> PAGE_SHIFT) /// mark_page_accessed(pvec.pages[0]); /// Note that this only works, if we read one page, that's why we /// call kernel_read multiple times, if necessary. /// /// Another approach might be to clear the page-reference-bit afterwards, /// but I'm not completely sure that this is legal..: /// struct address_space *mapping = file->f_mapping; /// struct page* page = find_get_entry(mapping, *pos / PAGE_SIZE); /// kernel_read(); /// ClearPageReferenced(page); ssize_t kutil_kernel_read_cachefriendly(struct file *file, void *buf, size_t count, loff_t *pos){ ssize_t read_size_total = 0; struct file_ra_state *ra = &file->f_ra; while(1){ ssize_t ret; ssize_t current_count; size_t count_to_page_end = round_up(*pos, PAGE_SIZE) - *pos; if(count_to_page_end == 0){ // At page-start. count_to_page_end = PAGE_SIZE; } current_count = min(count - read_size_total, count_to_page_end); ra->prev_pos = *pos; ret = kutil_kernel_read(file, buf, current_count, pos); if(unlikely(ret < 0)){ return ret; } read_size_total += ret; if(ret < current_count || read_size_total >= (ssize_t)count){ // EOF or everything read return read_size_total; } buf += ret; } } void kutil_take_name_snapshot(struct kutil_name_snapshot* snapshot, struct dentry *dentry){ // see lib/vsprintf.c:dentry_name and fs/dcache.c:dentry_cmp // -> apparently no locking is required for // reading d_name.name, because: // "dentry name is guaranteed to be properly terminated with a NUL byte". // However, linux/dache.h:take_dentry_name_snapshot // does locking (probably for consistent hash/len?). // (unsafe: snapshot->name = dentry->d_name;) const unsigned char *name; rcu_read_lock(); name = READ_ONCE(dentry->d_name.name); strncpy((char*)snapshot->inline_name, (const char*)name, sizeof (snapshot->inline_name) - 1); rcu_read_unlock(); // interface compatibility with struct name_snapshot from dcache.h snapshot->name.name = snapshot->inline_name; snapshot->name.len = (u32)strnlen((const char*)snapshot->name.name, sizeof (snapshot->inline_name)); if(unlikely(snapshot->name.len == sizeof (snapshot->inline_name))){ snapshot->inline_name[sizeof (snapshot->inline_name) - 1] = '\0'; pr_warn_once("Bug! Unterminated filename found: %s", snapshot->name.name); } } // interface compatibility with struct name_snapshot void kutil_release_name_snapshot(struct kutil_name_snapshot *name __attribute__ ((unused))) {} #ifdef kutil_BACKPORT_USE_MM #include // declare as weak to satisfy compiler. However, // one of use_mm or kthread_use_mm _must_ be defined (by kernel). void use_mm(struct mm_struct *mm) __attribute__((weak)); void unuse_mm(struct mm_struct *mm) __attribute__((weak)); void kthread_use_mm(struct mm_struct*) __attribute__((weak)); void kthread_unuse_mm(struct mm_struct*) __attribute__((weak)); void kutil_use_mm(struct mm_struct *mm) { if(use_mm) use_mm(mm); else if(kthread_use_mm) kthread_use_mm(mm); else pr_warn_once("kthread_use_mm and use_mm not defined - please report"); } void kutil_unuse_mm(struct mm_struct *mm) { if(unuse_mm) unuse_mm(mm); else if(kthread_unuse_mm) kthread_unuse_mm(mm); else pr_warn_once("kthread_unuse_mm and unuse_mm not defined - please report"); } #endif // kutil_BACKPORT_USE_MM // see commit cead18552660702a4a46f58e65188fe5f36e9dfe // declare as weak to satisfy compiler. However, // one of complete_and_exit and kthread_complete_and_exit // _must_ be defined (by kernel). void complete_and_exit(struct completion *comp, long code)__attribute__((weak)); void kthread_complete_and_exit(struct completion *comp, long code)__attribute__((weak)); void kutil_kthread_exit(struct completion *comp, long code){ if(complete_and_exit) complete_and_exit(comp, code); else if(kthread_complete_and_exit) kthread_complete_and_exit(comp, code); pr_err("Failed to stop kernel thread. Please unload this module " "immediatly and report this fatal bug."); } #ifdef RCU_WORK_BACKPORT static void rcu_work_rcufn(struct rcu_head *rcu) { struct rcu_work *rwork = container_of(rcu, struct rcu_work, rcu); queue_work(rwork->wq, &rwork->work); } bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork) { rwork->wq = wq; call_rcu(&rwork->rcu, rcu_work_rcufn); return true; } #endif // RCU_WORK_BACKPORT #ifdef GET_MEMCG_FROM_MM_BACKPORT struct mem_cgroup *_get_mem_cgroup_from_mm_backport(struct mm_struct *mm) { struct mem_cgroup *memcg = NULL; if (unlikely(!mm)) return NULL; rcu_read_lock(); do { memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); if (unlikely(!memcg)) break; } while (!css_tryget_online(&memcg->css)); rcu_read_unlock(); return memcg; } #endif // GET_MEMCG_FROM_MM_BACKPORT ================================================ FILE: kernel/kutil.h ================================================ #pragma once #include "shournalk_global.h" #include #include #include #include #include #include #include #include #include #include #include #include #if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)) #define mmgrab _mmgrab_backport static inline void _mmgrab_backport(struct mm_struct *mm) { atomic_inc(&mm->mm_count); } #else #include #endif struct pipe_inode_info; #ifdef DEBUG #define kutil_WARN_DBG WARN #define kutil_WARN_ON_DBG WARN_ON // if DEBUG always warn, else only once #define kutil_WARN_ONCE_IFN_DBG WARN #else #define kutil_WARN_DBG(condition, format...) #define kutil_WARN_ON_DBG(condition) #define kutil_WARN_ONCE_IFN_DBG WARN_ONCE #endif // see commit dcda9b04713c3f6ff0875652924844fae28286ea #if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0)) && \ !defined __GFP_RETRY_MAYFAIL #define __GFP_RETRY_MAYFAIL __GFP_REPEAT #endif // see commit a7c3e901a46ff54c016d040847eda598a9e3e653 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0)) #define KVMALLOC_BACKPORT #define kvmalloc_node _kvmalloc_node_backport void* _kvmalloc_node_backport(size_t size, gfp_t flags, int node); #define kvmalloc _kvmalloc_backport static inline void* _kvmalloc_backport(size_t size, gfp_t flags) { return kvmalloc_node(size, flags, NUMA_NO_NODE); } #define kvzalloc _kvzalloc_backport static inline void* _kvzalloc_backport(size_t size, gfp_t flags) { return kvmalloc(size, flags | __GFP_ZERO); } #endif // KVMALLOC_BACKPORT static inline int kutil_kthread_be_nice(void){ return cond_resched(); } char* resolve_reg_filepath(struct files_struct *files, struct file *file, char * buf); ssize_t kutil_kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos); ssize_t kutil_kernel_write_locked(struct file * file, const void *buf, size_t count); static inline ssize_t kutil_kernel_read(struct file *file, void *buf, size_t count, loff_t *pos){ ssize_t ret; #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) ret = kernel_read(file, buf, count, pos); #else mm_segment_t fs_save; fs_save = get_fs(); set_fs(get_ds()); ret = vfs_read(file, buf, count, pos); set_fs(fs_save); #endif return ret; } ssize_t kutil_kernel_read_locked(struct file *file, void *buf, size_t count); ssize_t kutil_kernel_read_cachefriendly(struct file *file, void *buf, size_t count, loff_t *pos); static inline unsigned long kutil_get_first_arg_from_reg(struct pt_regs *regs){ // see 3c88ee194c288205733d248b51f0aca516ff4940 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 20, 0)) && defined CONFIG_X86_64 return regs->di; #else return regs_get_kernel_argument(regs, 0); #endif } struct kutil_name_snapshot { struct qstr name; unsigned char inline_name[NAME_MAX + 1]; }; void kutil_take_name_snapshot(struct kutil_name_snapshot *, struct dentry *); void kutil_release_name_snapshot(struct kutil_name_snapshot*); // Replacement for the older memalloc_use_memcg, memalloc_unuse_memcg, // see also commit b87d8cefe43c7f22e8aa13919c1dfa2b4b4b4e01 // Actually (I think) it should be possible to call // set_active_memcg from KERNEL_VERSION(5, 10, 0) onwards // _but_ int_active_memcg is not exported as of 5.14. // current->active_memcg was introduced by // d46eb14b735b11927d4bdc2d1854c311af19de6d #if defined CONFIG_MEMCG && \ (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0)) static inline struct mem_cgroup * kutil_set_active_memcg(struct mem_cgroup *memcg) { struct mem_cgroup *old; if (unlikely(in_interrupt())) { kutil_WARN_ONCE_IFN_DBG(1, "Called in_interrupt..."); return NULL; } old = current->active_memcg; current->active_memcg = memcg; return old; } #else static inline struct mem_cgroup * kutil_set_active_memcg(struct mem_cgroup *memcg) { (void)(memcg); return NULL; } #endif #if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0)) #define kutil_BACKPORT_USE_MM // see commit f5678e7f2ac31c270334b936352f0ef2fe7dd2b3 void kutil_use_mm(struct mm_struct*); void kutil_unuse_mm(struct mm_struct*); // see commit 37c54f9bd48663f7657a9178fe08c47e4f5b537b #define USE_MM_SET_FS_OFF #else #define kutil_use_mm kthread_use_mm #define kutil_unuse_mm kthread_unuse_mm #endif void kutil_kthread_exit(struct completion *comp, long code); #if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) && \ !defined (INIT_RCU_WORK) #define RCU_WORK_BACKPORT #endif #ifdef RCU_WORK_BACKPORT struct rcu_work { struct work_struct work; struct rcu_head rcu; /* target workqueue ->rcu uses to queue ->work */ struct workqueue_struct *wq; }; static inline struct rcu_work *to_rcu_work(struct work_struct *work) { return container_of(work, struct rcu_work, work); } #define INIT_RCU_WORK(_work, _func) \ INIT_WORK(&(_work)->work, (_func)) bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork); #endif // RCU_WORK_BACKPORT #if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) #define GET_MEMCG_FROM_MM_BACKPORT #define get_mem_cgroup_from_mm _get_mem_cgroup_from_mm_backport struct mem_cgroup *_get_mem_cgroup_from_mm_backport(struct mm_struct *mm); #define mem_cgroup_put _mem_cgroup_put_backport static inline void _mem_cgroup_put_backport(struct mem_cgroup *memcg) { if (memcg) css_put(&memcg->css); } #endif #if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) #define vfs_fadvise _vfs_fadvise_dummy static inline int _vfs_fadvise_dummy(struct file *file, loff_t offset, loff_t len, int advice){ (void)file; (void)(offset); (void)(len); (void)(advice); return 0; } #endif // see commit 47291baa8ddfdae10663624ff0a15ab165952708 // and a6435940b62f81a1718bf2bd46a051379fc89b9d static inline int kutil_inode_permission(struct path* path, int mask){ #if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 12, 0)) && \ !defined FS_ALLOW_IDMAP return inode_permission(path->dentry->d_inode, mask); #else return path_permission(path, mask); #endif } // see commit 077c212f0344a #if (LINUX_VERSION_CODE > KERNEL_VERSION(6, 7, 0)) static inline time64_t kutil_get_mtime_sec(const struct inode *inode){ return inode_get_mtime_sec(inode); } #else static inline time64_t kutil_get_mtime_sec(const struct inode *inode){ return inode->i_mtime.tv_sec; } #endif // see f405df5de3170c00e5c54f8b7cf4766044a032ba #if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)) #define kuref_t atomic_t #define kuref_sub_and_test atomic_sub_and_test #define kuref_set atomic_set #define kuref_inc_not_zero atomic_inc_not_zero #define kuref_dec_and_test atomic_dec_and_test #else #define kuref_t refcount_t #define kuref_sub_and_test refcount_sub_and_test #define kuref_set refcount_set #define kuref_inc_not_zero refcount_inc_not_zero #define kuref_dec_and_test refcount_dec_and_test #endif ================================================ FILE: kernel/shournal_kio.c ================================================ #include "shournal_kio.h" #include #include #include #include "kutil.h" struct kbuffered_file* shournal_kio_from_file(struct file* file, size_t bufsize){ char* buf; struct kbuffered_file *buf_file; buf = kvmalloc(bufsize, SHOURNALK_GFP | __GFP_RETRY_MAYFAIL); if(! buf){ return ERR_PTR(-ENOMEM); } buf_file = kmalloc(sizeof(struct kbuffered_file), SHOURNALK_GFP); if(!buf_file){ kvfree(buf); return ERR_PTR(-ENOMEM); } buf_file->__file = file; buf_file->__buf = buf; buf_file->__pos = 0; buf_file->__bufsize = bufsize; return buf_file; } void shournal_kio_close(struct kbuffered_file* file){ shournal_kio_flush(file); fput(file->__file); kvfree(file->__buf); kfree(file); } ssize_t shournal_kio_write(struct kbuffered_file* file, const void *buf, size_t count){ ssize_t ret; if(count > file->__bufsize){ // flush and write as a whole if((ret = shournal_kio_flush(file)) < 0){ return ret; } return kutil_kernel_write_locked(file->__file, buf, count); } if(file->__pos + count > file->__bufsize){ if((ret = shournal_kio_flush(file)) < 0){ return ret; } } memcpy(&file->__buf[file->__pos] , buf, count); file->__pos += count; return count; } /// @return The number of bytes written or neg. errno ssize_t shournal_kio_flush(struct kbuffered_file* file){ ssize_t ret; if(file->__pos == 0){ return 0; } ret = kutil_kernel_write_locked( file->__file, file->__buf, file->__pos); if(ret < 0){ return ret; } if(ret != file->__pos){ // maybe_todo: mmove the rest of our buffer to the beginning? pr_devel("expected %ld bytes but wrote only %ld\n", file->__pos, ret); return -EIO; } file->__pos = 0; return ret; } ================================================ FILE: kernel/shournal_kio.h ================================================ #pragma once #include "shournalk_global.h" #include struct file; struct kbuffered_file { struct file* __file; char* __buf; ssize_t __pos; size_t __bufsize; }; struct kbuffered_file* shournal_kio_from_file(struct file* file, size_t bufsize); void shournal_kio_close(struct kbuffered_file* file); ssize_t shournal_kio_write(struct kbuffered_file* file, const void *buf, size_t count); ssize_t shournal_kio_flush(struct kbuffered_file* file); ================================================ FILE: kernel/shournalk_global.c ================================================ #include "shournalk_global.h" #include "kpathtree.h" struct kpathtree g_dummy_pathtree; long shournalk_global_constructor(void){ memset(&g_dummy_pathtree, 0, sizeof (struct kpathtree)); kpathtree_init(&g_dummy_pathtree); return 0; } void shournalk_global_destructor(void){ kpathtree_cleanup(&g_dummy_pathtree); } ================================================ FILE: kernel/shournalk_global.h ================================================ #pragma once // include module name in print_* messages: #ifdef pr_fmt #undef pr_fmt #endif #define pr_fmt(fmt) "shournalk %s(): " fmt, __func__ #include #define SHOURNALK_GFP GFP_KERNEL_ACCOUNT | __GFP_NOWARN extern struct kpathtree g_dummy_pathtree; long shournalk_global_constructor(void); void shournalk_global_destructor(void); ================================================ FILE: kernel/shournalk_main.c ================================================ #include "shournalk_global.h" #include #include #include #include MODULE_LICENSE("GPL"); MODULE_AUTHOR("Tycho Kirchner"); MODULE_DESCRIPTION("Trace and collect metadata (path, hash, etc.) about " "file close-events recursively for specific pid's"); MODULE_VERSION(SHOURNAL_VERSION); #include "event_handler.h" #include "event_handler.h" #include "shournalk_sysfs.h" #include "tracepoint_helper.h" #include "event_queue.h" #include "kutil.h" #include "shournalk_test.h" static int __init shournalk_init(void) { int ret; #ifdef DEBUG if(! run_tests()){ return -EHOSTDOWN; } #endif if((ret = (int)shournalk_global_constructor()) != 0){ return ret; } if((ret = event_handler_constructor()) != 0) goto error1; if ((ret = tracepoint_helper_constructor()) != 0) goto error2; if((ret = shournalk_sysfs_constructor()) != 0) goto error3; return 0; error3: tracepoint_helper_destructor(); error2: event_handler_destructor(); error1: shournalk_global_destructor(); return ret; } static void __exit shournalk_exit(void) { // Be very careful about the order here. shournalk_sysfs_destructor(); tracepoint_helper_destructor(); event_handler_destructor(); shournalk_global_destructor(); } module_init(shournalk_init) module_exit(shournalk_exit) ================================================ FILE: kernel/shournalk_sysfs.c ================================================ #include #include #include #include #include #include #include #include #include "shournalk_sysfs.h" #include "shournalk_user.h" #include "event_handler.h" #include "event_target.h" #include "kutil.h" // Use «default attribute groups». Kernel v5.1-rc3, // aa30f47cf666111f6bbfd15f290a27e8a7b9d854 added default attribute groups // while v5.18-rc1, cdb4f26a63c391317e335e6e683a614358e70aeb // dropped legacy support. So we switch somewhere in the middle. #if (LINUX_VERSION_CODE > KERNEL_VERSION(5, 8, 0)) #define SHOURNALK_USE_ATTR_GROUPS #endif struct shournal_obj { struct kobject kobj; }; #define to_shournal_obj(x) container_of(x, struct shournal_obj, kobj) struct shournal_attr { struct attribute attr; ssize_t (*show)(struct shournal_obj*, struct shournal_attr*, char*); ssize_t (*store)(struct shournal_obj*, struct shournal_attr*, const char*, size_t); }; #define to_shournal_attr(x) container_of(x, struct shournal_attr, attr) /// Entry point for all registered show-functions static ssize_t shournal_attr_show(struct kobject *kobj, struct attribute *attr, char *buf) { struct shournal_attr *attribute; struct shournal_obj *o; attribute = to_shournal_attr(attr); o = to_shournal_obj(kobj); if (!attribute->show) return -EIO; return attribute->show(o, attribute, buf); } /// Entry point for all registered store-functions static ssize_t shournal_attr_store(struct kobject *kobj, struct attribute *attr, const char *buf, size_t len) { struct shournal_attr *attribute; struct shournal_obj *o; attribute = to_shournal_attr(attr); o = to_shournal_obj(kobj); if (!attribute->store) return -EIO; return attribute->store(o, attribute, buf, len); } static struct sysfs_ops shournalk_ops = { .show = shournal_attr_show, .store = shournal_attr_store, }; static ssize_t __mark(struct shournal_obj*, struct shournal_attr*, const char*, size_t); static ssize_t __show_version(struct shournal_obj *o __attribute__ ((unused)), struct shournal_attr* attr __attribute__ ((unused)), char *buf) { return sprintf(buf, SHOURNAL_VERSION); } static struct shournal_attr attr_mark = __ATTR(mark, 0664, NULL, __mark); static struct shournal_attr attr_version = __ATTR(version, 0444, __show_version, NULL); static struct attribute *shournal_default_attrs[] = { &attr_mark.attr, &attr_version.attr, NULL, }; #ifdef SHOURNALK_USE_ATTR_GROUPS ATTRIBUTE_GROUPS(shournal_default); #endif static void shournal_obj_release(struct kobject *kobj){ struct shournal_obj* o; o = to_shournal_obj(kobj); kfree(o); } static struct kobj_type shournal_kobj_ktype = { .sysfs_ops = &shournalk_ops, #ifdef SHOURNALK_USE_ATTR_GROUPS .default_groups = shournal_default_groups, #else .default_attrs = (struct attribute **)&shournal_default_attrs, #endif .release = shournal_obj_release, }; static struct kset *shournal_kset; static struct shournal_obj *shournal_obj; /// Create kset and kobject and register our attribute function(s). /// kset *must* be created and set for kobject_uevent. /// See also: samples/kobject/kset-example.c int shournalk_sysfs_constructor(void){ int ret; shournal_kset = kset_create_and_add("shournalk_root", NULL, kernel_kobj); if (!shournal_kset){ return -ENOMEM; } shournal_obj = kzalloc(sizeof(*shournal_obj), GFP_KERNEL); if(! shournal_obj){ ret = -ENOMEM; goto err_shournal_obj_alloc; } shournal_obj->kobj.kset = shournal_kset; ret = kobject_init_and_add(&shournal_obj->kobj, &shournal_kobj_ktype, NULL, "%s", "shournalk_ctrl"); if (ret){ pr_err("kobject_init_and_add failed"); goto err_shournal_obj_add; } if((ret=kobject_uevent(&shournal_obj->kobj, KOBJ_ADD) )) { pr_warn("kobject_uevent failed\n"); goto err_shournal_obj_add; } return 0; err_shournal_obj_add: kobject_put(&shournal_obj->kobj); err_shournal_obj_alloc: kset_unregister(shournal_kset); return ret; } void shournalk_sysfs_destructor(void){ kobject_put(&shournal_obj->kobj); kset_unregister(shournal_kset); } ////////////////////////////////////////////////////////////////////// static long verify_hash_settings(struct shournalk_mark_struct * mark_struct){ if(mark_struct->settings.hash_max_count_reads == 0){ // hash is disabled. Be safe and also set chunksize to 0 mark_struct->settings.hash_chunksize = 0; return 0; } // maybe_todo: remove the upper limit: partial hashing can handle this // by digesting the max chunksize and *not* seeking afterwards. if(mark_struct->settings.hash_chunksize < 8 || mark_struct->settings.hash_chunksize > PART_HASH_MAX_CHUNKSIZE){ pr_debug("Invalid hashsettings. Chunksize must be:" " between 8 and %d bytes\n", PART_HASH_MAX_CHUNKSIZE); return -EINVAL; } if(mark_struct->settings.hash_max_count_reads < 1 || mark_struct->settings.hash_max_count_reads > 128){ pr_debug("Invalid hashsettings. Max count of reads " "must be between 1 and 128\n"); return -EINVAL; } return 0; } static long __handle_pid_add(struct shournalk_mark_struct* mark_struct){ pid_t pid = (pid_t)mark_struct->pid; struct event_target* event_target; // somewhat arbitrary limits const int STORE_MAX_SIZE = 1024*1024 * 2; const int STORE_MAX_FILECOUNT = 100; long ret; bool collect_exitcode = mark_struct->flags & SHOURNALK_MARK_COLLECT_EXITCODE; if((ret = verify_hash_settings(mark_struct))){ return ret; } if(mark_struct->settings.r_store_max_size > STORE_MAX_SIZE){ pr_debug("r_store_max_size > %d\n", STORE_MAX_SIZE); return -EINVAL; } if(mark_struct->settings.r_store_max_count_of_files > STORE_MAX_FILECOUNT){ pr_debug("r_store_max_count_of_files > %d\n", STORE_MAX_FILECOUNT); return -EINVAL; } if(mark_struct->settings.w_max_event_count == 0 || mark_struct->settings.r_max_event_count == 0){ pr_debug("max_event_count(s) must not be zero\n"); return -EINVAL; } event_target = event_target_create(mark_struct); if(IS_ERR(event_target)){ return PTR_ERR(event_target); } ret = event_handler_add_pid(event_target, pid, collect_exitcode); event_target_put(event_target); return ret; } static long __handle_pid_remove(const struct shournalk_mark_struct * mark_struct){ pid_t pid = (pid_t)mark_struct->pid; return event_handler_remove_pid(pid); } /// @return length of passed string or neg. error static ssize_t __copy_path_from_user(char* buf, const char* __user src){ long str_len = strncpy_from_user(buf, src, PATH_MAX); if(str_len <= 0){ pr_debug("strncpy_from_user returned %ld\n", str_len); if(str_len < 0) return str_len; return -EINVAL; } // we read something. While not a real sanity check, at least check for // leading / if(unlikely(buf[0] != '/')){ return -EINVAL; } return str_len; } /// If paths were not finalized yet, add param src to /// param pathtree static long __add_user_path(struct kpathtree* pathtree, const char* __user src){ long ret = 0; char* path_tmp; path_tmp = kzalloc(PATH_MAX, SHOURNALK_GFP); if(!path_tmp) return -ENOMEM; if((ret = __copy_path_from_user(path_tmp, src)) < 0){ goto out; } ret = kpathtree_add(pathtree, path_tmp, (int)ret); out: kfree(path_tmp); return ret; } static ssize_t __copy_file_extensions_from_user(char* buf, size_t buf_len, const char* __user src){ long str_len = strncpy_from_user(buf, src, buf_len); if(str_len <= 0){ pr_debug("strncpy_from_user returned %ld\n", str_len); if(str_len < 0) return str_len; return -EINVAL; } // shortes possible allowed extension, inluding trailing / // is e.g. o/ if(unlikely(str_len < 2)){ pr_debug("received extensions too short\n"); return -EINVAL; } return str_len; } /// If paths were not finalized yet, add param src to /// param pathtree static long __add_user_file_extensions(struct file_extensions* exts, const char* __user src){ long ret = 0; char* ext_tmp; ext_tmp = kzalloc(PAGE_SIZE, SHOURNALK_GFP); if(!ext_tmp) return -ENOMEM; if((ret = __copy_file_extensions_from_user(ext_tmp, PAGE_SIZE, src)) < 0){ goto out; } ret = file_extensions_add_multiple(exts, ext_tmp, (int)ret); out: kfree(ext_tmp); return ret; } static long __handle_mark_add(struct shournalk_mark_struct mark_struct){ long ret = -EINVAL; struct event_target* t; if(mark_struct.action == SHOURNALK_MARK_PID){ return __handle_pid_add(&mark_struct); } // for all other add-actions an existing event target is required t = get_event_target_from_pid((pid_t)mark_struct.pid); if(unlikely(IS_ERR(t))){ return PTR_ERR(t); } // locking applies to not committed targets only, so it is // no problem to lock (a bit) early mutex_lock(&t->lock); if(unlikely(event_target_is_commited(t))){ pr_debug("invalid action %d - " "event-target is already committed", mark_struct.action); ret = -EBUSY; goto unlock_put; } switch (mark_struct.action) { case SHOURNALK_MARK_W_INCL: ret = __add_user_path(&t->w_includes, mark_struct.data); break; case SHOURNALK_MARK_W_EXCL: ret = __add_user_path(&t->w_excludes, mark_struct.data); break; case SHOURNALK_MARK_R_INCL: ret = __add_user_path(&t->r_includes, mark_struct.data); break; case SHOURNALK_MARK_R_EXCL: ret = __add_user_path(&t->r_excludes, mark_struct.data); break; case SHOURNALK_MARK_SCRIPT_INCL: ret = __add_user_path(&t->script_includes, mark_struct.data); break; case SHOURNALK_MARK_SCRIPT_EXCL: ret = __add_user_path(&t->script_excludes, mark_struct.data); break; case SHOURNALK_MARK_SCRIPT_EXTS: ret = __add_user_file_extensions(&t->script_ext, mark_struct.data); break; default: ret = -EINVAL; } unlock_put: mutex_unlock(&t->lock); event_target_put(t); return ret; } static long __handle_mark_remove(struct shournalk_mark_struct mark_struct){ long ret; switch (mark_struct.action) { case SHOURNALK_MARK_PID: ret = __handle_pid_remove(&mark_struct); break; default: ret = -EINVAL; } return ret; } static long __handle_commit(struct shournalk_mark_struct mark_struct){ long ret = 0; struct event_target* event_target; event_target = get_event_target_from_pid((pid_t)mark_struct.pid); if(unlikely(IS_ERR(event_target))) { return PTR_ERR(event_target); } mutex_lock(&event_target->lock); if(likely(! event_target_is_commited(event_target))){ ret = event_target_commit(event_target); } else { pr_debug("__handle_commit: - " "event-target is already committed"); ret = -EBUSY; } mutex_unlock(&event_target->lock); event_target_put(event_target); return ret; } static ssize_t __mark(struct shournal_obj* obj __attribute__ ((unused)), struct shournal_attr* attr __attribute__ ((unused)), const char* buf, size_t count){ const struct shournalk_mark_struct * s_mark; ssize_t ret = 0; if(count != sizeof (struct shournalk_mark_struct)){ return -EILSEQ; } s_mark = (const struct shournalk_mark_struct*) buf; if(s_mark->flags & SHOURNALK_MARK_ADD) ret = __handle_mark_add(*s_mark); else if(s_mark->flags & SHOURNALK_MARK_REMOVE) ret = __handle_mark_remove(*s_mark); else if(s_mark->flags & SHOURNALK_MARK_COMMIT) ret = __handle_commit(*s_mark); else ret = -EINVAL; if(ret != 0){ WARN_ONCE(ret > 0, "pos. error received"); return ret; } return count; } ================================================ FILE: kernel/shournalk_sysfs.h ================================================ #pragma once #include "shournalk_global.h" int shournalk_sysfs_constructor(void); void shournalk_sysfs_destructor(void); ================================================ FILE: kernel/shournalk_test.c ================================================ #include "shournalk_test.h" #include "kpathtree.h" #include "hash_table_str.h" #define TEST_FAIL_ON(condition) ({ \ if(!!(condition)){ \ pr_warn("test fail at %s:%d/%s\n", __FILE__, __LINE__, __func__); \ goto test_err_out; \ } \ }) static bool test_kpathtree(void){ const char* p1 = "/home/user1"; const char* p2 = "/home/user2"; const char* p3 = "/mnt/d"; const char** current_ppath; const char* subpaths[] = { "/home/user1/a", "/home/user2/a", "/home/user1/abc/defg", "/home/user2/abc/defg__long_stuff.txt.tar.gz", "/mnt/d/1", "/mnt/d/2/abc/defg__long_stuff.txt.tar.gz", NULL }; const char* nosubpaths[] = { "/home/user3/a", "/home/user1", "/home/user2", "/home", "/", "/media/user1", "/mnt/data", "/mnt/e", "/mnt/defghijk/lmnop", NULL }; struct kpathtree* t = kpathtree_create(); TEST_FAIL_ON(IS_ERR(t)); // special case root node (all paths (except /) are subpaths TEST_FAIL_ON(kpathtree_add(t, "/", 1)); for(current_ppath = subpaths; *current_ppath != NULL; current_ppath++){ // pr_info("current path: %s\n", *current_ppath); TEST_FAIL_ON(! kpathtree_is_subpath(t, *current_ppath, (int)strlen(*current_ppath),0)); } kpathtree_free(t); t = kpathtree_create(); TEST_FAIL_ON(IS_ERR(t)); // before a path is added, all should fail: for(current_ppath = subpaths; *current_ppath != NULL; current_ppath++){ TEST_FAIL_ON(kpathtree_is_subpath(t, *current_ppath, (int)strlen(*current_ppath),0)); } for(current_ppath = nosubpaths; *current_ppath != NULL; current_ppath++){ TEST_FAIL_ON(kpathtree_is_subpath(t, *current_ppath, (int)strlen(*current_ppath),0)); } TEST_FAIL_ON(kpathtree_add(t, p1, (int)strlen(p1))); TEST_FAIL_ON(kpathtree_add(t, p2, (int)strlen(p2))); TEST_FAIL_ON(kpathtree_add(t, p3, (int)strlen(p3))); for(current_ppath = subpaths; *current_ppath != NULL; current_ppath++){ // pr_info("current path: %s\n", *current_ppath); TEST_FAIL_ON(! kpathtree_is_subpath(t, *current_ppath, (int)strlen(*current_ppath),0)); } for(current_ppath = nosubpaths; *current_ppath != NULL; current_ppath++){ // pr_info("current path: %s\n", *current_ppath); TEST_FAIL_ON(kpathtree_is_subpath(t, *current_ppath, (int)strlen(*current_ppath),0)); } kpathtree_free(t); t = kpathtree_create(); TEST_FAIL_ON(IS_ERR(t)); // test allow_equals TEST_FAIL_ON(kpathtree_add(t, p1, (int)strlen(p1))); TEST_FAIL_ON(kpathtree_is_subpath(t, p1,(int)strlen(p1),0)); TEST_FAIL_ON(!kpathtree_is_subpath(t, p1,(int)strlen(p1),1)); kpathtree_free(t); return true; test_err_out: if(! IS_ERR(t)) kpathtree_free(t); return false; } static bool test_hash_table_str(void){ struct hash_entry_str* orig_e = NULL; struct hash_entry_str* back_e = NULL; const char* str1 = "foobar"; DEFINE_HASHTABLE(hash_table, 6); orig_e = hash_entry_str_create(str1, strlen(str1)); TEST_FAIL_ON(IS_ERR_OR_NULL(orig_e)); hash_table_str_add(hash_table, orig_e); hash_table_str_find(hash_table, back_e, str1, strlen(str1)); TEST_FAIL_ON( back_e == NULL); hash_table_str_cleanup(hash_table); // was just freed orig_e = NULL; back_e = NULL; hash_table_str_find(hash_table, back_e, str1, strlen(str1)); TEST_FAIL_ON( back_e != NULL); return true; test_err_out: if(! IS_ERR_OR_NULL(orig_e)) hash_entry_str_free(orig_e); return false; } bool run_tests(void){ if(! test_kpathtree()) return false; if(! test_hash_table_str()) return false; pr_devel("Version %s - Tests successful!\n", SHOURNAL_VERSION); return true; } ================================================ FILE: kernel/shournalk_test.h ================================================ #pragma once #include "shournalk_global.h" #include bool run_tests(void); ================================================ FILE: kernel/shournalk_user.h ================================================ /* Common header for kernel and userspace to control * shournalk via a sysfs interface */ #pragma once #ifdef __KERNEL__ #include #else #include #include #endif // __KERNEL__ #ifdef __cplusplus extern "C" { #endif #define SHOURNALK_INVALID_EXIT_CODE -1 /* flags */ #define SHOURNALK_MARK_ADD 0x00000001 #define SHOURNALK_MARK_REMOVE 0x00000002 /* start collecting events after commit */ #define SHOURNALK_MARK_COMMIT 0x00000004 /* If this flag is set on MARK_PID, on process-tree end, return the exitcode of the given pid in the run_result (selected_exitcode). If the process did not end (e.g. marked by another target) it is SHOURNALK_INVALID_EXIT_CODE */ #define SHOURNALK_MARK_COLLECT_EXITCODE 0x00000008 /* actions */ #define SHOURNALK_MARK_PID 100 #define SHOURNALK_MARK_SCRIPT_INCL 111 /* include paths */ #define SHOURNALK_MARK_SCRIPT_EXCL 112 /* exclude paths */ #define SHOURNALK_MARK_SCRIPT_EXTS 113 /* file extensions */ #define SHOURNALK_MARK_R_INCL 120 #define SHOURNALK_MARK_R_EXCL 121 #define SHOURNALK_MARK_W_INCL 130 #define SHOURNALK_MARK_W_EXCL 131 struct shounalk_settings { bool w_exclude_hidden; uint64_t w_max_event_count; /* stop collecting after # written files */ bool r_only_writable; bool r_exclude_hidden; uint64_t r_max_event_count; /* stop collecting after # read files */ /* only store the content of a read file, if... */ bool r_store_only_writable; /* ...user also has write permission */ uint32_t r_store_max_size; /* ...file-size is less or equal to max_size */ uint16_t r_store_max_count_of_files; /* ...not already collected all desired files */ bool r_store_exclude_hidden; /* ...it is not hidden */ unsigned hash_max_count_reads; /* set to 0 to disable hash */ unsigned hash_chunksize; }; /// Mark specific paths of specific pid's (and their children) /// for observation struct shournalk_mark_struct { int pipe_fd; /* stats are written here after event processing finished */ int target_fd; /* close events are written to this binary file */ int flags; /* ADD, REMOVE, COMMIT */ int action; /* PID, SCRIPT_INCL/EXCL */ uint64_t pid; struct shounalk_settings settings; const void* data; }; /// Close events are written to a binary file. /// If 'bytes' is nonzero, the next N bytes are /// the complete file content. /// Followed by that is either the full filepath /// or filename (null-terminated). In case of a filename, the event occurred /// within the same directory as the previous event of the /// given type (O_RDONLY, O_WRONLY) struct shournalk_close_event { int flags; /* One of O_RDONLY, O_WRONLY, O_RDWR */ uint64_t mtime; /* as unix timestamp */ uint64_t size; uint64_t mode; uint64_t hash; bool hash_is_null; size_t bytes; /* read that many file-content-bytes next. */ /* filename as null-terminated cstring */ }; /// When the observation finishes, this struct is written to /// a pipe (created in user space) belonging to the notification /// group struct shournalk_run_result { int error_nb; uint64_t w_event_count; /* # of logged write-events */ uint64_t w_dropped_count; /* # dropped exceeding max_event_count */ uint64_t r_event_count; /* # of logged read-events */ uint64_t r_dropped_count; /* # dropped exceeding max_event_count */ uint32_t stored_event_count; /* number of (read) files in event target file */ uint64_t lost_event_count; /* if too many events occur, some may be dropped for performance reasons. */ int selected_exitcode; /* see SHOURNALK_MARK_COLLECT_EXITCODE */ }; #ifdef __cplusplus } #endif ================================================ FILE: kernel/tracepoint_helper.c ================================================ #include #include #include #include #include #include #include struct ftrace_ops; #include "tracepoint_helper.h" #include "event_handler.h" #include "event_queue.h" #include "event_consumer.h" #include "kutil.h" #if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0)) #define USE_LEGACY_TRACEPOINTS #endif /// Type of kernel traces used in here. enum {SHOURNALK_TP_TRACE, SHOURNALK_TP_FTRACE}; noinline notrace static void __probe_sched_process_fork(void *data __attribute__ ((unused)), struct task_struct *parent, struct task_struct *child) { event_handler_process_fork(parent, child); } noinline notrace static void __probe_process_exit(unsigned long ip __attribute__ ((unused)), unsigned long parent_ip __attribute__ ((unused)), struct ftrace_ops *op __attribute__ ((unused)), struct pt_regs *regs){ struct task_struct *task; task = (struct task_struct*)( kutil_get_first_arg_from_reg(tracepoint_helper_get_ftrace_regs(regs))); event_handler_process_exit(task); } /// Common structure to hold ftraces and tracepoints. struct trace_entry { char name[KSYM_NAME_LEN]; /* Don't use char* here! */ void *func; /* our own probe */ int tp_type; /* SHOURNALK_TP_TRACE, SHOURNALK_TP_FTRACE ... */ unsigned long flags; void *tracepoint; /* tracepoint in kernel */ bool init; struct ftrace_ops __ftrace_ops; }; // see commit a25d036d939a30623ff73ecad9c8b9116b02e823 : // ftrace: Reverse what the RECURSION flag means in the ftrace_ops #if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0)) #define SHOURNAL_FTRACE_RECURSION_SAFE FTRACE_OPS_FL_RECURSION_SAFE #else #define SHOURNAL_FTRACE_RECURSION_SAFE 0 #endif #define __DEFAULT_FTRACE_FLAGS \ FTRACE_OPS_FL_SAVE_REGS | SHOURNAL_FTRACE_RECURSION_SAFE static struct trace_entry interests[] = { {.name = "sched_process_fork", .func = (void*)__probe_sched_process_fork, .tp_type = SHOURNALK_TP_TRACE }, // Look at kernel/exit.c::do_exit : we need to run our exit // hook *after* the remaining open files were closed, // otherwise we would loose those events. Thus, the tracepoint // sched_process_exit is too early. sched_process_free on the other hand // runs too late, probably because free is called after parent processes // finished waiting but we want to allow waiting for them. // perf_event_exit_task would probably be ideal, but cannot be traced, but // cgroup_exit (or exit_notify) seems to be fine. Note that some of the // functions called within // do_exit are inlined (dependent on kernel version), thus cannot be ftraced. {.name = "cgroup_exit", .func = (void*)__probe_process_exit, .tp_type = SHOURNALK_TP_FTRACE, .flags = __DEFAULT_FTRACE_FLAGS}, // cannot use inline function fsnotify_close. // __close_fd is too highlevel (doesn't trigger on process exit, // CLO_EXEC files and (probably dup(2), etc.). // By using locks_remove_file instead of __fput, we avoid duplicate check // #ifdef FMODE_OPENED // unlikely(!(file->f_mode & FMODE_OPENED)) || // #endif {.name = "locks_remove_file", .func = (void*)event_handler_fput, .tp_type = SHOURNALK_TP_FTRACE, .flags = __DEFAULT_FTRACE_FLAGS}, }; static void init_ftrace_entry(struct trace_entry* e){ e->__ftrace_ops.func = e->func; e->__ftrace_ops.flags =e->flags; } #define FOR_EACH_INTEREST(i) \ for (i = 0; i < sizeof(interests) / sizeof(struct trace_entry); i++) static void init_interests(void){ size_t i; FOR_EACH_INTEREST(i) { interests[i].init = 0; if(interests[i].tp_type == SHOURNALK_TP_FTRACE){ init_ftrace_entry(&interests[i]); } } } #ifndef USE_LEGACY_TRACEPOINTS // Tracepoints are not exported. Look them up. static void lookup_tracepoints(struct tracepoint *tp, void *ignore __attribute__ ((unused)) ) { size_t i; FOR_EACH_INTEREST(i) { // pr_info("tracepoint: %s\n", tp->name); if (strcmp(interests[i].name, tp->name) == 0){ interests[i].tracepoint = tp; } } } #endif // USE_LEGACY_TRACEPOINTS static int __register_tracepoint(struct trace_entry * entry){ int ret; #ifndef USE_LEGACY_TRACEPOINTS if (entry->tracepoint == NULL) { return -ENXIO; } ret = tracepoint_probe_register(entry->tracepoint, entry->func, NULL); #else ret = tracepoint_probe_register(entry->name, entry->func, NULL); #endif entry->init = ret == 0; return ret; } static int __unregister_tracepoint(struct trace_entry * entry){ int ret; #ifndef USE_LEGACY_TRACEPOINTS ret = tracepoint_probe_unregister(entry->tracepoint, entry->func, NULL); #else ret = tracepoint_probe_unregister(entry->name, entry->func, NULL); #endif entry->init = 0; return ret; } static int __register_ftrace(struct trace_entry * entry){ int ret; if((ret = ftrace_set_filter(&entry->__ftrace_ops, entry->name, strlen(entry->name), 0)) < 0){ pr_warn("ftrace_set_filter %s failed\n", entry->name); return ret; } ret = register_ftrace_function(&entry->__ftrace_ops); entry->init = ret == 0; return ret; } static int __unregister_ftrace(struct trace_entry * entry){ int ret; ret = unregister_ftrace_function(&entry->__ftrace_ops); entry->init = 0; return ret; } static void cleanup(void) { size_t i; FOR_EACH_INTEREST(i) { int ret = 0; struct trace_entry* e = &interests[i]; if (! e->init) { continue; } switch (e->tp_type) { case SHOURNALK_TP_TRACE: ret = __unregister_tracepoint(e); break; case SHOURNALK_TP_FTRACE: ret = __unregister_ftrace(e); break; default: WARN_ON(1); break; } if(ret != 0){ pr_warn("failed to unregister trace %s\n", e->name); } } } int tracepoint_helper_constructor(void) { size_t i; int ret = 0; init_interests(); #ifndef USE_LEGACY_TRACEPOINTS for_each_kernel_tracepoint(lookup_tracepoints, NULL); #endif FOR_EACH_INTEREST(i) { struct trace_entry* e = &interests[i]; switch (e->tp_type) { case SHOURNALK_TP_TRACE: ret = __register_tracepoint(e);break; case SHOURNALK_TP_FTRACE: ret = __register_ftrace(e);break; default: WARN_ON(1); ret = -1; break; } if(ret != 0){ pr_warn("Failed to register trace %s\n", e->name); // Unload previously loaded cleanup(); return ret; } } return 0; } void tracepoint_helper_destructor(void) { cleanup(); } ================================================ FILE: kernel/tracepoint_helper.h ================================================ #pragma once #include "shournalk_global.h" #include #include struct pt_regs; int tracepoint_helper_constructor(void); void tracepoint_helper_destructor(void); static inline struct pt_regs* tracepoint_helper_get_ftrace_regs(struct pt_regs* regs){ #if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 0)) && \ ! defined arch_ftrace_get_regs return regs; #else // see commit d19ad0775dcd64b49eecf4fa79c17959ebfbd26b and // 02a474ca266a47ea8f4d5a11f4ffa120f83730ad // ftrace: Have the callbacks receive a struct ftrace_regs // instead of pt_regs struct ftrace_regs* fregs = (struct ftrace_regs*)regs; return ftrace_get_regs(fregs); #endif } ================================================ FILE: kernel/xxhash_shournalk.c ================================================ /* * xxHash - Extremely Fast Hash algorithm * Copyright (C) 2012-2016, Yann Collet. * * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following disclaimer * in the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License version 2 as published by the * Free Software Foundation. This program is dual-licensed; you may select * either version 2 of the GNU General Public License ("GPL") or BSD license * ("BSD"). * * You can contact the author at: * - xxHash homepage: http://cyan4973.github.io/xxHash/ * - xxHash source repository: https://github.com/Cyan4973/xxHash */ // See commit 5f60d5f "move asm/unaligned.h to linux/unaligned.h" #if __has_include() #include #else #include #endif #include #include #include #include #include #include "xxhash_shournalk.h" /*-************************************* * Macros **************************************/ #define xxh_rotl32(x, r) ((x << r) | (x >> (32 - r))) #define xxh_rotl64(x, r) ((x << r) | (x >> (64 - r))) #ifdef __LITTLE_ENDIAN # define XXH_CPU_LITTLE_ENDIAN 1 #else # define XXH_CPU_LITTLE_ENDIAN 0 #endif /*-************************************* * Constants **************************************/ static const uint32_t PRIME32_1 = 2654435761U; static const uint32_t PRIME32_2 = 2246822519U; static const uint32_t PRIME32_3 = 3266489917U; static const uint32_t PRIME32_4 = 668265263U; static const uint32_t PRIME32_5 = 374761393U; static const uint64_t PRIME64_1 = 11400714785074694791ULL; static const uint64_t PRIME64_2 = 14029467366897019727ULL; static const uint64_t PRIME64_3 = 1609587929392839161ULL; static const uint64_t PRIME64_4 = 9650029242287828579ULL; static const uint64_t PRIME64_5 = 2870177450012600261ULL; /*-************************** * Utils ***************************/ void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src) { memcpy(dst, src, sizeof(*dst)); } void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src) { memcpy(dst, src, sizeof(*dst)); } /*-*************************** * Simple Hash Functions ****************************/ static uint32_t xxh32_round(uint32_t seed, const uint32_t input) { seed += input * PRIME32_2; seed = xxh_rotl32(seed, 13); seed *= PRIME32_1; return seed; } uint32_t xxh32(const void *input, const size_t len, const uint32_t seed) { const uint8_t *p = (const uint8_t *)input; const uint8_t *b_end = p + len; uint32_t h32; if (len >= 16) { const uint8_t *const limit = b_end - 16; uint32_t v1 = seed + PRIME32_1 + PRIME32_2; uint32_t v2 = seed + PRIME32_2; uint32_t v3 = seed + 0; uint32_t v4 = seed - PRIME32_1; do { v1 = xxh32_round(v1, get_unaligned_le32(p)); p += 4; v2 = xxh32_round(v2, get_unaligned_le32(p)); p += 4; v3 = xxh32_round(v3, get_unaligned_le32(p)); p += 4; v4 = xxh32_round(v4, get_unaligned_le32(p)); p += 4; } while (p <= limit); h32 = xxh_rotl32(v1, 1) + xxh_rotl32(v2, 7) + xxh_rotl32(v3, 12) + xxh_rotl32(v4, 18); } else { h32 = seed + PRIME32_5; } h32 += (uint32_t)len; while (p + 4 <= b_end) { h32 += get_unaligned_le32(p) * PRIME32_3; h32 = xxh_rotl32(h32, 17) * PRIME32_4; p += 4; } while (p < b_end) { h32 += (*p) * PRIME32_5; h32 = xxh_rotl32(h32, 11) * PRIME32_1; p++; } h32 ^= h32 >> 15; h32 *= PRIME32_2; h32 ^= h32 >> 13; h32 *= PRIME32_3; h32 ^= h32 >> 16; return h32; } static uint64_t xxh64_round(uint64_t acc, const uint64_t input) { acc += input * PRIME64_2; acc = xxh_rotl64(acc, 31); acc *= PRIME64_1; return acc; } static uint64_t xxh64_merge_round(uint64_t acc, uint64_t val) { val = xxh64_round(0, val); acc ^= val; acc = acc * PRIME64_1 + PRIME64_4; return acc; } uint64_t xxh64(const void *input, const size_t len, const uint64_t seed) { const uint8_t *p = (const uint8_t *)input; const uint8_t *const b_end = p + len; uint64_t h64; if (len >= 32) { const uint8_t *const limit = b_end - 32; uint64_t v1 = seed + PRIME64_1 + PRIME64_2; uint64_t v2 = seed + PRIME64_2; uint64_t v3 = seed + 0; uint64_t v4 = seed - PRIME64_1; do { v1 = xxh64_round(v1, get_unaligned_le64(p)); p += 8; v2 = xxh64_round(v2, get_unaligned_le64(p)); p += 8; v3 = xxh64_round(v3, get_unaligned_le64(p)); p += 8; v4 = xxh64_round(v4, get_unaligned_le64(p)); p += 8; } while (p <= limit); h64 = xxh_rotl64(v1, 1) + xxh_rotl64(v2, 7) + xxh_rotl64(v3, 12) + xxh_rotl64(v4, 18); h64 = xxh64_merge_round(h64, v1); h64 = xxh64_merge_round(h64, v2); h64 = xxh64_merge_round(h64, v3); h64 = xxh64_merge_round(h64, v4); } else { h64 = seed + PRIME64_5; } h64 += (uint64_t)len; while (p + 8 <= b_end) { const uint64_t k1 = xxh64_round(0, get_unaligned_le64(p)); h64 ^= k1; h64 = xxh_rotl64(h64, 27) * PRIME64_1 + PRIME64_4; p += 8; } if (p + 4 <= b_end) { h64 ^= (uint64_t)(get_unaligned_le32(p)) * PRIME64_1; h64 = xxh_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; p += 4; } while (p < b_end) { h64 ^= (*p) * PRIME64_5; h64 = xxh_rotl64(h64, 11) * PRIME64_1; p++; } h64 ^= h64 >> 33; h64 *= PRIME64_2; h64 ^= h64 >> 29; h64 *= PRIME64_3; h64 ^= h64 >> 32; return h64; } /*-************************************************** * Advanced Hash Functions ***************************************************/ void xxh32_reset(struct xxh32_state *statePtr, const uint32_t seed) { /* use a local state for memcpy() to avoid strict-aliasing warnings */ struct xxh32_state state; memset(&state, 0, sizeof(state)); state.v1 = seed + PRIME32_1 + PRIME32_2; state.v2 = seed + PRIME32_2; state.v3 = seed + 0; state.v4 = seed - PRIME32_1; memcpy(statePtr, &state, sizeof(state)); } void xxh64_reset(struct xxh64_state *statePtr, const uint64_t seed) { /* use a local state for memcpy() to avoid strict-aliasing warnings */ struct xxh64_state state; memset(&state, 0, sizeof(state)); state.v1 = seed + PRIME64_1 + PRIME64_2; state.v2 = seed + PRIME64_2; state.v3 = seed + 0; state.v4 = seed - PRIME64_1; memcpy(statePtr, &state, sizeof(state)); } int xxh32_update(struct xxh32_state *state, const void *input, const size_t len) { const uint8_t *p = (const uint8_t *)input; const uint8_t *const b_end = p + len; if (input == NULL) return -EINVAL; state->total_len_32 += (uint32_t)len; state->large_len |= (len >= 16) | (state->total_len_32 >= 16); if (state->memsize + len < 16) { /* fill in tmp buffer */ memcpy((uint8_t *)(state->mem32) + state->memsize, input, len); state->memsize += (uint32_t)len; return 0; } if (state->memsize) { /* some data left from previous update */ const uint32_t *p32 = state->mem32; memcpy((uint8_t *)(state->mem32) + state->memsize, input, 16 - state->memsize); state->v1 = xxh32_round(state->v1, get_unaligned_le32(p32)); p32++; state->v2 = xxh32_round(state->v2, get_unaligned_le32(p32)); p32++; state->v3 = xxh32_round(state->v3, get_unaligned_le32(p32)); p32++; state->v4 = xxh32_round(state->v4, get_unaligned_le32(p32)); p32++; p += 16-state->memsize; state->memsize = 0; } if (p <= b_end - 16) { const uint8_t *const limit = b_end - 16; uint32_t v1 = state->v1; uint32_t v2 = state->v2; uint32_t v3 = state->v3; uint32_t v4 = state->v4; do { v1 = xxh32_round(v1, get_unaligned_le32(p)); p += 4; v2 = xxh32_round(v2, get_unaligned_le32(p)); p += 4; v3 = xxh32_round(v3, get_unaligned_le32(p)); p += 4; v4 = xxh32_round(v4, get_unaligned_le32(p)); p += 4; } while (p <= limit); state->v1 = v1; state->v2 = v2; state->v3 = v3; state->v4 = v4; } if (p < b_end) { memcpy(state->mem32, p, (size_t)(b_end-p)); state->memsize = (uint32_t)(b_end-p); } return 0; } uint32_t xxh32_digest(const struct xxh32_state *state) { const uint8_t *p = (const uint8_t *)state->mem32; const uint8_t *const b_end = (const uint8_t *)(state->mem32) + state->memsize; uint32_t h32; if (state->large_len) { h32 = xxh_rotl32(state->v1, 1) + xxh_rotl32(state->v2, 7) + xxh_rotl32(state->v3, 12) + xxh_rotl32(state->v4, 18); } else { h32 = state->v3 /* == seed */ + PRIME32_5; } h32 += state->total_len_32; while (p + 4 <= b_end) { h32 += get_unaligned_le32(p) * PRIME32_3; h32 = xxh_rotl32(h32, 17) * PRIME32_4; p += 4; } while (p < b_end) { h32 += (*p) * PRIME32_5; h32 = xxh_rotl32(h32, 11) * PRIME32_1; p++; } h32 ^= h32 >> 15; h32 *= PRIME32_2; h32 ^= h32 >> 13; h32 *= PRIME32_3; h32 ^= h32 >> 16; return h32; } int xxh64_update(struct xxh64_state *state, const void *input, const size_t len) { const uint8_t *p = (const uint8_t *)input; const uint8_t *const b_end = p + len; if (input == NULL) return -EINVAL; state->total_len += len; if (state->memsize + len < 32) { /* fill in tmp buffer */ memcpy(((uint8_t *)state->mem64) + state->memsize, input, len); state->memsize += (uint32_t)len; return 0; } if (state->memsize) { /* tmp buffer is full */ uint64_t *p64 = state->mem64; memcpy(((uint8_t *)p64) + state->memsize, input, 32 - state->memsize); state->v1 = xxh64_round(state->v1, get_unaligned_le64(p64)); p64++; state->v2 = xxh64_round(state->v2, get_unaligned_le64(p64)); p64++; state->v3 = xxh64_round(state->v3, get_unaligned_le64(p64)); p64++; state->v4 = xxh64_round(state->v4, get_unaligned_le64(p64)); p += 32 - state->memsize; state->memsize = 0; } if (p + 32 <= b_end) { const uint8_t *const limit = b_end - 32; uint64_t v1 = state->v1; uint64_t v2 = state->v2; uint64_t v3 = state->v3; uint64_t v4 = state->v4; do { v1 = xxh64_round(v1, get_unaligned_le64(p)); p += 8; v2 = xxh64_round(v2, get_unaligned_le64(p)); p += 8; v3 = xxh64_round(v3, get_unaligned_le64(p)); p += 8; v4 = xxh64_round(v4, get_unaligned_le64(p)); p += 8; } while (p <= limit); state->v1 = v1; state->v2 = v2; state->v3 = v3; state->v4 = v4; } if (p < b_end) { memcpy(state->mem64, p, (size_t)(b_end-p)); state->memsize = (uint32_t)(b_end - p); } return 0; } uint64_t xxh64_digest(const struct xxh64_state *state) { const uint8_t *p = (const uint8_t *)state->mem64; const uint8_t *const b_end = (const uint8_t *)state->mem64 + state->memsize; uint64_t h64; if (state->total_len >= 32) { const uint64_t v1 = state->v1; const uint64_t v2 = state->v2; const uint64_t v3 = state->v3; const uint64_t v4 = state->v4; h64 = xxh_rotl64(v1, 1) + xxh_rotl64(v2, 7) + xxh_rotl64(v3, 12) + xxh_rotl64(v4, 18); h64 = xxh64_merge_round(h64, v1); h64 = xxh64_merge_round(h64, v2); h64 = xxh64_merge_round(h64, v3); h64 = xxh64_merge_round(h64, v4); } else { h64 = state->v3 + PRIME64_5; } h64 += (uint64_t)state->total_len; while (p + 8 <= b_end) { const uint64_t k1 = xxh64_round(0, get_unaligned_le64(p)); h64 ^= k1; h64 = xxh_rotl64(h64, 27) * PRIME64_1 + PRIME64_4; p += 8; } if (p + 4 <= b_end) { h64 ^= (uint64_t)(get_unaligned_le32(p)) * PRIME64_1; h64 = xxh_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; p += 4; } while (p < b_end) { h64 ^= (*p) * PRIME64_5; h64 = xxh_rotl64(h64, 11) * PRIME64_1; p++; } h64 ^= h64 >> 33; h64 *= PRIME64_2; h64 ^= h64 >> 29; h64 *= PRIME64_3; h64 ^= h64 >> 32; return h64; } MODULE_LICENSE("Dual BSD/GPL"); MODULE_DESCRIPTION("xxHash"); ================================================ FILE: kernel/xxhash_shournalk.h ================================================ /* * xxHash - Extremely Fast Hash algorithm * Copyright (C) 2012-2016, Yann Collet. * * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following disclaimer * in the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License version 2 as published by the * Free Software Foundation. This program is dual-licensed; you may select * either version 2 of the GNU General Public License ("GPL") or BSD license * ("BSD"). * * You can contact the author at: * - xxHash homepage: http://cyan4973.github.io/xxHash/ * - xxHash source repository: https://github.com/Cyan4973/xxHash */ /* * Notice extracted from xxHash homepage: * * xxHash is an extremely fast Hash algorithm, running at RAM speed limits. * It also successfully passes all tests from the SMHasher suite. * * Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 * Duo @3GHz) * * Name Speed Q.Score Author * xxHash 5.4 GB/s 10 * CrapWow 3.2 GB/s 2 Andrew * MumurHash 3a 2.7 GB/s 10 Austin Appleby * SpookyHash 2.0 GB/s 10 Bob Jenkins * SBox 1.4 GB/s 9 Bret Mulvey * Lookup3 1.2 GB/s 9 Bob Jenkins * SuperFastHash 1.2 GB/s 1 Paul Hsieh * CityHash64 1.05 GB/s 10 Pike & Alakuijala * FNV 0.55 GB/s 5 Fowler, Noll, Vo * CRC32 0.43 GB/s 9 * MD5-32 0.33 GB/s 10 Ronald L. Rivest * SHA1-32 0.28 GB/s 10 * * Q.Score is a measure of quality of the hash function. * It depends on successfully passing SMHasher test set. * 10 is a perfect score. * * A 64-bits version, named xxh64 offers much better speed, * but for 64-bits applications only. * Name Speed on 64 bits Speed on 32 bits * xxh64 13.8 GB/s 1.9 GB/s * xxh32 6.8 GB/s 6.0 GB/s */ //////////////////////////// /* Tycho Kirchner, Oct 2020 - tychokirchner@mail.de * Old kernels do not provide xxhash. Further, at least on Debian Buster and * Ubuntu 18.04, xxhash's symbols cannot be found on module insert, although the symbols * are exported (and also do not appear in /proc/kallsyms, maybe because they are * nowhere used in the kernel?). Therefor, I copied the source from Linux 4.19.0 * in here, renamed it and removed the EXPORT_SYMBOL-calls. */ #pragma once #include /*-**************************** * Simple Hash Functions *****************************/ /** * xxh32() - calculate the 32-bit hash of the input with a given seed. * * @input: The data to hash. * @length: The length of the data to hash. * @seed: The seed can be used to alter the result predictably. * * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s * * Return: The 32-bit hash of the data. */ uint32_t xxh32(const void *input, size_t length, uint32_t seed); /** * xxh64() - calculate the 64-bit hash of the input with a given seed. * * @input: The data to hash. * @length: The length of the data to hash. * @seed: The seed can be used to alter the result predictably. * * This function runs 2x faster on 64-bit systems, but slower on 32-bit systems. * * Return: The 64-bit hash of the data. */ uint64_t xxh64(const void *input, size_t length, uint64_t seed); /*-**************************** * Streaming Hash Functions *****************************/ /* * These definitions are only meant to allow allocation of XXH state * statically, on stack, or in a struct for example. * Do not use members directly. */ /** * struct xxh32_state - private xxh32 state, do not use members directly */ struct xxh32_state { uint32_t total_len_32; uint32_t large_len; uint32_t v1; uint32_t v2; uint32_t v3; uint32_t v4; uint32_t mem32[4]; uint32_t memsize; }; /** * struct xxh32_state - private xxh64 state, do not use members directly */ struct xxh64_state { uint64_t total_len; uint64_t v1; uint64_t v2; uint64_t v3; uint64_t v4; uint64_t mem64[4]; uint32_t memsize; }; /** * xxh32_reset() - reset the xxh32 state to start a new hashing operation * * @state: The xxh32 state to reset. * @seed: Initialize the hash state with this seed. * * Call this function on any xxh32_state to prepare for a new hashing operation. */ void xxh32_reset(struct xxh32_state *state, uint32_t seed); /** * xxh32_update() - hash the data given and update the xxh32 state * * @state: The xxh32 state to update. * @input: The data to hash. * @length: The length of the data to hash. * * After calling xxh32_reset() call xxh32_update() as many times as necessary. * * Return: Zero on success, otherwise an error code. */ int xxh32_update(struct xxh32_state *state, const void *input, size_t length); /** * xxh32_digest() - produce the current xxh32 hash * * @state: Produce the current xxh32 hash of this state. * * A hash value can be produced at any time. It is still possible to continue * inserting input into the hash state after a call to xxh32_digest(), and * generate new hashes later on, by calling xxh32_digest() again. * * Return: The xxh32 hash stored in the state. */ uint32_t xxh32_digest(const struct xxh32_state *state); /** * xxh64_reset() - reset the xxh64 state to start a new hashing operation * * @state: The xxh64 state to reset. * @seed: Initialize the hash state with this seed. */ void xxh64_reset(struct xxh64_state *state, uint64_t seed); /** * xxh64_update() - hash the data given and update the xxh64 state * @state: The xxh64 state to update. * @input: The data to hash. * @length: The length of the data to hash. * * After calling xxh64_reset() call xxh64_update() as many times as necessary. * * Return: Zero on success, otherwise an error code. */ int xxh64_update(struct xxh64_state *state, const void *input, size_t length); /** * xxh64_digest() - produce the current xxh64 hash * * @state: Produce the current xxh64 hash of this state. * * A hash value can be produced at any time. It is still possible to continue * inserting input into the hash state after a call to xxh64_digest(), and * generate new hashes later on, by calling xxh64_digest() again. * * Return: The xxh64 hash stored in the state. */ uint64_t xxh64_digest(const struct xxh64_state *state); /*-************************** * Utils ***************************/ /** * xxh32_copy_state() - copy the source state into the destination state * * @src: The source xxh32 state. * @dst: The destination xxh32 state. */ void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src); /** * xxh64_copy_state() - copy the source state into the destination state * * @src: The source xxh64 state. * @dst: The destination xxh64 state. */ void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src); ================================================ FILE: shell-integration-scripts/CMakeLists.txt ================================================ configure_file( _source_me_generic.sh _source_me_generic.sh @ONLY) # Merge script files into _integration_ko.sh # Write to temporary so the real target only gets updated # if its content has changed set(integration_ko_tmp "${CMAKE_CURRENT_BINARY_DIR}/integration_ko.sh_tmp") file(WRITE ${integration_ko_tmp} "") foreach(f integration_main.sh.in util.sh integration_ko.sh) append_to_file(${integration_ko_tmp} ${f}) endforeach() # Copy the temporary file to the final location configure_file(${integration_ko_tmp} _integration_ko.sh @ONLY) # Merge script files into _integration_fan.sh set(integration_fan_tmp "${CMAKE_CURRENT_BINARY_DIR}/integration_fan.sh_tmp") file(WRITE ${integration_fan_tmp} "") foreach(f integration_main.sh.in util.sh integration_fan.sh) append_to_file(${integration_fan_tmp} ${f}) endforeach() configure_file(${integration_fan_tmp} _integration_fan.sh @ONLY) add_custom_target(target_SOURCE_SHELLSCRIPTS ALL COMMAND ${CMAKE_COMMAND} -E create_symlink _source_me_generic.sh SOURCE_ME.bash COMMAND ${CMAKE_COMMAND} -E create_symlink _source_me_generic.sh SOURCE_ME.zsh # keep those for backwards compatibility (SOURCE_ME.$shell should be used # in general as it automatically selects the correct backend) COMMAND ${CMAKE_COMMAND} -E create_symlink _integration_ko.sh integration_ko.bash COMMAND ${CMAKE_COMMAND} -E create_symlink _integration_ko.sh integration_ko.zsh COMMAND ${CMAKE_COMMAND} -E create_symlink _integration_fan.sh integration_fan.bash COMMAND ${CMAKE_COMMAND} -E create_symlink _integration_fan.sh integration_fan.zsh ) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/_source_me_generic.sh" "${CMAKE_CURRENT_BINARY_DIR}/SOURCE_ME.bash" "${CMAKE_CURRENT_BINARY_DIR}/SOURCE_ME.zsh" "${CMAKE_CURRENT_BINARY_DIR}/_integration_ko.sh" "${CMAKE_CURRENT_BINARY_DIR}/integration_ko.bash" "${CMAKE_CURRENT_BINARY_DIR}/integration_ko.zsh" "${CMAKE_CURRENT_BINARY_DIR}/_integration_fan.sh" "${CMAKE_CURRENT_BINARY_DIR}/integration_fan.bash" "${CMAKE_CURRENT_BINARY_DIR}/integration_fan.zsh" DESTINATION ${shournal_install_dir_script} ) ================================================ FILE: shell-integration-scripts/_source_me_generic.sh ================================================ # Select which shell-integration to be sourced based on config # and availability. # The backend is chosen in the following order: # • variable SHOURNAL_BACKEND # • config file at shournal's user cfg-dir # • config file at /etc... # Else default to the ko-backend or, if not found, the fanotify-backend. __shournal_eprint(){ >&2 printf "shournal-backend-selection: $*\n" } __shournal_cmd_exists(){ [ -x "$(command -v "$1")" ] } __shournal_select_backend(){ local scriptname="$1" local this_shell="$2" local backend_name_ko="integration_ko.$this_shell" local backend_name_fan="integration_fan.$this_shell" local backend_name_selected="" local backend_origin="UNKNOWN" if [ -n "${SHOURNAL_BACKEND+x}" ]; then backend_name_selected="$SHOURNAL_BACKEND" backend_origin="variable SHOURNAL_BACKEND" else for p in "$HOME/.config/shournal/backend" \ "/etc/shournal.d/backend"; do if test -f "$p"; then read -r backend_name_selected < "$p" backend_origin="$p" break fi done fi case "$backend_name_selected" in '') : ;; # use fallback below ko) backend_name_selected="$backend_name_ko";; fanotify) backend_name_selected="$backend_name_fan";; *) __shournal_eprint "Unsupported backend '$backend_name_selected' set in" \ "'$backend_origin'. Supported options: [fanotify, ko]." \ "Using defaults..." backend_name_selected="" ;; esac if [ -z "$backend_name_selected" ]; then if __shournal_cmd_exists 'shournal-run'; then backend_name_selected="$backend_name_ko" elif __shournal_cmd_exists 'shournal-run-fanotify'; then backend_name_selected="$backend_name_fan" else __shournal_eprint "Error: commands shournal-run and " \ "shournal-run-fanotify were not found in PATH." return 1 fi fi . "$(dirname -- "$scriptname")/$backend_name_selected" } __shournal_select_backend_return=0 if [ -n "${BASH_VERSION+x}" ]; then __shournal_select_backend "$BASH_SOURCE" bash || __shournal_select_backend_return=$? elif [ -n "${ZSH_VERSION+x}" ]; then # This has to be in global scope, $0 is different within function. __shournal_select_backend "$0" zsh || __shournal_select_backend_return=$? else __shournal_eprint "called from unsupported shell [currently only bash is supported]" __shournal_select_backend_return=1 fi unset __shournal_eprint unset __shournal_cmd_exists unset __shournal_select_backend return $__shournal_select_backend_return ================================================ FILE: shell-integration-scripts/integration_fan.sh ================================================ # shell-integration for shournal - fanotify backend. _shournal_run_backend='shournal-run-fanotify' _shournal_enable(){ local ret=0 if [ -n "${_shournal_is_running+x}" ] ; then _shournal_warn_on '! _libshournal_is_loaded' _shournal_debug "_shournal_enable: current session is already observed" return 0 fi if [ -n "${_shournal_shell_exec_string+x}" ]; then _shournal_handle_exec_string || return $? fi # This shell was _not_ invoked with the sh -c '...' option, # so clear our flag unconditionally: unset _shournal_parent_launched_us_noninteractive if ! _libshournal_is_loaded ; then if [ -n "${_shournal_enable_just_called+x}" ]; then _shournal_warn "Something went wrong during preloading of " \ "libshournal-shellwatch.so, the shell integration " \ "is _not_ enabled." unset _shournal_enable_just_called return 1 fi _shournal_interactive_exec_allowed || return $? _shournal_exec_ldpreloaded_shell # only get here on error return 1 fi _shournal_debug "_shournal_enable: about to enable..." unset _shournal_enable_just_called if [ -n "${LD_PRELOAD+x}" ] ; then # note: processes launched by this shell will not be observed by the preloaded library! LD_PRELOAD=${LD_PRELOAD//:$SHOURNAL_PATH_LIB_SHELL_INTEGRATION/} fi _libshournal_enable || return 1 _shournal_remove_prompts _shournal_add_prompts _shournal_is_running=true return 0 } _shournal_disable(){ local exitcode=$1 _shournal_debug "_shournal_disable: about to disable" \ "with exitcode $exitcode" if [ -n "${_shournal_shell_exec_string+x}" ]; then if [ -n "${_shournal_parent_launched_us_noninteractive+x}" ]; then _shournal_warn "SHOURNAL_DISABLE called, but the fanotify-based" \ "shell integration cannot be disabled for the" \ "'$_SHOURNAL_SHELL_NAME -c' invocation. Please use the" \ "kernel backend if this is a strong requirement." return 1 else _shournal_debug "_shournal_parent_launched_us_noninteractive is not" \ "set, likely disable was called without a" \ "prior enable." return 0 fi fi if [ -z "${_shournal_is_running+x}" ]; then _shournal_debug "_shournal_disable: not running" return 0 fi # In case we were called in a sequence, e.g. # $ (exit 123); SHOURNAL_DISABLE # we should cleanup first (otherwise the command was lost). # Be careful to avoid endless shutdown recursion here # ( _shournal_postexec may call SHOURNAL_DISABLE in case of erros) if [ -z "${_shournal_during_shutdown+x}" ]; then _shournal_during_shutdown=true _shournal_postexec "$exitcode" fi _libshournal_disable || _shournal_warn "_libshournal_disable failed" _shournal_remove_prompts unset _shournal_during_shutdown unset _shournal_is_running return 0 } # Check if it is _ok_ to call exec. # The shell is running interactive (no *_EXECUTION_STRING) # and _libshournal is not (pre-)loaded yet. To do so we # (currently) have to call exec. While doing so from # .shrc is fine, loading from interactive shell is ok (non-exported # variables are lost) we want to exclude the case where # commands are called in a row after SHOURNAL_ENABLE e.g. # $ SHOURNAL_ENABLE; important-command # because those are lost. _shournal_interactive_exec_allowed(){ local current_cmd current_cmd="$(_shournal_print_current_cmd)" current_cmd="$(_shournal_trim "$current_cmd")" if [ -z "$current_cmd" ]; then _shournal_debug "_shournal_enable exec granted, history " \ "is empty (likely called from .shrc)" return 0 fi if ! _shournal_endswith "$current_cmd" 'SHOURNAL_ENABLE'; then _shournal_warn "Command after SHOURNAL_ENABLE detected but" \ "we have to call exec first to enable the" \ "fanotify based shell integration." \ "Please ENABLE as separate command" \ "or switch backend. Command was '$current_cmd'" return 1 fi return 0 } _shournal_set_verbosity(){ local ret=0 # for libshournal-shellwatch.so export _SHOURNAL_LIB_SHELL_VERBOSITY="$1" if _libshournal_is_loaded; then _libshournal_update_verbosity || ret=$? fi return $ret } _shournal_print_versions(){ echo "shournal-run-fanotify: $(shournal-run-fanotify --version)" if _libshournal_is_loaded ; then _libshournal_print_version || _shournal_warn "printing version failed." else echo "To see the version of shournal's shell-integration " \ " (libshournal-shellwatch.so) please SHOURNAL_ENABLE first" fi return 0 } # If our libshournal-shellwatch.so is not loaded yet, do so # by "relaunching" this process (exec with same args) # within the "original" mount namespace. _shournal_exec_ldpreloaded_shell(){ declare -a args_array local cmd_path local IFS; unset IFS if ! [ -f "${SHOURNAL_PATH_LIB_SHELL_INTEGRATION-}" ]; then _shournal_error "Please provide a valid path for libshournal-shellwatch.so, e.g. " \ "export SHOURNAL_PATH_LIB_SHELL_INTEGRATION=" \ "'/usr/local/lib/shournal/libshournal-shellwatch.so'" return 1 fi cmd_path="$(readlink /proc/$$/exe)" while IFS= read -r -d '' line; do args_array+=("$line") done < /proc/$$/cmdline export _shournal_enable_just_called=true export LD_PRELOAD=${LD_PRELOAD-}":$SHOURNAL_PATH_LIB_SHELL_INTEGRATION" _shournal_debug "_shournal_exec_ldpreloaded_shell: calling preloaded " \ "$cmd_path ${args_array[@]:1}" # Relaunch the shell with shournals .so preloaded using the original arguments. exec shournal --verbosity "$_SHOURNAL_VERBOSITY" \ --backend-filename "$_shournal_run_backend" \ --msenter-orig-mountspace \ --exec-filename "$cmd_path" --exec -- "${args_array[@]}" # only get here on error return 1 } # Handle the sh -c '...' case. No PS0 (bash) or preexec_functions (zsh) _shournal_handle_exec_string(){ local cmd_trimmed local cmd_path declare -a args_array local IFS; unset IFS # In *this* backend we simply re-exec ourselves # with shournal and monitor the whole command sequence (SHOURNAL_DISABLE not # possible). Note that technically it would be possible to # move the original shell-binary somewhere else, execute a shournal-fake-shell # instead and invoke the original shell preloaded, so # we could allow for flexible enabling/disabling. This however would # be somewhat involved and possibly require one-time setup by the user. # On the other hand the ko-based shell integration does offer this # flexibility, so here we just ensure correctness: # We may only re-exec, if no command was executed before we were enabled, # otherwise it would be executed twice! Therefore we allow only two # cases: SHOURNAL_ENABLE as first command of the invocation e.g. # *sh -c 'SHOURNAL_ENABLE; ...' or when called from .shrc (which # must have been sourced before the command invocation starts. if [ -n "${_shournal_parent_launched_us_noninteractive+x}" ] ; then _shournal_debug "_shournal_handle_exec_string: we are (likely) already observed," \ "_shournal_parent_launched_us_noninteractive is set." \ "NOT performing re-exec" return 0 fi cmd_trimmed="$(_shournal_trim "$_shournal_shell_exec_string")" if ! _shournal_startswith "$cmd_trimmed" 'SHOURNAL_ENABLE' && ! _shournal_verbose_reexec_allowed; then _shournal_warn "we were enabled _during_ the $_SHOURNAL_SHELL_NAME -c" \ "invocation, however, the fanotify backend only supports" \ "enabling _before_ or at the beginning of the invocation." \ "Either switch to the kernel backend or" \ "put SHOURNAL_ENABLE into your shell's rc or at the" \ "beginning of the invocation. It may also be" \ "possible to directly call the command with 'shournal -e ...'" return 1 fi cmd_path="$(readlink /proc/$$/exe)" while IFS= read -r -d '' line; do args_array+=("$line") done < /proc/$$/cmdline _shournal_debug "_shournal_handle_exec_string: exec non-interactive" \ "$cmd_path ${args_array[@]}" # arg --fork: do not wait writing to the database. Otherwise # it blocks of course. _shournal_parent_launched_us_noninteractive=true exec \ shournal --backend-filename "$_shournal_run_backend" \ --verbosity "$_SHOURNAL_VERBOSITY" \ --exec-filename "$cmd_path" --exec --fork -- "${args_array[@]}" # only get here on error return 1 } _shournal_preexec_generic(){ _libshournal_prepare_cmd || : } _shournal_postexec_generic(){ local cmd_str="$1" local exitcode="$2" _shournal_debug "_shournal_postexec" # user might modify history settings at any time, so better be safe: if ! _shournal_verbose_history_check; then _shournal_warn "history settings were modified after the shell " \ "integration was turned on. " \ "Turning the shell integration off..." # Be careful to avoid endless shutdown recursion here. if [ -z "${_shournal_during_shutdown+x}" ]; then _shournal_during_shutdown=true SHOURNAL_DISABLE fi return 1 fi export _SHOURNAL_LAST_COMMAND="$cmd_str" export _SHOURNAL_LAST_RETURN_VALUE=$exitcode # cleanup may fail regularily, this function may also be executed # when no command ran before (e.g. when hitting enter on a blank line in bash) _libshournal_cleanup_cmd || : unset _SHOURNAL_LAST_COMMAND unset _SHOURNAL_LAST_RETURN_VALUE return 0 } # «Send» messages to libshournal-shellwatch.so by exporting a # variable and triggering a dummy-close event. _shournal_trigger_update(){ local desired_state="$1" local trigger_response local ret export _LIBSHOURNAL_TRIGGER="$desired_state"; export _SHOURNAL_SHELL_PID=$$ # Note: our .so detects this special (non-)filename and # writes its response to an unnamed tmp file. ret=0 read -d '' trigger_response < '_///shournal_trigger_response///_' || ret=$? unset _LIBSHOURNAL_TRIGGER unset _SHOURNAL_SHELL_PID if [ $ret -ne 0 ]; then _shournal_debug "_shournal_trigger_update: failed to read " \ "trigger_response" return 1 fi [ "$trigger_response" = ok ] && return 0 || return 1 } _libshournal_enable(){ _shournal_trigger_update 0 } _libshournal_disable(){ _shournal_trigger_update 1 } _libshournal_prepare_cmd(){ _shournal_trigger_update 2 } _libshournal_cleanup_cmd(){ _shournal_trigger_update 3 } _libshournal_print_version(){ _shournal_trigger_update 4 } _libshournal_update_verbosity(){ _shournal_trigger_update 5 } _libshournal_is_loaded(){ local word_arr local pathname local IFS; unset IFS if [ -n "${ZSH_VERSION+x}" ]; then setopt LOCAL_OPTIONS setopt sh_word_split fi while IFS="" read -r row || [ -n "$row" ]; do word_arr=($row) # see man 5 proc section /proc/[pid]/maps # word_arr[5] usually contains the pathname but may also be blank. [ "${#word_arr[@]}" -lt 6 ] && continue # portable array index access (zsh is one-based) pathname="${word_arr[@]:5:1}" _shournal_endswith "$pathname" libshournal-shellwatch.so && return 0 done < "/proc/$$/maps" return 1 } # The following non-portable, shell specific functions _must_ be set # for each supported shell: # _shournal_add_prompts # _shournal_remove_prompts # _shournal_postexec # _shournal_verbose_reexec_allowed case "$_SHOURNAL_SHELL_NAME" in 'bash') export _LIBSHOURNAL_SEQ_COUNTER=1 _shournal_ps0='${_SHOURNAL_SHELL_NAME:((_LIBSHOURNAL_SEQ_COUNTER++)):0}$(:)' _shournal_add_prompts(){ [ -z "${PS0+x}" ] && PS0='' [ -z "${PROMPT_COMMAND+x}" ] && PROMPT_COMMAND='' # Allright, what happens here? We use _SHOURNAL_SHELL_NAME as a dummy # variable in order to increment _LIBSHOURNAL_SEQ_COUNTER without printing # anything. Then we fork to notify libshournal-shellwatch.so that # we're about to execute a command. PS0="$PS0""$_shournal_ps0" PROMPT_COMMAND=$'_shournal_postexec\n'"$PROMPT_COMMAND" # no _shournal_preexec for bash, see below ... return 0 } _shournal_remove_prompts(){ [ -n "${PS0+x}" ] && PS0=${PS0//"$_shournal_ps0"/} [ -n "${PROMPT_COMMAND+x}" ] && PROMPT_COMMAND=${PROMPT_COMMAND//_shournal_postexec$'\n'/} return 0 } ## _____ End of must-override functions and variables _____ ## # _shournal_preexec(){ # For bash preexec is not implemented here but in # in an interplay of above PS0 and libshournal-shellwatch.so. # } _shournal_postexec(){ local exitcode=$? local cmd_str [ -n "${1+x}" ] && exitcode="$1" _shournal_get_current_cmd_bash cmd_str _shournal_postexec_generic "$cmd_str" "$exitcode" return $exitcode } ;; # END_OF bash _______________________________________________________ 'zsh') _shournal_add_prompts(){ preexec_functions+=(_shournal_preexec) precmd_functions+=(_shournal_postexec) return 0 } _shournal_remove_prompts(){ unset _shournal_zsh_last_cmd preexec_functions[$preexec_functions[(i)_shournal_preexec]]=() precmd_functions[$precmd_functions[(i)_shournal_postexec]]=() return 0 } ## _____ End of must-override functions and variables _____ ## _shournal_preexec(){ # maybe_todo: use $2 or $3 for expanded aliases instead of $1 _shournal_zsh_last_cmd="$1" _shournal_preexec_generic return 0 } _shournal_postexec(){ local exitcode=$? [ -n "${1+x}" ] && exitcode="$1" _shournal_postexec_generic "$_shournal_zsh_last_cmd" $exitcode return 0 } ;; # END_OF zsh ________________________________________________________ *) echo "shournal shell integration: something is seriously wrong, " \ "_SHOURNAL_SHELL_NAME is not correctly setup" >&2 return 1 ;; esac if [ -n "${_shournal_enable_just_called+x}" ] ; then # A parent process has called SHOURNAL_ENABLE and exec'd itself # again with the same arguments and our libshournal-shellwatch.so # preloaded. Let the tracking begin ... if ! _libshournal_is_loaded ; then _shournal_error "Although _'shournal_enable_just_called' is set, " \ "libshournal-shellwatch.so seems " \ "to be not loaded (bug?)." unset _shournal_enable_just_called return 1 fi _shournal_enable fi ================================================ FILE: shell-integration-scripts/integration_ko.sh ================================================ # shell-integration for shournal - kernel-module backend. _shournal_run_backend='shournal-run' _shournal_enable(){ local ret=0 [ -n "${_shournal_int_traps+x}" ] || _shournal_int_traps=() _shournal_trap_push '' INT || return _shournal_do_enable || ret=$? _shournal_trap_pop || : return $ret } _shournal_do_enable(){ local tmpdir local ret=0 local cmd_str if [ -n "${_shournal_is_running+x}" ] ; then # maybe_todo: check that our prompts are still there. _shournal_debug "_shournal_enable: current session is already observed" return 0 fi if ! "$_shournal_run_backend" --shournalk-is-loaded; then _shournal_warn "Cannot enable the shell-integration -" \ "the required kernel module is not loaded." return 1 fi if [ -e '/dev/shm' ]; then tmpdir='/dev/shm' else [ -n "${TMPDIR+x}" ] && tmpdir="$TMPDIR" || tmpdir=/tmp fi _shournal_fifo_basepath="$tmpdir/shournal-fifo-$USER-$$" # If an observed shell calls "exec bash" we end # up with an already existing fifo. # In almost all cases this is no problem, as the first time shournal-run is called # the pid is claimed and the old shournal-run process exits. However, in case of # sequence count 1 the previous and current fifo-paths collide, so just clean up # in any case. _shournal_detach_this_pid 0 if [ -n "${_shournal_shell_exec_string+x}" ]; then # invoked via sh -c cmd_str="" # FIXME: also collect /proc/$$/exe ? while IFS= read -r -d '' line; do [ -z "$cmd_str" ] && cmd_str="$line" || cmd_str="$cmd_str $line" done < /proc/$$/cmdline _shournal_preexec_generic 1 "$cmd_str" || return $? _shournal_is_running=true else SHOURNAL_SESSION_ID="$(shournal-run --make-session-uuid)" || return $? export SHOURNAL_SESSION_ID export SHOURNAL_CMD_COUNTER=0 # Usually removing prompts should not be necessary here, # however, if a user exports PS0/PROMPT_COMMAND # and starts a new bash-session, we need to get rid of the existing commands. _shournal_remove_prompts || return $? _shournal_add_prompts || return $? _shournal_is_running=true fi return 0 } _shournal_disable(){ # Note that there are at least three cases how we can get here: # • User-invoked SHOURNAL_DISABLE # • Error during pre/postexec # • exit trap local exitcode="$1" _shournal_debug "_shournal_disable: about to disable" \ "with exitcode $exitcode" if [ -z "${_shournal_is_running+x}" ]; then _shournal_debug "_shournal_disable: not running" return 0 fi _shournal_trap_push '' INT || : _shournal_remove_prompts _shournal_detach_this_pid "$exitcode" # Don't unset _shournal_int_traps here - we may have been called nested! unset _shournal_is_running _shournal_preexec_ret \ _shournal_fifo_basepath _shournal_trap_pop || : return 0 } _shournal_set_verbosity(){ : } _shournal_print_versions(){ echo "shournal-run: $(shournal-run --version)" } _shournal_send_msg(){ # send json string to last started shournal. # for the different message types (msgType), see enum FIFO_MSG in c++. local fifofd="$1" local msg_type="$2" local msg_data="$3" local ret=0 # simple json string type-data: { "msgType":0, "data":"stuff" } local full_msg="{\"msgType\":$msg_type,\"data\":\"$msg_data\"}" _shournal_debug "_shournal_send_msg: sending message: $full_msg" echo "$full_msg" >&$fifofd || ret=$? if [ $ret -ne 0 ]; then _shournal_error "_shournal_send_msg: failed to write to fifo-FD $fifofd: $ret" return $ret fi return 0 } _shournal_send_ret_val(){ _shournal_send_msg "$1" 0 "$2" } _shournal_send_unmark_pid(){ _shournal_send_msg "$1" 1 "$2" } _shournal_run_finalize(){ local fifopath="$1" local exitcode="$2" local _shournal_fifofd # Open the FIFO RDWR to be protected against deadlocks which may occur, e.g., # if shournal dies after having set up the FIFO. Note that shournal will ignore # this event, because a FIFO is not a regular file. if ! { exec {_shournal_fifofd}<>"$fifopath"; } 2>/dev/null; then _shournal_debug "_shournal_run_finalize: opening fifopath \"$fifopath\" failed." return 0 fi _shournal_send_ret_val $_shournal_fifofd $exitcode _shournal_send_unmark_pid $_shournal_fifofd $$ exec {_shournal_fifofd}<&- # If everything goes well, this rm is not needed, as shournal performs it for us. # However, if shournal died in the background, we have created the now REGULAR file at # $fifopath ourselves. So KISS and delete always. rm "$fifopath" 2>/dev/null } # Find a fifo (if any) that was created by this shell previously and # instruct the belonging shournal-run process to stop # observing this pid. _shournal_detach_this_pid(){ local exitcode="$1" local fifopath local ret=0 if [ "$_SHOURNAL_SHELL_NAME" = 'zsh' ]; then # supress nomatch error messages (and aborts) setopt LOCAL_OPTIONS unsetopt nomatch fi # use globbing to ignore the sequence number set -- "$_shournal_fifo_basepath"* # Note: in case of no results $1 is _not_ empty, so check # for existence. if [ -e "$1" ] ; then if [ $# -eq 1 ]; then fifopath=${1%%$'\n'*} # should not be necessary _shournal_debug "_shournal_detach_this_pid at $fifopath" _shournal_run_finalize "$fifopath" "$exitcode" || ret=$? else _shournal_error "_shournal_detach_this_pid: unexpected fifos $@" ret=1 fi fi return $ret } # preexec is run before a valid command (but not when ENTER or Ctrl+C is hit). # We launch a shournal-run process and wait for it to setup and # fork into background. _shournal_preexec_generic(){ local current_seq="$1" local cmd_str="$2" local fifopath local args_array if ! _shournal_verbose_history_check; then _shournal_warn "history settings were modified after the shell integration was turned on. " \ "Please correct that or call SHOURNAL_DISABLE " \ "to get rid of this message." return 1 fi fifopath="$_shournal_fifo_basepath-$current_seq" _shournal_debug "_shournal_preexec_generic: using fifo at $fifopath" _shournal_warn_on "[ -e \"$fifopath\" ]" args_array=( --verbosity "$_SHOURNAL_VERBOSITY" --pid $$ --fork --close-fds --fifoname "$fifopath" --cmd-string "$cmd_str" ) [ -n "${SHOURNAL_SESSION_ID+x}" ] && args_array+=(--shell-session-uuid "$SHOURNAL_SESSION_ID") # Argument --close-fds is important here for the following reasons: # * We may run within a subshell which waits for redirected stdout to # close (deadlock otherwise). # * We have created a custom redirection, e.g. with # exec 3> foo; echo "test" >&3; # exec 3>&-; # closes 3 # In this case **without closing** within shournal-run the close event would be lost, # as the final __fput is reached during shournal-run exit(). # Argument --fork: shournal forks itself into background once setup # is ready, so we can just wait here. if ! shournal-run "${args_array[@]}"; then # only debug here - there should already be two warnings - one from # shournal-run or bash not able to execute and (likely) one afterwards # from the prompt. _shournal_debug "_shournal_preexec_generic: shournal-run setup failed" return 1 fi return 0 } # postexec is run after any command, but eventually also after hitting ENTER # or Ctrl (other than PS0). However, the command sequence counter # SHOURNAL_CMD_COUNTER is only incremented in case of valid commands. # To avoid duplicate cleanups, we look at the return value set in PS0. # * If it's unset, no preexec has run yet. # * if it's -1, preexec was possibly run, but aborted in between _shournal_postexec_generic(){ local current_seq="$1" local exitcode="$2" local fifopath local die=false if [ -z "${_shournal_preexec_ret+x}" ]; then _shournal_debug "_shournal_postexec_generic: no preexec run yet " return 0 fi case "$_shournal_preexec_ret" in 0) : ;; '') _shournal_debug "_shournal_postexec_generic: already cleaned up" return 0 ;; -1|130) _shournal_debug "_shournal_postexec_generic: _shournal_preexec_ret is" \ "$_shournal_preexec_ret. This was likely caused by Ctrl+C (SIGINT)." ;; *) _shournal_debug "_shournal_postexec_generic: about to die due to" \ "_shournal_preexec_ret of $_shournal_preexec_ret" die=true ;; esac fifopath="$_shournal_fifo_basepath-$current_seq" _shournal_debug "_shournal_postexec_generic: using fifo at $fifopath" _shournal_trap_push '' INT || : _shournal_run_finalize "$fifopath" "$exitcode" _shournal_preexec_ret='' if [ "$die" = true ] ; then _shournal_warn "Disabling the shell-integration due to previous setup-erros..." SHOURNAL_DISABLE fi _shournal_trap_pop || : return $exitcode } # The following non-portable, shell specific functions _must_ be set # for each supported shell: # _shournal_add_prompts # _shournal_remove_prompts # # During the prompts _shournal_preexec_generic and # _shournal_postexec_generic must be # called respectively. case "$_SHOURNAL_SHELL_NAME" in 'bash') # We use _SHOURNAL_SHELL_NAME as a dummy variable in order to increment # SHOURNAL_CMD_COUNTER without printing anything in PS0. First increment, # then execute shournal. Otherwise a SIGINT may abort PS0 execution, # preventing the increment. _shournal_ps0='${_SHOURNAL_SHELL_NAME:((_shournal_preexec_ret=-1)):0}'\ '${_SHOURNAL_SHELL_NAME:((++SHOURNAL_CMD_COUNTER)):0}'\ '$(_shournal_preexec $SHOURNAL_CMD_COUNTER)'\ '${_SHOURNAL_SHELL_NAME:((_shournal_preexec_ret=$?)):0}' _shournal_prompt_command=$'_shournal_postexec\n' _shournal_add_prompts(){ [ -z "${PS0+x}" ] && PS0='' [ -z "${PROMPT_COMMAND+x}" ] && PROMPT_COMMAND='' PS0+="$_shournal_ps0" PROMPT_COMMAND="${_shournal_prompt_command}${PROMPT_COMMAND}" return 0 } _shournal_remove_prompts(){ [ -n "${PS0+x}" ] && PS0=${PS0//"$_shournal_ps0"/} [ -n "${PROMPT_COMMAND+x}" ] && PROMPT_COMMAND=${PROMPT_COMMAND//"$_shournal_prompt_command"/} return 0 } ## _____ End of must-override functions and variables _____ ## _shournal_preexec(){ local current_seq="$1" local cmd_str if [[ -z "${PROMPT_COMMAND+x}" || "$PROMPT_COMMAND" != *"$_shournal_prompt_command"* ]]; then _shournal_error "_shournal_preexec: Invalid PROMPT_COMMAND. Apparently" \ "PROMPT_COMMAND was modified after SHOURNAL_ENABLE" \ "was called. This is often caused by double-sourcing the bashrc, e.g. from" \ "~/.profile or .bash_profile." return 1 fi _shournal_get_current_cmd_bash cmd_str _shournal_preexec_generic "$current_seq" "$cmd_str" } # Disable the shell-integration in case of setup-errors, to avoid # spamming the user. Setup may in particular fail in cases where # shournal is updated while the kernel module of the old version # is still active. # Note that other than _shournal_preexec this # function is executed in the *parent shell*. _shournal_postexec(){ local ret=$? _shournal_postexec_generic "$SHOURNAL_CMD_COUNTER" "$ret" || : return $ret } ;; # END_OF bash _______________________________________________________ 'zsh') _shournal_add_prompts(){ preexec_functions+=(_shournal_preexec) precmd_functions+=(_shournal_postexec) return 0 } _shournal_remove_prompts(){ preexec_functions[$preexec_functions[(i)_shournal_preexec]]=() precmd_functions[$precmd_functions[(i)_shournal_postexec]]=() return 0 } ## _____ End of must-override functions and variables _____ ## _shournal_preexec(){ # maybe_todo: use $2 or $3 for expanded aliases instead of $1 local cmd_str="$1" local ret=0 _shournal_preexec_ret=-1 ((++SHOURNAL_CMD_COUNTER)) _shournal_preexec_generic $SHOURNAL_CMD_COUNTER "$cmd_str" || ret=$? _shournal_preexec_ret=$ret return $ret } _shournal_postexec(){ local exitcode=$? _shournal_postexec_generic $SHOURNAL_CMD_COUNTER $exitcode || return $? return 0 } ;; # END_OF zsh ________________________________________________________ *) echo "shournal shell integration: sourced from unsupported shell - " \ "currently only bash and zsh are supported." >&2 return 1 ;; esac ================================================ FILE: shell-integration-scripts/integration_main.sh.in ================================================ # Shell integration for shournal # This file contains all public functions and # must be compatible with all supported shells. SHOURNAL_ENABLE(){ local cmd_path args_array line ret=0 local clusterjob_reexec_counter if _shournal_is_subshell; then _shournal_warn "shournal's shell integration must not be enabled from" \ "within a subshell" return 1 fi if [ -z "$(command -v shournal)" ] ; then _shournal_error "cannot enable shournal's shell integration - command «shournal» not found" return 1 fi if [ -z "$(command -v "$_shournal_run_backend")" ] ; then _shournal_error "cannot enable shournal's shell integration - " \ "command «$_shournal_run_backend» not found" return 1 fi if ! shournal --validate-settings; then # informative mesg. should have been already printed by shournal _shournal_error "shell integration is _not_ enabled" return 1 fi if [ -n "${_SHOURNAL_CLUSTERJOB_JUST_REEXECUTED+x}" ]; then _shournal_debug "shournal just re-executed this clusterjob." \ "SHOURNAL_ENABLE is ignored this time." unset _SHOURNAL_CLUSTERJOB_JUST_REEXECUTED return 0 fi if _shournal_clusterjob_reexec_ok; then # Do not use BASH_EXECUTION_STRING, it is not set in all cluster jobs. args_array=() while IFS= read -r -d '' line; do args_array+=("$line") done < /proc/$$/cmdline if [ ${#args_array[@]} -lt 2 ]; then _shournal_warn "SHOURNAL_ENABLE: we detected ${args_array[*]} as" \ "cluster job without arguments. Please report." else cmd_path="$(readlink /proc/$$/exe)" _shournal_debug "running cluster job: $cmd_path ${args_array[*]}" clusterjob_reexec_counter=${_SHOURNAL_CLUSTERJOB_REEXEC_COUNTER:-0} [ $clusterjob_reexec_counter -gt 0 ] && _shournal_debug "unusual clusterjob_reexec_counter of $clusterjob_reexec_counter" ((++clusterjob_reexec_counter)) _SHOURNAL_CLUSTERJOB_REEXEC_COUNTER="$clusterjob_reexec_counter" \ _SHOURNAL_CLUSTERJOB_JUST_REEXECUTED=true exec \ shournal --backend-filename "$_shournal_run_backend" \ --verbosity "$_SHOURNAL_VERBOSITY" \ --exec-filename "$cmd_path" --exec -- "${args_array[@]}" # only get here on error return 1 fi fi if [ -z "${_shournal_shell_exec_string+x}" ]; then # Running somewhat "interactively" _shournal_verbose_history_check || return $? fi _shournal_enable || ret=$? return $ret } SHOURNAL_DISABLE() { # In case we were called in a sequence, e.g. # $ (exit 123); SHOURNAL_DISABLE # capture the exit code here local exitcode=$? local ret=0 if _shournal_is_subshell; then _shournal_warn "shournal's shell integration must not be disabled from" \ "within a subshell" return 1 fi if _shournal_is_clusterjob && [ -n "${_SHOURNAL_CLUSTERJOB_REEXEC_COUNTER+x}" ]; then _shournal_warn "shournal seems to trace this process as a cluster job" \ "and cannot be disabled in that mode." return 1 fi _shournal_disable $exitcode || return ret=$? if [ $ret -eq 0 ]; then unset _shournal_current_pid SHOURNAL_SESSION_ID SHOURNAL_CMD_COUNTER fi return $ret } # $1: pass one of dbg, info, warning, critical SHOURNAL_SET_VERBOSITY(){ case "$1" in "dbg") _shournal_shell_integration_log_level=0 ;; "info") _shournal_shell_integration_log_level=1 ;; "warning") _shournal_shell_integration_log_level=2 ;; "critical") _shournal_shell_integration_log_level=3 ;; *) _shournal_warn "Bad verbosity passed. Pass one of dbg, info, warning, critical" return 1 ;; esac # verbosity for shournal-run* export _SHOURNAL_VERBOSITY="$1" _shournal_set_verbosity "$1" } SHOURNAL_PRINT_VERSIONS() { echo "shournal $_SHOURNAL_SHELL_NAME integration version: $_shournal_version" echo "shournal: $(shournal --version)" _shournal_print_versions } # _________ End of public interface _________ # # BEGIN_SECTION auto-filled by cmake _shournal_version="@shournal_version@" # -z: Allow to be overwritten [ -z "${SHOURNAL_PATH_LIB_SHELL_INTEGRATION+x}" ] && export SHOURNAL_PATH_LIB_SHELL_INTEGRATION="@full_path_libshournal@" # END_SECTION auto-filled by cmake # We have to set several global variables here and _not_ in SHOURNAL_ENABLE # for the libshournal-shellwatch.so LD_PRELOAD-hack. If it is not # not loaded, on SHOURNAL_ENABLE we exec the current shell again and # perform the actual initialization. Therefore, *this* # script must be sourced within the shell's rc but we don't want to # require SHOURNAL_ENABLE within the rc-file. # 0: debug, 1: info, 2: warning, 3: error [ -z "${_shournal_shell_integration_log_level+x}" ] && _shournal_shell_integration_log_level=2 # verbosity for shournal-run* ( _must_ be exported ) [ -z "${_SHOURNAL_VERBOSITY+x}" ] && export _SHOURNAL_VERBOSITY="warning" # verbosity for libshournal-shellwatch.so ( _must_ be exported ) [ -z "${_SHOURNAL_LIB_SHELL_VERBOSITY+x}" ] && export _SHOURNAL_LIB_SHELL_VERBOSITY="warning" # Setup non-portable stuff for # each supported shell. The following variables _must_ be set: # _SHOURNAL_SHELL_NAME (name of the current shell). It must be exported # for libshournal-shellwatch.so. # _shournal_shell_exec_string - if and only if the command is executed non-interactively # For zsh aliases: avoid error on double source unalias _shournal_trap_set &>/dev/null || : if [ -n "${BASH_VERSION+x}" ]; then export _SHOURNAL_SHELL_NAME='bash' # The bash execution string is e.g. set when running bash -c 'echo foo', in which case we never get to # any prompt. Simply execute the whole command # within shournal. # Checking $BASH_EXECUTION_STRING seems to be more reliable than [[ $- == *i* ]], because # of commands like e.g. bash -i -c 'echo "wtf - is that interactive?"' [ -n "${BASH_EXECUTION_STRING+x}" ] && _shournal_shell_exec_string="$BASH_EXECUTION_STRING" _shournal_trap_push(){ local trap_tmp # First save to temporary variable, as we are not interrupt-safe yet trap_tmp="$(trap -p INT)" trap "$@" _shournal_int_traps+=("$trap_tmp") return 0 } _shournal_trap_pop(){ local old_trap if [ ${#_shournal_int_traps[@]} -eq 0 ]; then _shournal_error "_shournal_trap_pop - no int trap set" >&2 return 1 fi old_trap="${_shournal_int_traps[-1]}" # first unset: if we reset to default trap and user hits Ctrl+C, we would # leak the array element unset _shournal_int_traps[-1] if [ -z "$old_trap" ]; then trap - INT else eval "$old_trap" fi return 0 } elif [ -n "${ZSH_VERSION+x}" ]; then export _SHOURNAL_SHELL_NAME='zsh' [ -n "${ZSH_EXECUTION_STRING+x}" ] && _shournal_shell_exec_string="$ZSH_EXECUTION_STRING" # This has to be at the top, so aliases are expanded in the other functions (files # are appended to this script) setopt aliases alias _shournal_trap_push='setopt localtraps; trap' _shournal_trap_pop(){ :; } else echo "shournal shell integration: sourced from unsupported shell - " \ "currently only bash and zsh are supported." >&2 return 1 fi ================================================ FILE: shell-integration-scripts/util.sh ================================================ # don't call it directly, but use one of debug, info, warning, error functions # $1: loglevel. # all other args: is printed to stderr _shournal_log_msg(){ local loglevel=$1 shift [ "$loglevel" -ge "$_shournal_shell_integration_log_level" ] && >&2 printf "shournal $_SHOURNAL_SHELL_NAME integration - $*\n" } _shournal_error() { _shournal_log_msg 3 "ERROR: $*" } _shournal_warn(){ _shournal_log_msg 2 "warning: $*" } _shournal_info(){ _shournal_log_msg 1 "info: $*" } _shournal_debug(){ _shournal_log_msg 0 "debug: $*" } _shournal_warn_on(){ eval "$1" && _shournal_warn "$1" } # returns true, if $1 starts with $2 _shournal_startswith() { case $1 in "$2"*) return 0;; *) return 1;; esac; } # returns true, if $1 ends with $2 _shournal_endswith() { case $1 in *"$2") return 0;; *) return 1;; esac; } # returns true, if $1 contains $2 _shournal_contains() { case $1 in *"$2"*) return 0;; *) return 1;; esac; } # Trim leading and trailing spaces _shournal_trim(){ echo -e "${1}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' } _shournal_is_clusterjob(){ # Recent versions of the sun grid engine (SGE) use cgroups to manage # processes of a job. Once the main job-script finishes, all leftovers # are killed. Therefore, shournal cannot flush the events to the # database in background afterwards, so we fall back to foreground execution. # Usually SHOURNAL_ENABLE is expected to be part of the shell's rc, # e.g. ~/.bashrc, which we require for a "safe" re-execution. # A cluster-job should be: non-interactive and running within a login-shell. [ -n "${SHOURNAL_IS_CLUSTERJOB+x}" ] && return 0 [ -z "${SGE_O_WORKDIR+x}" -o -z "${JOB_NAME+x}" -o -t 0 -o \ -t 1 -o -t 2 -o -n "${SHOURNAL_NO_CLUSTER_JOB_DETECT+x}" ] && return 1 _shournal_sh_is_interactive && return 1 return 0 } _shournal_clusterjob_reexec_ok(){ # s. _shournal_is_clusterjob _shournal_is_clusterjob || return $? if ! _shournal_verbose_reexec_allowed; then _shournal_warn "cluster job detected, but we are not allowed to re-exec." \ "Please check your environment." return 1 fi return 0 } _shournal_is_subshell(){ _shournal_refresh_current_pid || return $? [ $_shournal_current_pid -ne $$ ] && return 0 return 1 } _shournal_refresh_current_pid(){ local pid ret=0 read -d ' ' pid < /proc/self/stat || ret=$? if [ $ret -ne 0 ]; then _shournal_error "_shournal_refresh_current_pid:" \ "failed to read from /proc/self/stat: $ret" return $ret fi _shournal_current_pid=$pid return 0 } # Non-portable, shell specific functions: case "$_SHOURNAL_SHELL_NAME" in 'bash') # returns 0, if all history settings were ok, else false. # is verbose, if a setting is not ok. _shournal_verbose_history_check(){ # no history needed if running non-interactively [ -n "${BASH_EXECUTION_STRING+x}" ] && return 0 local ret=0 if ! [ -o history ]; then ret=1 _shournal_error "bash history is off. Please enable it: set -o history" fi if [[ ${HISTSIZE-0} -lt 2 ]]; then ret=1 _shournal_error "bash HISTSIZE is too small (or not set). Please set it at least to 2: HISTSIZE=2" fi if [[ ${HISTCONTROL-} == *"ignorespace"* || ${HISTCONTROL-} == *"ignoreboth"* ]]; then ret=1 _shournal_error "Commands with spaces are set to be ignored from history. Please disable that, " \ "e.g. HISTCONTROL=ignoredups or HISTCONTROL=''" fi if [[ -n ${HISTIGNORE-} ]] ; then ret=1 _shournal_error "HISTIGNORE is not empty. Please unset it: unset HISTIGNORE" fi return $ret } _shournal_get_current_cmd_bash(){ declare -n ret=$1 local cmd # history output is e.g. # " 6989 echo foo" # so strip the leading " 6989 " cmd="$(HISTTIMEFORMAT='' history 1)" [[ "$cmd" =~ ([[:space:]]*[0-9]+[[:space:]]*)(.*) ]] ret="${BASH_REMATCH[2]}" return 0 } _shournal_print_current_cmd(){ local cmd_str _shournal_get_current_cmd_bash cmd_str printf '%s\n' "$cmd_str" } _shournal_refresh_current_pid(){ _shournal_current_pid=$BASHPID return 0 } _shournal_sh_is_interactive(){ [[ $- == *i* ]] && return 0 return 1 } # Return true, if we are allowed to reexec, which we do in case # of non-interactive ssh commands (for the fanotify backend), where the # BASH_EXECUTION_STRING is always set, or in case of cluster jobs, which # are usually invoked as login_shell. # Re-exec is not allowed, if the a command # within the -c '..' arg was already executed # (it would be executed twice otherwise). _shournal_verbose_reexec_allowed(){ local i sourced_from_bashrc if [[ -z ${BASH_EXECUTION_STRING+x} ]] && ! shopt -q login_shell; then return 1 fi # Only consider this a running cluster job, if we are sourced from .bashrc. # FIXME: this is not robust, bash -c 'echo foo; source ~/.bashrc' # should _not_ be allowed. if [[ -z ${BASH_SOURCE+x} ]]; then _shournal_error "BASH_SOURCE is not set. Something is seriously" \ "wrong here, aborting..." return 1 fi sourced_from_bashrc=false for ((i=1; i<${#BASH_SOURCE[@]}; i++)); do if [[ "${BASH_SOURCE[i]##*/}" == .bashrc ]]; then sourced_from_bashrc=true break fi done if [[ $sourced_from_bashrc == false ]]; then _shournal_warn "The command was considered for re-execution, but" \ "we require to be sourced from .bashrc for SHOURNAL_ENABLE." \ "Alternatively, invoke »shournal -e« directly." return 1 fi return 0 } ;; # END_OF bash 'zsh') _shournal_verbose_history_check(){ # no history needed if running non-interactively [ -n "${ZSH_EXECUTION_STRING+x}" ] && return 0 # While in bash we retrieve the command-string # from history, in zsh it is directly passed # to our preexec_function, which seems to work # regardless of history options like e.g. # $ setopt HIST_IGNORE_SPACE return 0 } _shournal_print_current_cmd(){ printf '%s\n' "$history[$HISTCMD]" } _shournal_sh_is_interactive(){ [[ -o interactive ]] && return 0 return 1 } _shournal_verbose_reexec_allowed(){ local toplevel_contex if [[ -z ${ZSH_EXECUTION_STRING+x} && ! -o login ]]; then return 1 fi zmodload zsh/parameter toplevel_context="${zsh_eval_context[1]}" case "$toplevel_context" in file) :;; cmdarg) _shournal_warn "eval-toplevel-context $toplevel_context not allowed" return 1;; *) _shournal_warn "unhandled eval-toplevel-context $toplevel_context." \ "Please report if you" \ "think that SHOURNAL_ENABLE should be possible here." return 1;; esac return 0 } ;; # END_OF zsh *) echo "shournal shell integration: something is seriously wrong, " \ "_SHOURNAL_SHELL_NAME is not correctly setup" >&2 ;; esac ================================================ FILE: src/CMakeLists.txt ================================================ include_directories( ../extern ../kernel common/ common/database common/oscpp common/oscpp common/qoptargparse common/qsimplecfg common/util ) add_subdirectory("common") add_subdirectory("shournal") if(${SHOURNAL_EDITION} MATCHES "full|docker|ko") add_subdirectory("shournal-run") endif() if(${SHOURNAL_EDITION} MATCHES "full|docker|fanotify") add_subdirectory("shournal-run-fanotify") add_subdirectory("shell-integration-fanotify") endif() ================================================ FILE: src/common/CMakeLists.txt ================================================ add_subdirectory(util) add_subdirectory(oscpp) add_subdirectory(qoptargparse) add_subdirectory(qsimplecfg) SET(lib_shournal_common_files app.cpp cefd.cpp console_dialog.cpp cxxhash.cpp fdcommunication.cpp fileeventhandler.cpp fileevents.cpp generic_container.h groupcontrol.cpp hashcontrol.cpp hashmeta.cpp idmapentry.h interrupt_handler.cpp logger.cpp limited_priority_queue.h pidcontrol.cpp pathtree.cpp qfddummydevice.cpp qfilethrow.cpp qresource_helper.cpp safe_file_update.h settings.cpp shournal_run_common.cpp stdiocpp.cpp stupidinject.cpp socket_message.cpp subprocess.cpp user_kernerl.h xxhash_common.h xxhash_common.c ) set(database_files database/db_connection.cpp database/db_controller.cpp database/sqlite_database_scheme.cpp database/commandinfo.cpp database/sessioninfo.cpp database/fileinfos.cpp database/sqlquery.cpp database/file_query_helper.cpp database/insertifnotexist.cpp database/query_columns.h database/db_conversions.cpp database/sqlite_database_scheme_updates.cpp database/storedfiles.cpp database/db_globals.cpp database/command_query_iterator.cpp database/qexcdatabase.cpp database/qsqlquerythrow.cpp ) add_library(lib_shournal_common ${lib_shournal_common_files} ${database_files} ) target_link_libraries(lib_shournal_common PUBLIC Qt5::Core Qt5::Sql Qt5::Network xxhash uuid ${CMAKE_DL_LIBS} cap lib_util oscpp_lib lib_qoptargparse lib_qsimplecfg ) ================================================ FILE: src/common/app.cpp ================================================ #include #include #include #include #include "qoutstream.h" #include "app.h" #include "util.h" #include "osutil.h" // may be supplied at buildtime, else should be defined in cmake file #ifndef SHOURNAL_MSENTERGROUP static_assert (false, "SHOURNAL_MSENTERGROUP not defined"); #endif const char* app::CURRENT_NAME = "UNDEFINED"; const char* app::SHOURNAL = "shournal"; const char* app::SHOURNAL_RUN = "shournal-run"; const char* app::SHOURNAL_RUN_FANOTIFY = "shournal-run-fanotify"; // groupnames should be smaller than 16 characters (portability). const char* app::MSENTER_ONLY_GROUP = SHOURNAL_MSENTERGROUP; // defined in cmake const char* app::ENV_VAR_SOCKET_NB = "_SHOURNAL_SOCKET_NB"; const std::unordered_set &app::VERBOSITIES = {"dbg", #if QT_VERSION >= QT_VERSION_CHECK(5, 5, 0) "info", #endif "warning", "critical"}; static bool g_inIntegrationTestMode=false; void app::setupNameAndVersion(const char* currentName) { app::CURRENT_NAME = currentName; QIErr::setPreambleCallback([]() { return QString(app::CURRENT_NAME) + ": "; }); g_inIntegrationTestMode = getenv("_SHOURNAL_IN_INTEGRATION_TEST_MODE") != nullptr; QString integrationSuffix; if(g_inIntegrationTestMode){ integrationSuffix = "-integration-test"; QStandardPaths::setTestModeEnabled(true); // QIErr() << "running in integration test mode"; } QCoreApplication::setApplicationName(app::SHOURNAL + integrationSuffix); QCoreApplication::setApplicationVersion(app::version().toString()); } bool app::inIntegrationTestMode() { return g_inIntegrationTestMode; } int app::findIntegrationTestFd(){ if(! app::inIntegrationTestMode()){ return -1; } QByteArray fdStr = getenv("_SHOURNAL_INTEGRATION_TEST_PIPE_FD"); if(fdStr.isNull()){ QIErr() << "Although in integration-test, cannot" "find pipe fd in env!"; return -1; } int fd = qVariantTo_throw(fdStr); if(! osutil::fdIsOpen(fd)){ QIErr() << "_SHOURNAL_INTEGRATION_TEST_PIPE_FD is not open - number:" << fd; return -1; } return fd; } const QVersionNumber &app::version() { // defined in cmake static const QVersionNumber v = QVersionNumber::fromString(SHOURNAL_VERSION); return v; } const QVersionNumber &app::initialVersion() { static const QVersionNumber v = QVersionNumber{0, 1}; // first version ever; return v; } ================================================ FILE: src/common/app.h ================================================ #pragma once #include #include namespace app { const extern char* CURRENT_NAME; const extern char* SHOURNAL; const extern char* SHOURNAL_RUN; const extern char* SHOURNAL_RUN_FANOTIFY; const extern char* MSENTER_ONLY_GROUP; const extern char* ENV_VAR_SOCKET_NB; const extern std::unordered_set &VERBOSITIES; void setupNameAndVersion(const char *currentName); bool inIntegrationTestMode(); int findIntegrationTestFd(); const QVersionNumber& version(); const QVersionNumber& initialVersion(); } ================================================ FILE: src/common/cefd.cpp ================================================ #include #include "cefd.h" #include "os.h" #include "excos.h" #include "osutil.h" CEfd::CEfd() { m_fd = eventfd(0, EFD_CLOEXEC); if (m_fd == -1){ throw os::ExcOs("eventfd failed"); } } CEfd::~CEfd() { teardown(); } void CEfd::sendMsg(uint64_t n) { os::write(m_fd, &n, sizeof (n)); } uint64_t CEfd::recvMsg() { uint64_t n; // Block until parent process did the setup if(os::read(m_fd, &n, sizeof(n)) != sizeof(n)){ throw os::ExcOs("cefd: read wrong size."); } return n; } void CEfd::teardown() { if(m_fd != -1){ osutil::closeVerbose(m_fd); m_fd = -1; } } ================================================ FILE: src/common/cefd.h ================================================ #pragma once #include #include "util.h" class CEfd { public: static const uint64_t MSG_OK {7}; static const uint64_t MSG_FAIL {8}; CEfd(); ~CEfd(); void sendMsg(uint64_t n); uint64_t recvMsg(); void teardown(); private: Q_DISABLE_COPY(CEfd) DISABLE_MOVE(CEfd) int m_fd; }; ================================================ FILE: src/common/console_dialog.cpp ================================================ #include #include #include #include "compat.h" #include "console_dialog.h" #include "qoutstream.h" #include "util.h" #include "subprocess.h" using subprocess::Subprocess; /// Ask a simple yesno-question and return the result. /// @returns true, if "y", false if "n" was entered bool console_dialog::yesNo(const QString &question) { const QString yesStr = qtr("y"); const QString noStr = qtr("n"); QOut() << QString("%1 (%2/%3) ").arg(question, yesStr, noStr); QTextStream input(stdin); while (true) { QString respone = input.readLine(); if(respone.compare(yesStr, Qt::CaseSensitivity::CaseInsensitive) == 0){ return true; } if(respone.compare(noStr, Qt::CaseSensitivity::CaseInsensitive) == 0){ return false; } QOut() << qtr("Please enter %1 or %2").arg(yesStr, noStr) << "\n"; } } /// Open filepath within the users favourite editor, /// exported in environment variable EDITOR. If not set, try /// to find a typical editor such as nano, vim,... /// @return return value of the launched process. In case it did'nt exit normally, /// an os-exception is thrown, /// @throws QExcIo, os::ExcOs int console_dialog::openFileInExternalEditor(const QString &filepath) { QString editor = getenv("EDITOR"); subprocess::Args_t args; if(editor.isEmpty()){ if((editor=QStandardPaths::findExecutable("nano")).isEmpty()) if((editor=QStandardPaths::findExecutable("vim")).isEmpty()) if((editor=QStandardPaths::findExecutable("vi")).isEmpty()){ throw QExcIo(qtr("No texteditor found, please set EDITOR " "environment variable.")); } args.push_back(editor.toStdString()); } else { // support also EDITOR-strings like e.g. 'geany -i' -> if we cannot find // the executable, try to split by space if((QStandardPaths::findExecutable(editor)).isEmpty() ){ const auto splitted = editor.split(' ', Qt::SkipEmptyParts); if(splitted.length() > 1){ for(const QString& s : splitted){ args.push_back(s.toStdString()); } } else { // let it (probably) fail below: args.push_back(editor.toStdString()); } } else { args.push_back(editor.toStdString()); } } args.push_back(filepath.toStdString()); Subprocess subproc; subproc.call(args); return subproc.waitFinish(); } ================================================ FILE: src/common/console_dialog.h ================================================ #pragma once #include namespace console_dialog { bool yesNo(const QString& question); int openFileInExternalEditor(const QString& filepath); } ================================================ FILE: src/common/cxxhash.cpp ================================================ #include #include "cxxhash.h" #include #include "excos.h" #include "os.h" CXXHash::CXXHash() : m_pXXState(XXH64_createState()) { assert(m_pXXState != nullptr); m_buf.resize(sysconf(_SC_PAGESIZE)); } CXXHash::~CXXHash() { XXH64_freeState(m_pXXState); } void CXXHash::resizeBuf(size_t n) { assert(n > 0); m_buf.resize(n); } /// @throws CXXHashError void CXXHash::reset(unsigned long long seed) { XXH_errorcode err=XXH64_reset(m_pXXState, seed); if(err == XXH_ERROR ){ throw ExcCXXHash("reset failed", err); } } /// @throws CXXHashError void CXXHash::update(const void *buffer, size_t len) { XXH_errorcode err = XXH64_update(m_pXXState, buffer, len); if(err == XXH_ERROR ){ throw ExcCXXHash("update failed", err); } } struct partial_xxhash_result CXXHash::digestWholeFile(int fd, int chunksize) { return this->digestFile(fd, chunksize, 0, std::numeric_limits::max()); } partial_xxhash_result CXXHash::digestFile(int fd, int chunksize, off64_t seekstep, int maxCountOfReads) { struct partial_xxhash part_hash; part_hash.xxh_state = m_pXXState; part_hash.chunksize = chunksize; part_hash.seekstep = seekstep; part_hash.max_count_of_reads = maxCountOfReads; part_hash.buf = m_buf.data(); part_hash.bufsize = m_buf.size(); struct partial_xxhash_result res; auto ret = partial_xxh_digest_file(fd, &part_hash, &res); if(ret != 0){ throw ExcCXXHash("digest failed: ", int(ret)); } return res; } /* /// XXHASH-digest a whole file or parts of it at regular intervals. /// @param fd the fildescriptor of the file. Note that in general you would want /// to make sure, that the offset is at 0. Note that the offset /// may be changed during the call. /// @param chunksize size of the chunks to read at once. /// @param seekstep Read chunks from the file every seekstep bytes. The read chunk /// does not count into this, so if you actually want to skip bytes, /// seekstep must be greater than chunksize. Otherwise NO SEEK is /// performed at all. /// @param maxCountOfReads stop reading and digest after that count of 'read'- /// operations. /// @returns the calculated hash and the actual count of bytes read. /// If the count of bytes is zero, the hash is invalid. /// @throws ExcOs, CXXHashError CXXHash::DigestResult CXXHash::digestFile(int fd, int chunksize, off64_t seekstep, int maxCountOfReads) { /// Implementation detail: /// Calling XXH64_update introduces some overhead, which can be avoided by /// calling XXH64() directly with a sufficiently large buffer. /// So, if our buffer is large enough, read the chunks from file /// one by one into our own buffer. If it's full, call XXH64_update, /// else do it alltogether at the end. assert(maxCountOfReads > 0); assert(chunksize > 0); if(chunksize > int(m_buf.size())){ m_buf.resize(chunksize); } const bool doSeek = seekstep > chunksize; DigestResult res; off64_t offset=0; res.countOfbytes = 0; char* bufRaw = m_buf.data(); char* bufRawEnd = bufRaw + m_buf.size(); bool updateNecessary = false; for(int countOfReads=0; countOfReads < maxCountOfReads ; ++countOfReads) { ssize_t readBytes = os::read(fd, bufRaw, static_cast(chunksize)); bufRaw += readBytes; res.countOfbytes += readBytes; if(readBytes < chunksize) { break; // EOF } if(bufRawEnd - bufRaw <= chunksize){ // not enough space for another read: flush buffer if(! updateNecessary){ updateNecessary = true; this->reset(0); } this->update(m_buf.data(), bufRaw - m_buf.data()); bufRaw = m_buf.data(); } if( doSeek ) { offset += seekstep; os::lseek(fd, offset, SEEK_SET); } } if(res.countOfbytes == 0){ res.hash = 0; return res; } // we read something if(updateNecessary){ if(bufRaw != m_buf.data()){ this->update(m_buf.data(), bufRaw - m_buf.data()); } res.hash = XXH64_digest(m_pXXState); return res; } // No update was needed (all chunks fitted into buffer). Flush the whole buffer at once without // xxhash state overhead (update/reset) assert(bufRaw != m_buf.data()); res.hash = XXH64(m_buf.data(), bufRaw - m_buf.data(), 0 ); return res; } */ CXXHash::ExcCXXHash::ExcCXXHash(const std::string &msg, int errorcode) : m_errorcode(errorcode) { m_descrip = "XXHashError occurred: " + msg + " - errorcode: " + std::to_string(m_errorcode) ; } const char *CXXHash::ExcCXXHash::what() const noexcept { return m_descrip.c_str(); } ================================================ FILE: src/common/cxxhash.h ================================================ #pragma once #include #include #include #include #include "xxhash.h" #include "strlight.h" #include "xxhash_common.h" /// A cpp interface around the needed c-functions of XXHASH and /// some other methods (digestFile). /// For further documentation of the wrapper-only-functions please head to the /// documentation of the c-api. class CXXHash { public: class ExcCXXHash : public std::exception { public: explicit ExcCXXHash(const std::string & msg, int errorcode); const char *what () const noexcept override; private: std::string m_descrip; int m_errorcode; }; CXXHash(); ~CXXHash(); void resizeBuf(size_t n); struct partial_xxhash_result digestWholeFile(int fd, int chunksize); struct partial_xxhash_result digestFile(int fd, int chunksize, off64_t seekstep, int maxCountOfReads=std::numeric_limits::max()); public: CXXHash(const CXXHash&) = delete; void operator=(const CXXHash&) = delete; private: void reset(unsigned long long seed=0); void update(const void* buffer, size_t len); XXH64_state_t * const m_pXXState; StrLight m_buf; }; ================================================ FILE: src/common/database/command_query_iterator.cpp ================================================ #include #include "command_query_iterator.h" #include "util.h" #include "db_connection.h" #include "db_conversions.h" #include "db_controller.h" /// @param reverseIter: if true, instead of calling next(), previous() will be called /// on the passed query. CommandQueryIterator::CommandQueryIterator(std::shared_ptr& query, bool reverseIter) : m_cmdQuery(query), m_tmpQuery(db_connection::mkQuery()), m_reverseIter(reverseIter) { } // set cursor to next or previous, if reverseIter was set on constructor bool CommandQueryIterator::next() { m_cmd.clear(); const bool nextRet = (m_reverseIter) ? m_cmdQuery->previous() : m_cmdQuery->next(); if(nextRet){ fillCommand(); } return nextRet; } CommandInfo &CommandQueryIterator::value() { return m_cmd; } int CommandQueryIterator::computeSize() { return m_cmdQuery->computeSize(); } void CommandQueryIterator::fillCommand() { int i=0; m_cmd.idInDb = qVariantTo_throw(m_cmdQuery->value(i++)); m_cmd.text = m_cmdQuery->value(i++).toString(); m_cmd.returnVal = m_cmdQuery->value(i++).toInt(); m_cmd.startTime = m_cmdQuery->value(i++).toDateTime(); m_cmd.endTime = m_cmdQuery->value(i++).toDateTime(); m_cmd.workingDirectory = m_cmdQuery->value(i++).toString(); m_cmd.sessionInfo.uuid = m_cmdQuery->value(i++).toByteArray(); m_cmd.sessionInfo.comment = m_cmdQuery->value(i++).toString(); QVariant hashChunksize = m_cmdQuery->value(i++); if(! hashChunksize.isNull()){ qVariantTo_throw(hashChunksize, &m_cmd.hashMeta.chunkSize) ; qVariantTo_throw(m_cmdQuery->value(i++), &m_cmd.hashMeta.maxCountOfReads); } else { i++; } m_cmd.username = m_cmdQuery->value(i++).toString(); m_cmd.hostname = m_cmdQuery->value(i++).toString(); fillWrittenFiles(); m_cmd.fileReadInfos = db_controller::queryReadInfos_byCmdId(m_cmd.idInDb); } void CommandQueryIterator::fillWrittenFiles() { m_tmpQuery->prepare("select writtenFile.id,writtenFile_path.path,writtenFile.name," "writtenFile.mtime,writtenFile.size,writtenFile.hash " "from writtenFile " "join pathtable as writtenFile_path " "on writtenFile.pathId=writtenFile_path.id " "where cmdId=?"); m_tmpQuery->addBindValue(m_cmd.idInDb); m_tmpQuery->exec(); while(m_tmpQuery->next()){ int i=0; FileWriteInfo fInfo; fInfo.idInDb = qVariantTo_throw(m_tmpQuery->value(i++)); fInfo.path = m_tmpQuery->value(i++).toString(); fInfo.name = m_tmpQuery->value(i++).toString(); fInfo.mtime = m_tmpQuery->value(i++).toDateTime(); fInfo.size = qVariantTo_throw(m_tmpQuery->value(i++)); fInfo.hash = db_conversions::toHashValue(m_tmpQuery->value(i++)); m_cmd.fileWriteInfos.push_back(fInfo); } } ================================================ FILE: src/common/database/command_query_iterator.h ================================================ #pragma once #include #include "qsqlquerythrow.h" #include "commandinfo.h" #include "db_connection.h" class CommandQueryIterator { public: CommandQueryIterator(std::shared_ptr &query, bool reverseIter); bool next(); CommandInfo& value(); int computeSize(); public: CommandQueryIterator(const CommandQueryIterator &) = delete ; void operator=(const CommandQueryIterator &) = delete ; private: void fillCommand(); void fillWrittenFiles(); std::shared_ptr m_cmdQuery; QueryPtr m_tmpQuery; CommandInfo m_cmd; bool m_reverseIter; }; ================================================ FILE: src/common/database/commandinfo.cpp ================================================ #include #include #include #include "commandinfo.h" #include "os.h" #include "settings.h" #include "db_globals.h" #include "conversions.h" /// Settings must be loaded beforehand! /// Fill commandInfo with those information independent from the current /// command. The following properties yet *have* to be set: /// startTime, endTime, text /// The following *may* be set: /// returnVal CommandInfo CommandInfo::fromLocalEnv() { CommandInfo cmd; cmd.hostname = QHostInfo::localHostName(); cmd.username = os::getUserName(); // Do not: // cmd.workingDirectory = QDir::currentPath(); // If the working directory is deleted, this returns a null string, which is not very // informative (and also not allowed in the database scheme). Using below approach returns // a valid string with a trailing ' (deleted)', if appropriate. cmd.workingDirectory = QString::fromLocal8Bit(os::readlink("/proc/self/cwd")); auto & sets = Settings::instance(); if(sets.hashSettings().hashEnable){ cmd.hashMeta = sets.hashSettings().hashMeta; } return cmd; } CommandInfo::CommandInfo() : idInDb(db::INVALID_INT_ID), text(""), // empty string, so QString.isNull() returns false -> no null-inserts into database returnVal(INVALID_RETURN_VAL) {} void CommandInfo::write(QJsonObject &json, bool withMilliseconds, const CmdJsonWriteCfg &writeCfg) const { if(writeCfg.idInDb) json["id"] = idInDb; if(writeCfg.text) json["command"] = text; if(writeCfg.returnVal) json["returnValue"] = returnVal; if(writeCfg.username) json["username"] = username; if(writeCfg.hostname) json["hostname"] = hostname; if(writeCfg.hashMeta) { QJsonValue hashChunkSize; QJsonValue hashMaxCountOfReads; if(! hashMeta.isNull()){ hashChunkSize = hashMeta.chunkSize; hashMaxCountOfReads = hashMeta.maxCountOfReads; } json["hashChunkSize"] = hashChunkSize; json["hashMaxCountOfReads"] = hashMaxCountOfReads; } // A null-session-QString becomes a quoted string in json, instead of null, so below // effort is necessary (invalid session should always be null: in database, shournal and js-plot...). if(writeCfg.sessionInfo){ json["sessionUuid"] = (sessionInfo.uuid.isNull()) ? QJsonValue() : QString::fromLatin1(sessionInfo.uuid.toBase64()); } if(withMilliseconds){ if(writeCfg.startEndTime){ json["startTime"] = startTime.toString(Conversions::dateIsoFormatWithMilliseconds()); json["endTime"] = endTime.toString(Conversions::dateIsoFormatWithMilliseconds()); } } else { if(writeCfg.startEndTime){ json["startTime"] = QJsonValue::fromVariant(startTime); json["endTime"] = QJsonValue::fromVariant(endTime); } } if(writeCfg.workingDirectory) json["workingDir"] = workingDirectory; if(writeCfg.fileReadInfos){ QJsonArray fReadArr; int idx = 0; for(const auto& info : fileReadInfos){ QJsonObject fReadObj; info.write(fReadObj); fReadObj["status"] = (writeCfg.fileStatus) ? info.currentStatus(*this) : "NA"; fReadArr.append(fReadObj); ++idx; if(idx >= writeCfg.maxCountRFiles){ break; } } json["fileReadEvents"] = fReadArr; } if(writeCfg.fileWriteInfos){ QJsonArray fWriteArr; int idx = 0; for(const auto& info : fileWriteInfos){ QJsonObject fWObject; info.write(fWObject); fWObject["status"] = (writeCfg.fileStatus) ? info.currentStatus(*this) : "NA"; fWriteArr.append(fWObject); ++idx; if(idx >= writeCfg.maxCountWFiles){ break; } } json["fileWriteEvents"] = fWriteArr; } } bool CommandInfo::operator==(const CommandInfo &rhs) const { if(idInDb != db::INVALID_INT_ID && rhs.idInDb != db::INVALID_INT_ID){ return idInDb == rhs.idInDb; } return text == rhs.text && returnVal == rhs.returnVal && username == rhs.username && hostname == rhs.hostname && hashMeta == rhs.hashMeta && sessionInfo == rhs.sessionInfo && fileWriteInfos == rhs.fileWriteInfos && fileReadInfos == rhs.fileReadInfos && startTime == rhs.startTime && endTime == rhs.endTime && workingDirectory == rhs.workingDirectory; } void CommandInfo::clear() { fileWriteInfos.clear(); fileReadInfos.clear(); idInDb = db::INVALID_INT_ID; } ================================================ FILE: src/common/database/commandinfo.h ================================================ #pragma once #include #include #include #include "hashmeta.h" #include "sessioninfo.h" #include "fileinfos.h" typedef QVector FileWriteInfos; typedef QVector FileReadInfos; /// Configure which fields shall be written /// to JSON on CommandInfo.write() (and how many /// entries of some fields) struct CmdJsonWriteCfg { CmdJsonWriteCfg(bool initAll) : idInDb(initAll), text(initAll), returnVal(initAll), username(initAll), hostname(initAll), hashMeta(initAll), sessionInfo(initAll), startEndTime(initAll), workingDirectory(initAll), fileWriteInfos(initAll), fileReadInfos(initAll) {} bool idInDb; bool text; bool returnVal; bool username; bool hostname; bool hashMeta; bool sessionInfo; bool startEndTime; bool workingDirectory; bool fileWriteInfos; bool fileReadInfos; int maxCountWFiles{std::numeric_limits::max()}; int maxCountRFiles{std::numeric_limits::max()}; bool fileStatus{false}; }; struct CommandInfo { // Invalid return value set, if no return value could be determined (e.g. because // the shell-process called execve() before fork static const qint32 INVALID_RETURN_VAL = std::numeric_limits::max(); static CommandInfo fromLocalEnv(); CommandInfo(); qint64 idInDb; QString text; qint32 returnVal; QString username; QString hostname; HashMeta hashMeta; SessionInfo sessionInfo; QDateTime startTime; QDateTime endTime; QString workingDirectory; FileWriteInfos fileWriteInfos; FileReadInfos fileReadInfos; void write(QJsonObject &json, bool withMilliseconds=false, const CmdJsonWriteCfg& writeCfg=CmdJsonWriteCfg(true)) const; bool operator==(const CommandInfo& rhs) const; void clear(); }; ================================================ FILE: src/common/database/db_connection.cpp ================================================ #include #include #include #include #include #include #include #include #include #include "compat.h" #include "db_connection.h" #include "cflock.h" #include "sqlite_database_scheme.h" #include "sqlite_database_scheme_updates.h" #include "qexcdatabase.h" #include "qfilethrow.h" #include "qsqlquerythrow.h" #include "logger.h" #include "app.h" #include "util.h" #include "staticinitializer.h" #include "settings.h" static QSqlDatabase* g_db = nullptr; static bool versionTableExists(QSqlQueryThrow& query){ logDebug << "checking for version table..."; query.exec("SELECT name FROM sqlite_master WHERE type='table' AND name='version'"); return query.next(); } static QVersionNumber queryVersion(QSqlQueryThrow& query){ query.exec("select ver from version"); query.next(true); return QVersionNumber::fromString(query.value(0).toString()); } static void newSqliteDbIfNeeded(){ static StaticInitializer loader( [](){ // maybe_todo: according to documentation of QSqlDatabase, rather // call QSqlDatabase::database() instead of storing the database // ourselves. g_db = new QSqlDatabase(QSqlDatabase::addDatabase("QSQLITE")); if(! g_db->isValid()){ throw QExcDatabase(qtr("Failed to add qt's sqlite database driver. " "Is the driver installed?")); } // give enough time, e.g. for cases where the db is stored on a nfs-drive. g_db->setConnectOptions("QSQLITE_BUSY_TIMEOUT=15000"); }); } static void updateDbScheme(QSqlQueryThrow& query, const QVersionNumber& latestSchemeVer){ const auto dbVersion = queryVersion(query); if(dbVersion == latestSchemeVer){ return; } if(dbVersion > latestSchemeVer){ logWarning << qtr("The database version (%1) is higher than the scheme version " "(%2). Note that downgrades of the database are *not* " "supported, so things may go wrong. Please update shournal " "(on this machine).") .arg(dbVersion.toString()).arg(latestSchemeVer.toString()); return; } // the version is smaller -> perform all necessary updates if(dbVersion < QVersionNumber{0, 9}){ logDebug << "updating db to 0.9..."; sqlite_database_scheme_updates::v0_9(query); } if(dbVersion < QVersionNumber{2, 1}){ logDebug << "updating db to 2.1..."; sqlite_database_scheme_updates::v2_1(query); } if(dbVersion < QVersionNumber{2, 2}){ logDebug << "updating db to 2.2..."; sqlite_database_scheme_updates::v2_2(query); } if(dbVersion < QVersionNumber{2, 4}){ logDebug << "updating db to 2.4..."; sqlite_database_scheme_updates::v2_4(query); } if(dbVersion < QVersionNumber{2, 5}){ logDebug << "updating db to 2.5..."; sqlite_database_scheme_updates::v2_5(query); } query.prepare("replace into version (id, ver) values (1, ?)"); query.addBindValue(latestSchemeVer.toString()); query.exec(); } static void createOrUpDateDb(QSqlQueryThrow &query, const QVersionNumber& latestSchemeVer){ logDebug << "about to lockExclusive database for scheme update..."; // quoting sqlite.org/foreignkeys.html // "It is not possible to enable or disable foreign key constraints in the // middle of a multi-statement transaction (when SQLite is not in autocommit mode)" // The scheme-updates require foreign_keys=OFF, so call below pragma: query.exec("PRAGMA foreign_keys=OFF"); QFileThrow lockfile(db_connection::getDatabaseDir() + "/.shournal-dblock"); lockfile.open(QFile::OpenModeFlag::ReadWrite); // Lock exclusively on scheme update. Note that for some reason concurrent // processes executing "PRAGMA locking_mode=EXCLUSIVE; BEGIN EXCLUSIVE;" deadlocked // during integration tests, so be careful with that directive. CFlock lock(lockfile.handle()); lock.lockExclusive(); query.transaction(); if(! versionTableExists(query)){ logInfo << qtr("Creating new sqlite database"); QStringList statements = QString( SQLITE_DATABASE_SCHEME).split(';', Qt::SkipEmptyParts); for(const QString& stmt : statements){ query.exec(stmt); } QFile dbDir(db_connection::getDatabaseDir()); if(! dbDir.setPermissions( QFileDevice::ReadOwner|QFileDevice::WriteOwner|QFileDevice::ExeOwner)){ logWarning << qtr("Failed to initially set permissions on the database-" "directory at %1: %2. Other users might be able " "to browse your command history...") .arg(db_connection::getDatabaseDir(), dbDir.errorString()); } } updateDbScheme(query, latestSchemeVer); query.commit(); lock.unlock(); // outside of transaction (see above): // Allow for delete queries with cascades query.exec("PRAGMA foreign_keys=ON"); } /// @throws QExcDatabase static void openAndPrepareSqliteDb() { const QString appDataLoc = db_connection::mkDbPath(); const QString dbPath = appDataLoc + "/database.db"; g_db->setDatabaseName(dbPath); if(! g_db->open()) { throw QExcDatabase(__func__, g_db->lastError()); } // Until shournal v3.2 the database version was always set to the application version. // This required a synchronized update of all machines sharing the same database. // Therefore, only update the database version if a scheme update is necessary. auto latestSchemeVer = QVersionNumber{3, 2}; QSqlQueryThrow query(*g_db); if(! versionTableExists(query)){ logDebug << "version table did not exist yet.."; createOrUpDateDb(query, latestSchemeVer); } else { const auto dbVersion = queryVersion(query); logDebug << "current db-version" << dbVersion.toString() << "latestSchemeVer" << latestSchemeVer.toString(); if(dbVersion != latestSchemeVer){ createOrUpDateDb(query, latestSchemeVer); } } // Allow for delete queries with cascades query.exec("PRAGMA foreign_keys=ON"); } QString db_connection::getDatabaseDir(){ auto & sets = Settings::instance(); return sets.dataDir(); } /// @return the created dir QString db_connection::mkDbPath() { const QString & appDataLoc = db_connection::getDatabaseDir(); QDir d(appDataLoc); if( ! d.mkpath(appDataLoc)){ throw QExcIo(qtr("Failed to the create directory for the database at %1") .arg(appDataLoc)); } return appDataLoc; } QueryPtr db_connection::mkQuery() { setupIfNeeded(); return std::make_shared(*g_db); } /// merely for test purposes void db_connection::close() { g_db->close(); } void db_connection::setupIfNeeded() { newSqliteDbIfNeeded(); if(! g_db->isOpen()){ openAndPrepareSqliteDb(); } } ================================================ FILE: src/common/database/db_connection.h ================================================ #pragma once #include #include "qsqlquerythrow.h" typedef std::shared_ptr QueryPtr; namespace db_connection { QString getDatabaseDir(); QString mkDbPath(); void setupIfNeeded(); QueryPtr mkQuery(); void close(); } ================================================ FILE: src/common/database/db_controller.cpp ================================================ #include #include #include #include #include #include #include #include #include "db_controller.h" #include "db_connection.h" #include "db_conversions.h" #include "db_globals.h" #include "qexcdatabase.h" #include "qsqlquerythrow.h" #include "insertifnotexist.h" #include "query_columns.h" #include "logger.h" #include "util.h" #include "cleanupresource.h" #include "storedfiles.h" #include "interrupt_handler.h" #include "os.h" #include "qoutstream.h" using namespace db_conversions; using db_controller::InsertIfNotExist; static void insertFileWriteEvent(const QueryPtr& query, const CommandInfo &cmd, FileEvent* e ) { auto pathFnamePair = splitAbsPath(QString(e->path())); query->prepare(query->insertIgnorePreamble() + " into pathtable (path)" "values (?)"); query->addBindValue(pathFnamePair.first); query->exec(); query->prepare(query->insertIgnorePreamble() + " into writtenFile (cmdId,pathId,name,mtime,size,hash) " "values (?," "(select `id` from pathtable where path=?)," "?,?,?,?)"); query->addBindValue(cmd.idInDb); query->addBindValue(pathFnamePair.first); query->addBindValue(pathFnamePair.second); query->addBindValue(fromMtime(e->mtime())); query->addBindValue(static_cast(e->size())); query->addBindValue(fromHashValue(e->hash())); query->exec(); } /// Move or copy the file captured along the read event e /// to the read files directory in shournal's database dir. static void copyToStoredFiles(const FileEvent* e, const QByteArray& storedFilesDir, const QByteArray& idInDatabase){ const auto fullDestPath = pathJoinFilename(storedFilesDir, idInDatabase); int out_fd = os::open(strDataAccess(fullDestPath), os::OPEN_WRONLY | os::OPEN_CREAT); auto autoCloseOutFd = finally([&out_fd] { close(out_fd); }); try { os::sendfile(out_fd, fileno_unlocked(e->file()), e->fileContentSize(), e->fileContentStart()); } catch (const os::ExcOs& ex) { logWarning << QString("Failed to send file to %1 - %2") .arg(fullDestPath.constData()) .arg(ex.what()); throw; } } static void insertFileReadEvent(const QueryPtr& query, const CommandInfo &cmd, const QVariant& envId, const QVariant& hashMetaId, FileEvent* e ) { StoredFiles storedFiles; const QByteArray storedFilesDir = storedFiles.getReadFilesDir().toUtf8(); const auto pathFnamePair = splitAbsPath(QString(e->path())); query->prepare(query->insertIgnorePreamble() + " into pathtable (path)" "values (?)"); query->addBindValue(pathFnamePair.first); query->exec(); InsertIfNotExist insIfnExist(*query, "readFile"); insIfnExist.addSimple("envId", envId); insIfnExist.addSimple("name", pathFnamePair.second); insIfnExist.addEntry("pathId", {pathFnamePair.first}, "(select id from pathtable where path=?)"); insIfnExist.addSimple("mtime",fromMtime(e->mtime())); insIfnExist.addSimple("size", qint64(e->size())); insIfnExist.addSimple("mode", qint64(e->mode())); insIfnExist.addSimple("hash", fromHashValue(e->hash())); insIfnExist.addSimple("hashmetaId", hashMetaId); insIfnExist.addSimple("isStoredToDisk", e->fileContentSize() > 0); bool existed; const auto readFileId = insIfnExist.exec(&existed); if(! existed && e->fileContentSize() > 0){ copyToStoredFiles(e, storedFilesDir, readFileId.toByteArray()); } query->prepare(query->insertIgnorePreamble() + " into readFileCmd (cmdId, readFileId) values (?,?)"); query->addBindValue(cmd.idInDb); query->addBindValue(readFileId); query->exec(); } /// sql allows for cascade deleting orphans (children), here we kill /// parents, where all children died static void deleteChildlessParents(const QueryPtr& query){ logDebug << "delete from hashmeta..."; query->exec("delete from hashmeta where not exists " "(select 1 from cmd where cmd.hashmetaId=hashmeta.id)"); logDebug << "delete from session..."; query->exec("delete from session where not exists " "(select 1 from cmd where cmd.sessionId=session.id)"); // delete stored read files (script files) in filesystem AND database query->setForwardOnly(true); query->prepare("select readFile.id from readFile where " "readFile.isStoredToDisk=? and " "not exists (select 1 from readFileCmd where readFileCmd.readFileId=readFile.id) "); query->bindValue(0, true); query->exec(); StoredFiles storedFiles; logDebug << "looping though read 'script' files to evtl. delete from filesystem..."; while(query->next()){ const QString fname = query->value(0).toString(); if(! storedFiles.deleteReadFile(fname) ){ logWarning << qtr("failed to remove the file with name %1 " "from the read files dir.").arg(fname); } } logDebug << "delete from readFile..."; query->exec("delete from readFile where not exists " "(select 1 from readFileCmd where readFileCmd.readFileId=readFile.id)"); // Do it last -> foreign key in readFile logDebug << "delete from env..."; query->exec("delete from env where not exists (select 1 from cmd where " "cmd.envId=env.id)"); query->exec("delete from pathtable where not exists " "(select 1 from writtenFile where writtenFile.pathId=pathtable.id) " "and not exists " "(select 1 from readFile where readFile.pathId=pathtable.id)"); } static FileReadInfos queryFileReadInfos(const SqlQuery& sqlQ, const QueryPtr& query_=nullptr, const QString& optionalJoins={}){ const QueryPtr query = (query_ != nullptr) ? query_ : db_connection::mkQuery(); FileReadInfos readInfos; query->prepare("select readFile.id,readFile_path.path,name,mtime,size," "mode,hash,isStoredToDisk from readFile " "join pathtable as readFile_path " "on readFile.pathId=readFile_path.id " + optionalJoins + " where " + sqlQ.query()); query->addBindValues(sqlQ.values()); query->exec(); while(query->next()){ int i=0; FileReadInfo fInfo; fInfo.idInDb = qVariantTo_throw(query->value(i++)); fInfo.path = query->value(i++).toString(); fInfo.name = query->value(i++).toString(); fInfo.mtime = query->value(i++).toDateTime(); fInfo.size = qVariantTo_throw(query->value(i++)); fInfo.mode = qVariantTo_throw(query->value(i++)); fInfo.hash = db_conversions::toHashValue(query->value(i++)); fInfo.isStoredToDisk = query->value(i++).toBool(); readInfos.push_back(fInfo); } return readInfos; } /////////////////////// public //////////////////////////////// /// @return the new command id in database /// @throws QExcDatabase qint64 db_controller::addCommand(const CommandInfo &cmd) { auto query = db_connection::mkQuery(); query->transaction(); query->prepare(query->insertIgnorePreamble() + " into env (hostname, username) values (?,?)"); query->addBindValue(cmd.hostname); query->addBindValue(cmd.username); query->exec(); query->prepare("select id from env where hostname=? and username=?"); query->addBindValue(cmd.hostname); query->addBindValue(cmd.username); query->exec(); query->next(true); const auto envId = qVariantTo_throw(query->value(0)); if(! cmd.hashMeta.isNull()) { query->prepare(query->insertIgnorePreamble() + " into hashmeta (chunkSize, maxCountOfReads) values (?,?)"); query->addBindValue(cmd.hashMeta.chunkSize); query->addBindValue(cmd.hashMeta.maxCountOfReads); query->exec(); } if(! cmd.sessionInfo.uuid.isNull()) { query->prepare(query->insertIgnorePreamble() + " into session (id) values (?)"); query->addBindValue(cmd.sessionInfo.uuid); query->exec(); } query->prepare("insert into cmd (txt,envId,hashmetaId,returnVal," "startTime,endTime,workingDirectory,sessionId) " "values (?,?," "(select id from hashmeta where chunkSize=? and maxCountOfReads=?)," "?,?,?,?,?)" ); query->addBindValue(cmd.text); query->addBindValue(envId); query->addBindValue(cmd.hashMeta.chunkSize); query->addBindValue(cmd.hashMeta.maxCountOfReads); query->addBindValue(cmd.returnVal); query->addBindValue(cmd.startTime); query->addBindValue(cmd.endTime); query->addBindValue(cmd.workingDirectory); query->addBindValue(cmd.sessionInfo.uuid); query->exec(); return qVariantTo_throw(query->lastInsertId()); } /// update only relevant command fields, which are those that are not /// known from the beginning. void db_controller::updateCommand(const CommandInfo &cmd) { assert(cmd.idInDb != db::INVALID_INT_ID); auto query = db_connection::mkQuery(); query->prepare("update cmd set txt=?,returnVal=?,startTime=?,endTime=? " "where `id`=?"); query->addBindValue(cmd.text); query->addBindValue(cmd.returnVal); query->addBindValue(cmd.startTime); query->addBindValue(cmd.endTime); query->addBindValue(cmd.idInDb); query->exec(); } /// Add file events belonging to param cmd which must belong to a valid /// database entry (idInDb must valid) void db_controller::addFileEvents(const CommandInfo &cmd, FileEvents &fileEvents) { assert(cmd.idInDb != db::INVALID_INT_ID); assert(ftell(fileEvents.file()) == 0); auto query = db_connection::mkQuery(); query->transaction(); query->prepare("select envId,hashmetaId from cmd where `id`=?"); query->addBindValue(cmd.idInDb); query->exec(); query->next(true); const QVariant envId = query->value(0); const QVariant hashMetaId = query->value(1); FileEvent* e; uint counter = 0; InterruptProtect ip(SIGTERM); while ((e = fileEvents.read()) != nullptr) { if(FileEvents::isReadEvent(e->flags())){ insertFileReadEvent(query, cmd, envId, hashMetaId, e); } if(FileEvents::isWriteEvent(e->flags())){ insertFileWriteEvent(query, cmd, e); } // Be 'nice' to others and sleep a bit every now and then if(++counter % 500 == 0 && ! ip.signalOccurred()){ // if we shall terminate don't sleep. query->commit(); usleep(10 * 1000); query->transaction(); } } } /// Deletes the command and corresponding file events (read and write). /// @param sqlQuery: may only refer to columns of the 'cmd'-table. /// @returns numRowsAffected int db_controller::deleteCommand(const SqlQuery &sqlQuery) { auto query = db_connection::mkQuery(); query->transaction(); logDebug << "deleting cmd" << sqlQuery.query(); query->prepare("delete from cmd where " + sqlQuery.query()); query->addBindValues(sqlQuery.values()); query->exec(); int numRowsAffected = query->numRowsAffected(); // the respective triggers have also caused the deletion of orphans in // writtenFile, readFileCmd, etc., however, we still need to handle childless parents: deleteChildlessParents(query); return numRowsAffected; } /// @param reverseResultIter: if true, the returned Iterator will traverse the resultset in /// reverse order on continous 'next'-calls. std::unique_ptr db_controller::queryForCmd(const SqlQuery &sqlQ, bool reverseResultIter){ auto pQuery = db_connection::mkQuery(); std::unique_ptr cmdIter( new CommandQueryIterator(pQuery, reverseResultIter)); const QString queryStr = "select cmd.id,cmd.txt," "cmd.returnVal,cmd.startTime,cmd.endTime,cmd.workingDirectory," "session.id,session.comment," "hashmeta.chunkSize,hashmeta.maxCountOfReads," "env.username,env.hostname " "from cmd " + QString((sqlQ.containsTablename("writtenFile") || sqlQ.containsTablename("writtenFile_path")) ? // an alias "join writtenFile on cmd.id=writtenFile.cmdId " "join pathtable as writtenFile_path " "on writtenFile.pathId=writtenFile_path.id " : "") + QString((sqlQ.containsTablename("readFile") || sqlQ.containsTablename("readFile_path")) ? "join readFileCmd on cmd.id=readFileCmd.cmdId " "join readFile on readFileCmd.readFileId=readFile.id " "join pathtable as readFile_path " "on readFile.pathId=readFile_path.id " : "") + "join env on cmd.envId=env.id " "left join hashmeta on hashmeta.id=cmd.hashmetaId " // left joins last, if possible! "left join `session` on cmd.sessionId=session.id " "where "; // do not change this -> order matters in html-plot... const QString orderBy = "order by cmd.startTime " + sqlQ.ascendingStr() + sqlQ.mkLimitString(); // we need the size (at other places) but QSQLITE does not support QSqlQuery::size. // To use a workaround, forward mode must not be enabled. // See also https://stackoverflow.com/a/26500811/7015849 // if( ! reverseResultIter){ // pQuery->setForwardOnly(true); // } const QString fullQuery = queryStr + sqlQ.query() + " group by cmd.id " + orderBy; pQuery->prepare(fullQuery); pQuery->addBindValues(sqlQ.values()); logDebug << "executing" << fullQuery; pQuery->exec(); if(reverseResultIter){ // place cursor right after the last record, so a call to "previous" points to last. pQuery->last(); pQuery->next(); } return cmdIter; } /// if no entry can be found, the id of the returned file info is invalid. FileReadInfo db_controller::queryReadInfo_byId(const qint64 id, const QueryPtr& query_) { SqlQuery sqlQ; sqlQ.addWithAnd("readFile.id", id); auto fileReadInfos = queryFileReadInfos(sqlQ, query_); if(fileReadInfos.isEmpty()){ return FileReadInfo(); } assert(fileReadInfos.size() == 1); return fileReadInfos.first(); } FileReadInfos db_controller::queryReadInfos_byCmdId(qint64 cmdId, const QueryPtr &query_) { SqlQuery sqlQ; sqlQ.addWithAnd("cmdId", cmdId); return queryFileReadInfos(sqlQ, query_, " join readFileCmd on " "readFile.id=readFileCmd.readFileId "); } /// @param restrictingFilesize: only return hashmeta-entries for which at least one file /// exists which was recorded using a given hashmeta and whose size is exactly that. /// Set to -1 to return all HashMeta entries. /// @param isReadFile: if true, consider read files, else written. Only used, if /// restrictingFilesize!=-1 db_controller::HashMetas db_controller::queryHashmetas(qint64 restrictingFilesize, bool isReadFile){ const QString FIELDS = " chunkSize,maxCountOfReads,hashmeta.id "; QString sql; if(restrictingFilesize == -1){ sql = "select "+FIELDS+" from `hashmeta`"; } else { sql = (isReadFile) ? "select "+FIELDS+" from `readFile` " "left join hashmeta on readFile.hashmetaId=hashmeta.id " "where readFile.size=? " "group by chunkSize,maxCountOfReads " : "select "+FIELDS+" from cmd " "join `writtenFile` on cmd.id=writtenFile.cmdId " "left join hashmeta on cmd.hashmetaId=hashmeta.id " "where writtenFile.size=? " "group by chunkSize,maxCountOfReads "; } auto query = db_connection::mkQuery(); query->prepare(sql); query->addBindValue(restrictingFilesize); query->exec(); db_controller::HashMetas hashMetas; bool noHashAdded = false; while(query->next()){ if( query->value(0).isNull()){ if( ! noHashAdded){ hashMetas.push_back(HashMeta()); noHashAdded = true; } } else { HashMeta h; qVariantTo_throw(query->value(0), &h.chunkSize); qVariantTo_throw(query->value(1), &h.maxCountOfReads); qVariantTo_throw(query->value(2), &h.idInDb); hashMetas.push_back(h); } } return hashMetas; } /// Find the database id of a given hasmeta entry (by chunkSize and maxCountOfReads) qint64 db_controller::queryHashmetaId(const HashMeta &hashMeta ) { qint64 idIndDb = db::INVALID_INT_ID; auto query = db_connection::mkQuery(); query->prepare("select `id` from hashmeta where " "chunkSize=? and maxCountOfReads=?"); query->addBindValue(hashMeta.chunkSize); query->addBindValue(hashMeta.maxCountOfReads); query->exec(); if(query->next()){ qVariantTo_throw(query->value(0), &idIndDb); } return idIndDb; } ================================================ FILE: src/common/database/db_controller.h ================================================ #pragma once #include #include #include #include "fileevents.h" #include "commandinfo.h" #include "sqlquery.h" #include "db_connection.h" #include "qsqlquerythrow.h" #include "command_query_iterator.h" namespace db_controller { typedef QVector HashMetas; qint64 addCommand(const CommandInfo &cmd); void updateCommand(const CommandInfo &cmd); void addFileEvents(const CommandInfo &cmd, FileEvents& fileEvents); int deleteCommand(const SqlQuery &query); std::unique_ptr queryForCmd(const SqlQuery& sqlQ, bool reverseResultIter=false); FileReadInfo queryReadInfo_byId(qint64 id, const QueryPtr& query_=nullptr); FileReadInfos queryReadInfos_byCmdId(qint64 cmdId, const QueryPtr& query_=nullptr); HashMetas queryHashmetas(qint64 restrictingFilesize=-1, bool isReadFile=false); qint64 queryHashmetaId(const HashMeta&); } ================================================ FILE: src/common/database/db_conversions.cpp ================================================ #include #include "db_conversions.h" #include "util.h" QVariant db_conversions::fromMtime(time_t mtime) { return QVariant(QDateTime::fromTime_t(static_cast(mtime))); } /// sqlite cannot store uint64 as int - store as blob instead. QVariant db_conversions::fromHashValue(const HashValue &val) { QByteArray hashBytes(val.isNull() ? "" : qBytesFromVar(val.value())); return { hashBytes }; } HashValue db_conversions::toHashValue(const QVariant &var) { QByteArray hashBytes = var.toByteArray(); if(hashBytes.isEmpty()){ return {}; } return varFromQBytes(hashBytes); } ================================================ FILE: src/common/database/db_conversions.h ================================================ #pragma once #include #include #include "nullable_value.h" namespace db_conversions { QVariant fromMtime(time_t mtime); // Not toMtime, because we work with QDateTime afterwards QVariant fromHashValue(const HashValue& val); HashValue toHashValue(const QVariant& var); } ================================================ FILE: src/common/database/db_globals.cpp ================================================ #include "db_globals.h" // SQLITE begins int-ids at one. const qint64 db::INVALID_INT_ID = 0; ================================================ FILE: src/common/database/db_globals.h ================================================ #pragma once #include namespace db { extern const qint64 INVALID_INT_ID; } ================================================ FILE: src/common/database/file_query_helper.cpp ================================================ #include #include #include #include "db_controller.h" #include "db_conversions.h" #include "db_globals.h" #include "exccommon.h" #include "file_query_helper.h" #include "hashcontrol.h" #include "logger.h" #include "os.h" #include "qfilethrow.h" #include "query_columns.h" #include "settings.h" #include "translation.h" using std::vector; using db_controller::QueryColumns; using namespace db_conversions; struct FileQueryColumns { FileQueryColumns(bool readFile) : readFile(readFile){ // readFile, else writtenFile QueryColumns& c = QueryColumns::instance(); if(readFile){ col_hash = c.rFile_hash; col_size = c.rFile_size; col_mtime = c.rFile_mtime; col_hashMetaId = c.rFile_hashmetaId; } else { col_hash = c.wFile_hash; col_size = c.wFile_size; col_mtime = c.wFile_mtime; col_hashMetaId = c.cmd_hashmetaId; } } QString col_hash; QString col_size; QString col_mtime; QString col_hashMetaId; bool readFile; }; /// Strictly speaking a given hash for a file is /// only valid in combination with it's hash-settings as those determine what /// parts of a file to hash. struct HashMetaValuePair { HashMeta meta; HashValue value; }; static void addToHashQuery(SqlQuery& query, const HashValue& hashVal, const QVariant& hashMetaId, const FileQueryColumns& c){ // Hash-values and hashmeta are paired, so e.g. // (hashval=123 and hashMetaId=1) or (hashval=35 and hashMetaId=2). // If we query for a command, where hashing was disabled, the hash // naturally has be null as well. if(hashMetaId.isNull() && ! hashVal.isNull()){ throw QExcProgramming("hashMetaId.isNull && ! hashVal.isNull"); } SqlQuery subquery; subquery.addWithAnd(c.col_hash, fromHashValue(hashVal)); subquery.addWithAnd(c.col_hashMetaId, hashMetaId); query.addWithOr(subquery); } static void addToHashQuery(SqlQuery& query, const HashValue& hashVal, const HashMeta& hashMeta, const FileQueryColumns& c){ QVariant hashMetaId; if(! hashMeta.isNull()){ hashMetaId = hashMeta.idInDb; } addToHashQuery(query, hashVal, hashMetaId, c); } static void addToHashQuery(SqlQuery& query, const HashMetaValuePair& p, const FileQueryColumns& c){ addToHashQuery(query, p.value, p.meta, c); } static void addToHashQuery(SqlQuery& query, const vector& hashMetaValuePairs, const FileQueryColumns& c){ for(const auto& p : hashMetaValuePairs){ addToHashQuery(query, p, c); } } /// Generate all necessary HashMeta-HashValue-pairs for the given fd, /// ignoring a possibly existing knownPair. static vector generateHashMetaValuePairs(QFileThrow& file, qint64 filesize, const FileQueryColumns* c=nullptr, const HashMetaValuePair* knownPair=nullptr){ if(filesize == 0){ return {}; } const auto hashMetas = (c==nullptr) ? db_controller::queryHashmetas() : db_controller::queryHashmetas(filesize, c->readFile); HashControl hashCtrl; vector pairs; for(const auto& hashMeta : hashMetas){ HashValue hashVal; if(! hashMeta.isNull()){ if(knownPair != nullptr && hashMeta.idInDb == knownPair->meta.idInDb){ // already got this one. continue; } hashVal = hashCtrl.genPartlyHash(file.handle(), filesize, hashMeta); if(hashVal.isNull()){ throw QExcIo(qtr("file %1 - failed to hash, although it " "was not empty.").arg(file.fileName())); } } pairs.push_back({hashMeta, hashVal}); } return pairs; } static bool entriesExists(const SqlQuery& query){ return db_controller::queryForCmd(query)->next(); } /// Typically, changing hash-settings is a rare event, /// so optimistically generate a hash using current settings (if /// enabled) static HashMetaValuePair goodLuckHashAttempt(QFileThrow& file, const qint64 size){ if(size == 0){ return {}; } const auto &sets = Settings::instance(); auto hashMeta = sets.hashSettings().hashMeta; HashValue hashVal; HashControl hashCtrl; if(! sets.hashSettings().hashEnable || hashMeta.isNull()){ return {}; } hashMeta.idInDb = db_controller::queryHashmetaId(hashMeta); if(hashMeta.idInDb == db::INVALID_INT_ID){ logDebug << "unusual event: hashmeta settings not found in db"; return {}; } hashVal = hashCtrl.genPartlyHash(file.handle(), size, hashMeta); if(hashVal.isNull()){ // no need to print a warning here - is caught in // generateHashMetaValuePairs return {}; } HashMetaValuePair p; p.meta = hashMeta; p.value = hashVal; return p; } /// Build a database-query based on the following file-attributes, which are collected /// automatically: size, hash and mtime. The query attempts to be "smart", by /// preferably returning more likely matches, lowering the strictness if nothing /// is found. /// It is looked preliminarily, if a file exactly matching the specs and using *only* /// the current hash-settings exists. If nothing is found, other hash-settings (if any) /// are used to calculate the other hashes. /// If no entry was found, the query is set to ignore the mtime. /// Empty files (size==0) are a special case: their hash is always null, so we can /// ommit the hashing altogether. Over the time quite a number of empty files /// may exist, so in this case we do **not** ignore the mtime and return 100k results. /// In legacy shournal versions, for written files an mtime /// not later than the file's current /// mtime was set in the assumption that changing the mtime afterwards should /// only increase it. However, for example wget uses the /// «Last-Modified header» for HTTP if available, /// which is naturally older than the system's mtime of the just downloaded /// file. /// @param filename: existing file, where attributes are collected from /// @param readFile: if true, query for read files, else for written files SqlQuery file_query_helper::buildFileQuerySmart(const QString &filename, bool readFile) { FileQueryColumns c(readFile); QFileThrow file(filename); file.open(QFile::OpenModeFlag::ReadOnly); auto st_ = os::fstat(file.handle()); const QVariant mtimeVar = fromMtime(st_.st_mtime); const qint64 size = st_.st_size; if(size == 0){ SqlQuery query; query.addWithAnd(c.col_size, size); query.addWithAnd(c.col_mtime, mtimeVar); return query; } vector hashMetaValuePairs; auto firstHashRes = goodLuckHashAttempt(file, size); if(! firstHashRes.meta.isNull()){ SqlQuery query; addToHashQuery(query, firstHashRes, c); query.addWithAnd(c.col_size, size); query.addWithAnd(c.col_mtime, mtimeVar); if(entriesExists(query)){ return query; } } // Our goodluck first attempt failed (bad size, hash or mtime) - now query based // on all other hashMetaValuePairs hashMetaValuePairs = generateHashMetaValuePairs(file, size, &c, &firstHashRes); if(firstHashRes.meta.isNull() && hashMetaValuePairs.empty()){ logDebug << filename << "no file with matching size exists"; return mkInertSqlQuery(); } if(! hashMetaValuePairs.empty()){ SqlQuery query; addToHashQuery(query, hashMetaValuePairs, c); query.addWithAnd(c.col_size, size); query.addWithAnd(c.col_mtime, mtimeVar); if(entriesExists(query)){ return query; } } // We failed to find a match with exact mtime, so, finally, perform the // query ignoring the mtime. Note that our first hash-result (if any) // is not part of hashMetaValuePairs yet. if(! firstHashRes.meta.isNull()){ hashMetaValuePairs.push_back(firstHashRes); } logDebug << "will perform query on" << filename << "ignoring mtime"; SqlQuery query; addToHashQuery(query, hashMetaValuePairs, c); query.addWithAnd(c.col_size, size); return query; } /// Build a database-query based on the following file-attributes, which are collected /// automatically: size, hash or mtime. Any combination of those can be used. /// @param filename: existing file, where attributes are collected from /// @param readFile: if true, query for read files, else for written file SqlQuery file_query_helper::buildFileQuery(const QString &filename, bool readFile, bool use_mtime, bool use_hash, bool use_size) { FileQueryColumns c(readFile); SqlQuery query; QFileThrow file(filename); file.open(QFile::OpenModeFlag::ReadOnly); const auto st_ = os::fstat(file.handle()); const QVariant mtimeVar = fromMtime(st_.st_mtime); const qint64 size = st_.st_size; if(use_mtime) query.addWithAnd(c.col_mtime, mtimeVar); if(use_hash){ if(size == 0){ if(! use_mtime && ! use_size){ logWarning << qtr("File %1 is empty, so hash-only queries are " "not possible.").arg(filename); return mkInertSqlQuery(); } } else { auto hashMetaValuePairs = generateHashMetaValuePairs( file, use_size, (use_size)? &c : nullptr); SqlQuery hashQuery; addToHashQuery(hashQuery, hashMetaValuePairs, c); query.addWithAnd(hashQuery); } } if(use_size) query.addWithAnd(c.col_size, qint64(size)); return query; } ================================================ FILE: src/common/database/file_query_helper.h ================================================ #pragma once #include #include "sqlquery.h" #include "nullable_value.h" #include "fileinfos.h" namespace file_query_helper { SqlQuery buildFileQuerySmart(const QString& filename, bool readFile); SqlQuery buildFileQuery(const QString& filename, bool readFile, bool use_mtime, bool use_hash, bool use_size); } ================================================ FILE: src/common/database/fileinfos.cpp ================================================ #include "fileinfos.h" #include "db_conversions.h" #include "commandinfo.h" #include "hashcontrol.h" #include "logger.h" #include "qfilethrow.h" #include "util.h" FileInfo::~FileInfo() {}; QString FileInfo::currentStatus(const CommandInfo &cmd) const { auto filename = pathJoinFilename(this->path, this->name); try{ QFileThrow f(filename); if(!f.exists()){ return "N"; } HashControl hashCtrl; f.open(QFile::OpenModeFlag::ReadOnly); const auto st_ = os::fstat(f.handle()); if(size != st_.st_size || QDateTime::fromTime_t(static_cast(st_.st_mtime))!= mtime || hash!= hashCtrl.genPartlyHash(f.handle(), st_.st_size, cmd.hashMeta, false)){ return "M"; } return "U"; } catch (const std::exception& ex) { logWarning << qtr("Failed to determine status of file %1 - %2").arg(filename) .arg(QString(ex.what())); return "ERROR"; } } void FileWriteInfo::write(QJsonObject &json) const { json["id"] = idInDb; json["path"] = pathJoinFilename(path, name); json["size"] = size; json["mtime"] = QJsonValue::fromVariant(mtime); json["hash"] = QJsonValue::fromVariant(QVariant::fromValue(hash)); } bool FileWriteInfo::operator==(const FileInfo &rhs) const { if(idInDb != db::INVALID_INT_ID && rhs.idInDb != db::INVALID_INT_ID){ return idInDb == rhs.idInDb; } return mtime == rhs.mtime && size == rhs.size && path == rhs.path && name == rhs.name && hash == rhs.hash; } //////////////////////////////////////////////////////////// void FileReadInfo::write(QJsonObject &json) const { json["id"] = idInDb; json["path"] = pathJoinFilename(path, name); json["size"] = size; json["mtime"] = QJsonValue::fromVariant(mtime); // Note: in case of a non-null hash, this results in a quoted string. // While useful in the html-export (javascript INT-limit..), technically this is not totally // correct. However, it has always been so, so do not change. json["hash"] = QJsonValue::fromVariant(QVariant::fromValue(hash)); json["isStoredToDisk"] = isStoredToDisk; } bool FileReadInfo::operator==(const FileReadInfo &rhs) const { if(idInDb != db::INVALID_INT_ID && rhs.idInDb != db::INVALID_INT_ID){ return idInDb == rhs.idInDb; } return mtime == rhs.mtime && size == rhs.size && path == rhs.path && name == rhs.name && mode == rhs.mode && hash == rhs.hash; } bool FileReadInfo::operator==(const FileInfo&) const { throw QExcProgramming("Unimplemented FileReadInfo::operator==(const FileInfo &rhs)"); } ================================================ FILE: src/common/database/fileinfos.h ================================================ #pragma once #include #include #include #include "nullable_value.h" #include "db_globals.h" struct CommandInfo; struct FileInfo { virtual ~FileInfo() = 0; qint64 idInDb { db::INVALID_INT_ID }; QDateTime mtime; qint64 size {}; QString path; QString name; HashValue hash; virtual QString currentStatus(const CommandInfo &cmd) const; virtual void write(QJsonObject &json) const = 0; virtual bool operator==(const FileInfo& rhs) const = 0 ; }; struct FileWriteInfo : public FileInfo { virtual void write(QJsonObject &json) const; virtual bool operator==(const FileInfo& rhs) const; }; struct FileReadInfo : public FileInfo { mode_t mode {}; bool isStoredToDisk {false}; virtual void write(QJsonObject &json) const; virtual bool operator==(const FileReadInfo& rhs) const; virtual bool operator==(const FileInfo& rhs) const; }; ================================================ FILE: src/common/database/insertifnotexist.cpp ================================================ #include "insertifnotexist.h" #include "qsqlquerythrow.h" db_controller::InsertIfNotExist::InsertIfNotExist(QSqlQueryThrow &parentQuery, const QString &tablename) : m_query(parentQuery), m_tablename(tablename) {} void db_controller::InsertIfNotExist::addSimple(const QString &colname, const QVariant &value) { InsertIfNotExist::addEntry(colname, {value}, "?"); } /// Add a columnname-value pair for the prospective selcet/insert queuey. /// @param colname: Column-name. /// @param values: value-list for QSqlQuery::addBindValue. In most cases /// only one value is supplied, however, multiple values are possible to /// allow for sub-queries. /// @param placeholder: In simple cases the placeholder is ?, e.g. /// «colname is ?», however, for sub-queries this may expand to e.g. /// «(select id from table2 where foo is ? and bar is ?)». void db_controller::InsertIfNotExist:: addEntry(const QString &colname, const QVariantList &values, const QString &placeholder) { InsertIfNotExistEntry e; e.colname = colname; e.placeholder = placeholder; m_entries.push_back(e); for(const auto& val : values){ m_values.push_back(val); } } /// Execute the insert-if-not exist query using the /// previously added column-value-pairs. /// @param existed: If non-null, set it to true, if /// the entry already existed (so no insert was necessary). /// @return: the existing or newly created id QVariant db_controller::InsertIfNotExist::exec(bool *existed) { QString query = "select id from " + m_tablename + " where "; bool first = true; for(const auto& entry : m_entries){ if(! first){ query += " and "; } first = false; query += entry.colname + " is " + entry.placeholder; } m_query.prepare(query); m_query.addBindValues(m_values); m_query.exec(); bool nextSuccess = m_query.next(); if(existed != nullptr){ *existed = nextSuccess; } if(nextSuccess){ return m_query.value(0); } // record did not exist, insert it query = "insert into " + m_tablename + " ("; first = true; QString placeholders('('); for(const auto& entry : m_entries){ if(! first){ query += ','; placeholders += ','; } first = false; query += entry.colname; placeholders += entry.placeholder; } query += ')'; placeholders += ')'; query += " values " + placeholders; m_query.prepare(query); m_query.addBindValues(m_values); m_query.exec(); return m_query.lastInsertId(); } ================================================ FILE: src/common/database/insertifnotexist.h ================================================ #pragma once #include #include class QSqlQueryThrow; namespace db_controller { /// Insert values in a sql table, if these values do not already exist. /// Requires an existing column named `id`. /// If the value-combination does not exist, the insert-operation inserts /// these values. class InsertIfNotExist { public: InsertIfNotExist(QSqlQueryThrow& parentQuery, const QString& tablename); void addSimple(const QString& colname, const QVariant& value); void addEntry(const QString& colname, const QVariantList& values, const QString& placeholder); QVariant exec(bool* existed=nullptr); private: struct InsertIfNotExistEntry { QString colname; QString placeholder; }; QSqlQueryThrow& m_query; QString m_tablename; QVector m_entries; QVariantList m_values; }; } ================================================ FILE: src/common/database/qexcdatabase.cpp ================================================ #include "qexcdatabase.h" QExcDatabase::QExcDatabase(const QString &preamble, const QSqlError &err) : QExcCommon (preamble) { if(! descrip().isEmpty()){ setDescrip( descrip() + ": "); } setDescrip( descrip() + err.text() + '('+ err.nativeErrorCode() + ')'); } QExcDatabase::QExcDatabase(const QString &preamble) : QExcCommon (preamble) { } ================================================ FILE: src/common/database/qexcdatabase.h ================================================ #include #include "exccommon.h" class QExcDatabase : public QExcCommon { public: QExcDatabase(const QString & preamble, const QSqlError & err); QExcDatabase(const QString & preamble); }; ================================================ FILE: src/common/database/qsqlquerythrow.cpp ================================================ #include #include #include #include #include "logger.h" #include "osutil.h" #include "qsqlquerythrow.h" #include "qexcdatabase.h" #include "util.h" enum SQLITE_ERR { SQLITE_ERR_BUSY= 5 }; // // QSqlQueryThrow::QSqlQueryThrow(QSqlResult *r) // :QSqlQuery (r), // m_execWasCalled(false) // {} // // QSqlQueryThrow::QSqlQueryThrow(const QString &query, const QSqlDatabase& db) // :QSqlQuery(query, db), // m_execWasCalled(false) // {} // static QString mkInsertIgnorePreamble(const QString& driverName) { static const QHash preambles { {"QSQLITE", "insert or ignore"} }; auto it = preambles.find(driverName); if(it != preambles.end()){ return it.value(); } return "insert ignore"; } static int sqlerrToNumber(const QSqlError & err){ try { return qVariantTo_throw(err.nativeErrorCode()); } catch (ExcQVariantConvert& ex) { ex.setDescrip("Failed to convert Sqlerror to number - " + ex.descrip()); throw; } } QSqlQueryThrow::QSqlQueryThrow(const QSqlDatabase& db) : QSqlQuery (db), m_insertIgnorePreamble(mkInsertIgnorePreamble(db.driverName())), m_execWasCalled(false), m_withinTransaction(false) {} QSqlQueryThrow::~QSqlQueryThrow() { if(! m_withinTransaction){ return; } try { if (std::uncaught_exception()) { this->rollback(); } else { this->commit(); } } catch (const std::exception& e) { fprintf(stderr, "%s: %s\n", __func__, e.what()); } } void QSqlQueryThrow::exec() { this->_doExec(QString()); } void QSqlQueryThrow::exec(const QString &query) { this->_doExec(query); } void QSqlQueryThrow::prepare(const QString &query) { if(! QSqlQuery::prepare(query)){ throw QExcDatabase(qtr("prepare <%1> failed").arg(query), this->lastError()); } m_execWasCalled = false; } bool QSqlQueryThrow::next(bool throwIfEmpty) { if(! m_execWasCalled){ throw QExcDatabase(QString("%1 was called without previous exec ") .arg(__func__)); } bool ret = QSqlQuery::next(); if(throwIfEmpty && ! ret){ throw QExcDatabase(qtr("The query %1 was expected to have (another) result which is " "not the case").arg(this->lastQuery())); } return ret; } void QSqlQueryThrow::addBindValues(const QVariantList &vals) { for(const auto& val : vals){ this->addBindValue(val); } } /// Note: while qt's QSqlDatabase starts transactions in SQLITE in 'deferred' mode, /// we rather choose 'immediate'. See also /// https://www.sqlite.org/lang_transaction.html /// and /// https://stackoverflow.com/a/1063768 /// for the rationale. void QSqlQueryThrow::transaction() { assert(! m_withinTransaction); this->exec("BEGIN IMMEDIATE"); m_withinTransaction = true; } void QSqlQueryThrow::commit() { assert(m_withinTransaction); m_withinTransaction = false; this->exec("COMMIT"); } void QSqlQueryThrow::rollback() { assert(m_withinTransaction); m_withinTransaction = false; this->exec("ROLLBACK"); } /// QSQLITE does not support size(), this is a workaround /// which only works if forwardOnly is false. int QSqlQueryThrow::computeSize() { if(this->isForwardOnly()){ throw QExcDatabase(qtr("attempted to compute size although forwardOnly " "is enabled.")); } // see also https://stackoverflow.com/a/26500811/7015849 const int initialPos = this->at(); const int size = (this->last()) ? this->at() + 1 : 0; // restore initial pos switch (initialPos) { case QSql::BeforeFirstRow: this->first(); this->previous(); break; case QSql::AfterLastRow: this->last(); this->next(); break; default: this->seek(initialPos); break; } return size; } QString QSqlQueryThrow::generateExcMsgExec(const QString &queryStr) { QStringList vals; for(const auto& entry : this->boundValues()){ vals.push_back(entry.value().toString()); } QString valStr; if(! vals.isEmpty()){ valStr = " with values <" + vals.join(", ") + ">"; } QString msg = "exec <" + queryStr + ">" + valStr + " failed"; return msg; } void QSqlQueryThrow::_doExec(const QString &query) { for(int i=0; i<10; i++){ bool success = query.isEmpty() ? QSqlQuery::exec() : QSqlQuery::exec(query); if(success){ m_execWasCalled = true; return; } if(sqlerrToNumber(this->lastError()) == SQLITE_ERR_BUSY){ logInfo << "Sqlquery failed with busy timeout. trying again in a " "few seconds:" << (query.isEmpty()?this->lastQuery():query) ; osutil::randomSleep(5 *1000, 20 *1000); } else { // throw immediatly (below) break; } } throw QExcDatabase(generateExcMsgExec(query.isEmpty()?this->lastQuery():query), this->lastError()); } const QString &QSqlQueryThrow::insertIgnorePreamble() const { return m_insertIgnorePreamble; } ================================================ FILE: src/common/database/qsqlquerythrow.h ================================================ #pragma once #include #include #include class QSqlQueryThrow : public QSqlQuery { public: // explicit QSqlQueryThrow(QSqlResult *r); // explicit QSqlQueryThrow(const QString& query = QString(), const QSqlDatabase& db = QSqlDatabase()); explicit QSqlQueryThrow(const QSqlDatabase& db); ~QSqlQueryThrow(); void exec(); void exec(const QString& query); void prepare(const QString& query); bool next(bool throwIfEmpty=false); void addBindValues(const QVariantList& vals); void transaction(); void commit(); void rollback(); int computeSize(); public: typedef QVector > ColnameValuePairs; const QString& insertIgnorePreamble() const; public: // disable-copies: transactions cannot be copied... QSqlQueryThrow(const QSqlQueryThrow &) = delete ; void operator=(const QSqlQueryThrow &) = delete ; private: QString generateExcMsgExec(const QString& queryStr); void _doExec(const QString& query); QString m_insertIgnorePreamble; bool m_execWasCalled; bool m_withinTransaction; }; ================================================ FILE: src/common/database/query_columns.h ================================================ #pragma once #include #include "util.h" namespace db_controller { class QueryColumns { public: static QueryColumns& instance() { static QueryColumns s_instance; return s_instance; } const QString cmd_id {"cmd.id"}; const QString cmd_txt {"cmd.txt"}; const QString cmd_workingDir {"cmd.workingDirectory"}; const QString cmd_comment {"cmd.comment"}; const QString cmd_endtime {"cmd.endTime"}; const QString cmd_starttime {"cmd.startTime"}; const QString cmd_hashmetaId {"cmd.hashmetaId"}; const QString env_hostname {"env.hostname"}; const QString env_username {"env.username"}; const QString rFile_name {"readFile.name"}; const QString rFile_path {"readFile_path.path"}; // separate table, join alias const QString rFile_mtime {"readFile.mtime"}; const QString rFile_size {"readFile.size"}; const QString rFile_hash {"readFile.hash"}; const QString rFile_hashmetaId {"readFile.hashmetaId"}; const QString wFile_id {"writtenFile.id"}; const QString wFile_name {"writtenFile.name"}; const QString wFile_mtime {"writtenFile.mtime"}; const QString wFile_size {"writtenFile.size"}; const QString wFile_hash {"writtenFile.hash"}; const QString wFile_path {"writtenFile_path.path"}; // separate table, join alias const QString session_id {"session.id"}; const QString session_comment {"session.comment"}; private: QueryColumns() = default; public: ~QueryColumns() = default; Q_DISABLE_COPY(QueryColumns) DEFAULT_MOVE(QueryColumns) }; } ================================================ FILE: src/common/database/sessioninfo.cpp ================================================ #include "sessioninfo.h" bool SessionInfo::operator==(const SessionInfo &rhs) const { return uuid == rhs.uuid && comment == rhs.comment; } ================================================ FILE: src/common/database/sessioninfo.h ================================================ #pragma once #include struct SessionInfo { QByteArray uuid; QString comment; bool operator==(const SessionInfo& rhs) const; }; ================================================ FILE: src/common/database/sqlite_database_scheme.cpp ================================================ #include "sqlite_database_scheme.h" // Note: this is the initial scheme, please don't change it. // To add new stuff (tables/columns/indexes) please do that in // sqlite_database_scheme_updates.cpp const char* SQLITE_DATABASE_SCHEME = R"SOMERANDOMTEXT( CREATE TABLE IF NOT EXISTS `version` ( `id` INTEGER PRIMARY KEY, `ver` TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS `env` ( `id` INTEGER, `username` TEXT NOT NULL, `hostname` TEXT NOT NULL, PRIMARY KEY(`id`), CONSTRAINT unq UNIQUE (username, hostname) ); CREATE TABLE IF NOT EXISTS `hashmeta` ( `id` INTEGER , `chunkSize` INTEGER NOT NULL, `maxCountOfReads` INTEGER NOT NULL, PRIMARY KEY(`id`), CONSTRAINT unq UNIQUE (`chunkSize`,`maxCountOfReads`) ); CREATE TABLE IF NOT EXISTS `cmd` ( `id` INTEGER, `sessionId` BLOB references session(id), `envId` INTEGER NOT NULL references env(id), `hashmetaId` INTEGER, /* NULL-able because hash may be disabled */ `txt` TEXT NOT NULL, `returnVal` INTEGER NOT NULL, `startTime` timestamp NOT NULL, `endTime` timestamp NOT NULL, `workingDirectory` TEXT NOT NULL, PRIMARY KEY(`id`) ); CREATE TABLE IF NOT EXISTS `file` ( `id` INTEGER, `path` TEXT NOT NULL, `name` TEXT NOT NULL, `cmdId` INTEGER NOT NULL references cmd(id) ON DELETE CASCADE, `mtime` timestamp NOT NULL, `size` INTEGER NOT NULL, `hash` BLOB, /* 64 bit unsigned int, so use blob... */ PRIMARY KEY(`id`) ); CREATE TABLE IF NOT EXISTS `exeMeta` ( `id` INTEGER, `envId` INTEGER NOT NULL references `env`(id), `exepath` TEXT NOT NULL, PRIMARY KEY(`id`) ); CREATE TABLE IF NOT EXISTS `exeFile` ( `id` INTEGER, `exeMetaId` INTEGER NOT NULL references `exeMeta`(id), `name` TEXT NOT NULL, `mtime` timestamp NOT NULL, `size` INTEGER NOT NULL, `isExecutable` bool NOT NULL, PRIMARY KEY(`id`) ); CREATE TABLE IF NOT EXISTS `exeFileCmd` ( `id` INTEGER, `cmdId` INTEGER NOT NULL, `exeFileId` INTEGER references exeFile(id), PRIMARY KEY(`id`) ); CREATE TABLE IF NOT EXISTS `session` ( `id` BLOB, `comment` TEXT, PRIMARY KEY(`id`) ); CREATE INDEX IF NOT EXISTS idx_file_name ON `file` (`name`); CREATE INDEX IF NOT EXISTS idx_file_mtime ON `file` (`mtime`); CREATE INDEX IF NOT EXISTS idx_file_size ON `file` (`size`); CREATE INDEX IF NOT EXISTS idx_file_hash ON `file` (`hash`); CREATE INDEX IF NOT EXISTS idx_exemeta_exepath ON `exeMeta` (`exepath`); CREATE INDEX IF NOT EXISTS idx_exeFile_name ON `exeFile` (`name`); CREATE INDEX IF NOT EXISTS idx_exeFile_mtime ON `exeFile` (`mtime`); CREATE INDEX IF NOT EXISTS idx_exeFile_size ON `exeFile` (`size`); replace into version (id, ver) values (1, '0.1');)SOMERANDOMTEXT"; ================================================ FILE: src/common/database/sqlite_database_scheme.h ================================================ #pragma once extern const char* SQLITE_DATABASE_SCHEME; ================================================ FILE: src/common/database/sqlite_database_scheme_updates.cpp ================================================ #include "sqlite_database_scheme_updates.h" void sqlite_database_scheme_updates::v0_9(QSqlQueryThrow &query) { // until this version no scripts (read files) were stored in the database // so the tables can be dropped (and re-created) safely. // Further rename tables to better represent read/write events query.exec("ALTER TABLE `file` RENAME TO `writtenFile`"); query.exec("drop index idx_exemeta_exepath"); query.exec("drop index idx_exeFile_name"); query.exec("drop index idx_exeFile_mtime"); query.exec("drop index idx_exeFile_size"); query.exec("drop table exeMeta"); query.exec("drop table exeFile"); query.exec("drop table exeFileCmd"); query.exec( "CREATE TABLE IF NOT EXISTS `readFile` (" "`id` INTEGER," "`envId` INTEGER NOT NULL references `env`(id)," "`name` TEXT NOT NULL," "`path` TEXT NOT NULL," "`mtime` timestamp NOT NULL," "`size` INTEGER NOT NULL," "`mode` BLOB NOT NULL," "PRIMARY KEY(`id`)" ")" ); query.exec( "CREATE TABLE IF NOT EXISTS `readFileCmd` (" "`id` INTEGER," "`cmdId` INTEGER NOT NULL references `cmd`(id) ON DELETE CASCADE," "`readFileId` INTEGER references readFile(id)," "PRIMARY KEY(`id`)" ")" ); } void sqlite_database_scheme_updates::v2_1(QSqlQueryThrow &query) { // Add support for read files without belonging scripts. // Also start hashing read files as well. Because the same read file // can refer to multiple commands (many-to-many), it would be wrong to // refererence the hashMetaId of a command -> add hashmetaId column. query.exec("alter table `readFile` add column `hash` BLOB"); query.exec("alter table `readFile` add column `hashmetaId` INTEGER"); query.exec("alter table `readFile` add column `isStoredToDisk` INTEGER DEFAULT 1"); } void sqlite_database_scheme_updates::v2_2(QSqlQueryThrow &query) { // Create indeces to improve query and delete performance. query.exec("CREATE INDEX IF NOT EXISTS `idx_writtenFile_cmdId` ON `writtenFile` (`cmdId`)"); query.exec("CREATE INDEX IF NOT EXISTS `idx_readFileCmd_cmdId` ON `readFileCmd` (`cmdId`)"); query.exec("CREATE INDEX IF NOT EXISTS `idx_readFileCmd_readFileId` ON `readFileCmd` (`readFileId`)"); query.exec("CREATE INDEX IF NOT EXISTS `idx_cmd_envId` ON `cmd` (`envId`)"); query.exec("CREATE INDEX IF NOT EXISTS `idx_cmd_sessionId` ON `cmd` (`sessionId`)"); query.exec("CREATE INDEX IF NOT EXISTS `idx_cmd_hashmetaId` ON `cmd` (`hashmetaId`)"); query.exec("CREATE INDEX IF NOT EXISTS `idx_readFile_envId` ON `readFile` (`envId`)"); } void sqlite_database_scheme_updates::v2_4(QSqlQueryThrow& query){ // Create unified, deduplicated paths for read- and written file-events // Ideally we would rename path to pathId and add a foreign key to it, // but this is not possible with current (e.g. 3.22.0) sqlite-versions. // So we recreate the tables writtenFile and readFile query.exec(R"SOMERANDOMTEXT( create table if not exists `pathtable` ( `id` INTEGER, `path` TEXT NOT NULL, PRIMARY KEY(`id`) ) )SOMERANDOMTEXT" ); query.exec("create unique index if not exists " " idx_unq_pathtable_path on pathtable (path)"); query.exec("insert or ignore into pathtable (path) " "select path from writtenFile"); query.exec ( R"SOMERANDOMTEXT( CREATE TABLE `writtenFile_TMP` ( `id` INTEGER, `name` TEXT NOT NULL, `pathId` INTEGER NOT NULL, /* pathId instead of path */ `cmdId` INTEGER NOT NULL, `mtime` timestamp NOT NULL, `size` INTEGER NOT NULL, `hash` BLOB, PRIMARY KEY(`id`), FOREIGN KEY(`cmdId`) REFERENCES `cmd`(`id`) ON DELETE CASCADE, FOREIGN KEY(`pathId`) REFERENCES `pathtable`(`id`) /* new */ ))SOMERANDOMTEXT" ); // copy all data, use pathId from newly created pathtable query.exec("insert into writtenFile_TMP " "(id,name,pathId,cmdId,mtime,size,hash) " "select id,name," "(select id from pathtable where pathtable.path=writtenFile.path)," "cmdId,mtime,size,hash " "from writtenFile"); query.exec("drop table if exists writtenFile"); query.exec("ALTER TABLE `writtenFile_TMP` RENAME TO `writtenFile`"); // (re-)create indices // Foreign keys are always recommended to be indexed: // (see https://www.sqlite.org/foreignkeys.html#fk_indexes ) query.exec("create index `idx_writtenFile_cmdId` ON `writtenFile` (`cmdId`)"); query.exec("create index `idx_writtenFile_size` ON `writtenFile` (`size`)"); query.exec("create index `idx_writtenFile_name` ON `writtenFile` (`name`)"); query.exec("create index `idx_writtenFile_mtime` ON `writtenFile` (`mtime`)"); query.exec("create index `idx_writtenFile_hash` ON `writtenFile` (`hash`)"); // new one: query.exec("create index `idx_writtenFile_pathId` ON `writtenFile` (`pathId`)"); // -------- done with writtenFile query.exec("insert or ignore into pathtable (path) " "select path from readFile"); query.exec ( R"SOMERANDOMTEXT( CREATE TABLE `readFile_TMP` ( `id` INTEGER, `envId` INTEGER NOT NULL, `name` TEXT NOT NULL, `pathId` INTEGER NOT NULL, /* pathId instead of path */ `mtime` timestamp NOT NULL, `size` INTEGER NOT NULL, `mode` BLOB NOT NULL, `hash` BLOB, `hashmetaId` INTEGER, `isStoredToDisk` INTEGER DEFAULT 1, FOREIGN KEY(`envId`) REFERENCES `env`(`id`), FOREIGN KEY(`pathId`) REFERENCES `pathtable`(`id`), /* new */ PRIMARY KEY(`id`) ))SOMERANDOMTEXT" ); // copy all data, use pathId from newly created pathtable query.exec( R"SOMERANDOMTEXT( insert into readFile_TMP (id,envId,name,pathId,mtime,size,mode,hash,hashmetaId,isStoredToDisk) select id,envId,name, (select id from pathtable where pathtable.path=readFile.path), mtime,size,mode,hash,hashmetaId,isStoredToDisk from readFile )SOMERANDOMTEXT" ); query.exec("drop table if exists readFile"); query.exec("alter table `readFile_TMP` rename to `readFile`"); query.exec("create index if not exists `idx_readFile_envId` ON `readFile` (`envId`)"); // new one: query.exec("create index `idx_readFile_pathId` ON `readFile` (`pathId`)"); } void sqlite_database_scheme_updates::v2_5(QSqlQueryThrow &query) { // Forbid sense- and useless duplicates in readFileCmd. // Before the kernel-module backend these occurred rarely, // because fanotify already merges many equal events. // The kernel-module backend does not do so (at least not in // v2.4), so this became apparent quite late. // Delete existing duplicates query.exec( R"SOMERANDOMTEXT( delete from readFileCmd where `id` not in ( select min(`id`) from readFileCmd group by cmdId, readFileId ) )SOMERANDOMTEXT" ); query.exec("create unique index if not exists " " idx_unq_readFileCmd on readFileCmd (cmdId,readFileId)"); // Null-values in sql can be nasty, e.g. null != null. // Replace null-hashes by empty strings. query.exec("update writtenFile set `hash`='' where `hash` is null"); query.exec("update readFile set `hash`='' where `hash` is null"); // Also delete and forbid duplicate written file-events. query.exec( R"SOMERANDOMTEXT( delete from writtenFile where `id` not in ( select min(`id`) from writtenFile group by `name`,pathId,cmdId,mtime,size,hash ) )SOMERANDOMTEXT" ); query.exec("create unique index if not exists " " idx_unq_writtenFile on writtenFile (`name`,pathId,cmdId,mtime,size,hash)"); } ================================================ FILE: src/common/database/sqlite_database_scheme_updates.h ================================================ #pragma once #include "qsqlquerythrow.h" namespace sqlite_database_scheme_updates { void v0_9(QSqlQueryThrow& query); // 0.8 -> 0.9 void v2_1(QSqlQueryThrow& query); // 2.0 -> 2.1 void v2_2(QSqlQueryThrow& query); // 2.1 -> 2.2 void v2_4(QSqlQueryThrow& query); // 2.3 -> 2.4 void v2_5(QSqlQueryThrow& query); // 2.4 -> 2.5 } ================================================ FILE: src/common/database/sqlquery.cpp ================================================ #include #include "sqlquery.h" #include "exccommon.h" #include "util.h" const QString &SqlQuery::query() const { return m_query; } QString &SqlQuery::query() { return m_query; } const QVariantList &SqlQuery::values() const { return m_values; } void SqlQuery::clear() { m_query.clear(); m_values.clear(); m_columnSet.clear(); m_tablenames.clear(); m_ascending = true; m_limit = NO_LIMIT; } bool SqlQuery::isEmpty() const { return m_query.isEmpty(); } void SqlQuery::addWithAnd(const QString &columnName, const QVariant &value, const CompareOperator &operator_) { addWithAnd(columnName, { value }, QVector{operator_}); } /// @overload void SqlQuery::addWithAnd(const QString &columnName, const QVariantList &values, const CompareOperator &operator_, bool innerAND) { addWithAnd(columnName, values, QVector{operator_}, innerAND); } void SqlQuery::addWithAnd(const QString &columnName, const QVariantList &values, const QVector &operators, bool innerAND) { addWithConnector(columnName, values, operators, innerAND, true); } void SqlQuery::addWithAnd(const SqlQuery &sqlQ) { addWithConnector(sqlQ, true); } void SqlQuery::addWithOr(const SqlQuery &sqlQ) { addWithConnector(sqlQ, false); } /// Add the given values to the query-string and -values. /// If a QVariant.isNull insert "is null" instead of a placeholder. /// @param values: The number of values must match to the number of operators, /// except for the BETWEEN-operator (see below) /// @param operators: The comparsion operators used for each value. In case of /// BETWEEN, only one operator may be passed. /// @param innerAnd: If true, connect the column-value pair with AND, else with OR /// (in case of BETWEEN it is ignored) /// @param outerAnd: if true, connect the whole column-value pair with a single /// AND, else use OR. /// /// @throws QExcIllegalArgument void SqlQuery::addWithConnector(const QString& columnName, const QVariantList& values, const QVector& operators, bool innerAND, bool outerAnd){ if(values.isEmpty()){ throw QExcIllegalArgument(QString("%1: %2 must no be empty") .arg(__func__, GET_VARIABLE_NAME(values))); } auto actualOps = expandOperatorsIfNeeded(operators, values.size()); writeConnectorPrefix(outerAnd); auto valueIt = values.begin(); auto operatorIt = actualOps.begin(); QByteArray innerJunction; if(innerAND || (operators.size() == 1 && operators.first().asEnum() == E_CompareOperator::BETWEEN)){ innerJunction = " and "; } else { innerJunction = " or "; } while(valueIt != values.end()){ if(valueIt != values.begin()){ m_query += innerJunction; } const QVariant & var = *valueIt; if(var.isNull()){ // null values are only allowed for certain operators: QString operatorNow; switch (operatorIt->asEnum()) { case E_CompareOperator::EQ: case E_CompareOperator::LIKE: operatorNow = " is null "; break; case E_CompareOperator::NE: operatorNow = " is not null "; break; default: throw QExcIllegalArgument("null is illegal for operator " + operatorIt->asSql() + " in column " + columnName ); } m_query += columnName + operatorNow ; } else { if(operatorIt->asEnum() == E_CompareOperator::BETWEEN){ throw QExcIllegalArgument("BETWEEN passed within list with len > 1"); } m_query += columnName + operatorIt->asSql() + "? "; m_values.push_back(var); } ++valueIt; ++operatorIt; } writeConnectorSuffix(); addToTableCols(columnName); } /// Add the other query "as is" using the given connector void SqlQuery::addWithConnector(const SqlQuery& other, bool outerAnd){ if(other.isEmpty()){ return; } writeConnectorPrefix(outerAnd); m_query += other.query(); m_values.append(other.values()); m_columnSet.insert(other.m_columnSet.begin(), other.m_columnSet.end()); m_tablenames.insert(other.m_tablenames.begin(), other.m_tablenames.end()); writeConnectorSuffix(); } /// If the number of operators does not match the number of values, duplicate them, so they /// do (in that case len(operators) *must* be 1). /// The BETWEEN operator is a special case, it is transformed into >= and <=. QVector SqlQuery::expandOperatorsIfNeeded(const QVector &operators, int nValues) const { if(operators.size() == nValues){ return operators; } if(operators.size() != 1){ throw QExcIllegalArgument( QString("len(operators) %1 !=len(values) %2 but not 1") .arg(operators.size(), nValues)); } const CompareOperator & op = operators.first(); if(op.asEnum() == E_CompareOperator::BETWEEN){ if(nValues != 2){ throw QExcIllegalArgument(QString("BETWEEN operator requires 2 values but %1" "were given").arg(nValues)); } return { E_CompareOperator::GE, E_CompareOperator::LE }; } // same operator for all values auto newOps = QVector(); newOps.reserve(nValues); for(int i=0; i < nValues; i++){ newOps.push_back(op); } return newOps; } /// remeber that this table-column was used. If it contains a dot, /// the part before it is interpreted as tablename, after it as column. void SqlQuery::addToTableCols(const QString &tableCol) { int dotIdx = tableCol.indexOf('.'); if(dotIdx == -1){ // assume column name without table name m_columnSet.insert(tableCol); } else { m_tablenames.insert(tableCol.left(dotIdx)); m_columnSet.insert(tableCol.mid(dotIdx + 1)); } } void SqlQuery::writeConnectorPrefix(bool outerAnd) { if(! m_query.isEmpty()){ m_query += (outerAnd) ? " and " : " or "; } m_query += " ( "; } void SqlQuery::writeConnectorSuffix() { m_query += " ) "; } /// setting the query is only allowed, it no values were set (yet) void SqlQuery::setQuery(const QString &query) { if(! m_values.isEmpty()){ throw QExcProgramming("setting query while values not empty"); } m_query = query; } /// @return true, if the *exact* columnname was added via 'addWithAnd' bool SqlQuery::containsColumn(const QString &col) const { return m_columnSet.find(col) != m_columnSet.end(); } bool SqlQuery::containsTablename(const QString &table) const { return m_tablenames.find(table) != m_tablenames.end(); } int SqlQuery::limit() const { return m_limit; } /// @param limit: NO_LIMIT means *not* to impose a limit void SqlQuery::setLimit(int limit) { m_limit = limit; } /// @return 'limit x '-string or space character, if NO_LIMIT is imposed QString SqlQuery::mkLimitString() const { return (m_limit == NO_LIMIT) ? " " : "limit " + QString::number(m_limit) + " "; } bool SqlQuery::ascending() const { return m_ascending; } const QString &SqlQuery::ascendingStr() const { static const QString ASC_STR = "asc "; static const QString DESC_STR = "desc "; if(m_ascending) return ASC_STR; return DESC_STR; } void SqlQuery::setAscending(bool ascending) { m_ascending = ascending; } /// Make an sql-query that always finds zero results (where 0) SqlQuery mkInertSqlQuery() { SqlQuery q; q.query() = " 0 "; return q; } ================================================ FILE: src/common/database/sqlquery.h ================================================ #pragma once #include #include #include #include #include "compareoperator.h" #include "util.h" /// Helper class to build the query part after a sql where-clause. /// Column-value(s)-pairs can be added to the SqlQuery in a NULL-safe /// manner. An AND-operator is only added, if necessary (query-columncount > 1). class SqlQuery { public: static const int NO_LIMIT {-1}; void addWithAnd(const QString& columnName, const QVariant& value, const CompareOperator& operator_=CompareOperator()); void addWithAnd(const QString& columnName, const QVariantList& values, const CompareOperator& operator_=CompareOperator(), bool innerAND=false); void addWithAnd(const QString& columnName, const QVariantList& values, const QVector& operators, bool innerAND=false); void addWithAnd(const SqlQuery& sqlQ); void addWithOr(const SqlQuery& sqlQ); const QString& query() const; QString& query(); const QVariantList& values() const; void clear(); bool isEmpty() const; bool ascending() const; const QString& ascendingStr() const; void setAscending(bool ascending); int limit() const; QString mkLimitString() const; void setLimit(int limit); void setQuery(const QString &query); bool containsColumn(const QString& col) const; bool containsTablename(const QString& table) const; private: void addWithConnector(const QString& columnName, const QVariantList& values, const QVector& operators, bool innerAND=false, bool outerAnd=false); void addWithConnector(const SqlQuery& sqlQ, bool outerAnd); QVector expandOperatorsIfNeeded( const QVector &operators, int nValues) const; void addToTableCols(const QString& tableCol); void writeConnectorPrefix(bool outerAnd); void writeConnectorSuffix(); QString m_query; QVariantList m_values; std::unordered_set m_columnSet; std::unordered_set m_tablenames; bool m_ascending {true}; int m_limit {NO_LIMIT}; }; SqlQuery mkInertSqlQuery(); ================================================ FILE: src/common/database/storedfiles.cpp ================================================ #include #include "storedfiles.h" #include "db_connection.h" #include "util.h" #include "qfilethrow.h" #include "os.h" const QString& StoredFiles::getReadFilesDir() { static const QString path = db_connection::getDatabaseDir() + "/readFiles"; return path ; } /// creates path of stored files if not exist /// @return the created path /// @throws QExcIo const QString& StoredFiles::mkpath() { const auto & p = getReadFilesDir(); if( ! QDir(p).mkpath(p)){ throw QExcIo(qtr("Failed to the create directory for the stored read files at %1") .arg(p)); } return p; } StoredFiles::StoredFiles() { const auto & path = getReadFilesDir(); m_readFilesDir.setPath(path); this->mkpath(); } QString StoredFiles::mkPathStringToStoredReadFile(const FileReadInfo &info) { return mkPathStringToStoredReadFile(info.idInDb); } QString StoredFiles::mkPathStringToStoredReadFile(qint64 idInDb) { return pathJoinFilename(StoredFiles::getReadFilesDir(), QString::number(idInDb)); } bool StoredFiles::deleteReadFile(const QString &fname) { return m_readFilesDir.remove(fname); } /// @throws QExcIo void StoredFiles::addReadFile(const QString &fname, const QByteArray &data) { const QString fPath = m_readFilesDir.absoluteFilePath(fname); QFileThrow f(fPath); try { f.open(QFile::OpenModeFlag::WriteOnly | QFile::OpenModeFlag::Truncate); f.write(data); } catch (const QExcIo&) { f.remove(); throw ; } } /// @param info: the read file already loaded from the database /// @param dir: the directory where to restore it (warning: override without confirmation) /// @param openReadFileInDb: the for reading opened file corresponding to the info-database-entry. void StoredFiles::restoreReadFileAtDIr(const FileReadInfo &info, const QDir& dir, const QFile &openReadFileInDb) { assert(openReadFileInDb.isOpen()); const QString filePath = dir.absoluteFilePath(info.name); QFileThrow dstFile(filePath); dstFile.open(QFile::OpenModeFlag::WriteOnly); os::sendfile(dstFile.handle(), openReadFileInDb.handle(), static_cast(info.size)); os::fchmod(dstFile.handle(), info.mode); } /// @overload void StoredFiles::restoreReadFileAtDIr(const FileReadInfo &info, const QDir &dir) { QFileThrow f(m_readFilesDir.absoluteFilePath(QString::number(info.idInDb))); f.open(QFile::OpenModeFlag::ReadOnly); restoreReadFileAtDIr(info, dir, f); } ================================================ FILE: src/common/database/storedfiles.h ================================================ #pragma once #include #include "fileinfos.h" class StoredFiles { public: static const QString &getReadFilesDir(); static const QString &mkpath(); StoredFiles(); QString mkPathStringToStoredReadFile(const FileReadInfo& info); QString mkPathStringToStoredReadFile(qint64 idInDb); bool deleteReadFile(const QString& fname); void addReadFile(const QString& fname, const QByteArray& data); void restoreReadFileAtDIr(const FileReadInfo &info, const QDir& dir, const QFile &openReadFileInDb); void restoreReadFileAtDIr(const FileReadInfo &info, const QDir& dir); private: QDir m_readFilesDir; }; ================================================ FILE: src/common/fdcommunication.cpp ================================================ #include #include #include "fdcommunication.h" #include "os.h" #include "cleanupresource.h" #include "util.h" using namespace fdcommunication; struct MessageHeader { int msgId; size_t len; // length of custom payload bool containsFd; }; static_assert (std::is_pod(), ""); SocketCommunication::SocketCommunication() : m_sockFd(-1) {} /// Block until we receive a message from the other endpoint of the set socket. /// Make sure, the internal receive buffer is large enough /// (* it most not be empty*). void SocketCommunication::receiveMessages(Messages *messages){ messages->clear(); if(m_receiveCtrlMsgBuf.size() < int(CMSG_SPACE(sizeof(int)))){ m_receiveCtrlMsgBuf.resize(CMSG_SPACE(sizeof(int))); } assert(! m_receiveBuf.isEmpty()); iovec ioVector = { m_receiveBuf.data(), static_cast(m_receiveBuf.size()) }; struct msghdr msgHdr{}; msgHdr.msg_iov = &ioVector; msgHdr.msg_iovlen = 1; msgHdr.msg_control = m_receiveCtrlMsgBuf.data(); msgHdr.msg_controllen = size_t(m_receiveCtrlMsgBuf.size()); size_t len = os::recvmsg(m_sockFd, &msgHdr); if (len == 0) { messages->push_back(-1); return; } if(len < sizeof (MessageHeader)){ throw ExcFdComm(qtr("Bad socket message received (too small)")); } char* pData = m_receiveBuf.data(); const char* finalpData = pData + len; struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msgHdr); int* currentFd = reinterpret_cast(CMSG_DATA(cmsg)); while(true){ auto *customHeader = reinterpret_cast(pData); // Consume the header, find out the payload-length... pData += sizeof (MessageHeader); assert(pData <= finalpData); QByteArray payload(pData, static_cast(customHeader->len)); Message message(customHeader->msgId, payload); if (customHeader->containsFd) { // we received an fd via SCM_RIGHTS. // see also man 3 cmsg assert(cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS); message.fd = *currentFd; ++currentFd; } messages->push_back(message); // ... consume the payload pData += customHeader->len; assert(pData <= finalpData); if(pData >= finalpData){ break; } } } void SocketCommunication::sendMsg(const SocketCommunication::Message &message) { this->sendMessages({message}); } void SocketCommunication::sendMessages(const SocketCommunication::Messages &messages) { QVector iovects; iovects.reserve(messages.size() * 2); QVector headers; headers.reserve(messages.size()); QVector fds; for(const auto& msg : messages){ assert(msg.msgId >= 0); // brace-initialze MessageHeader{}, otherwise valgrind complains (unitialized) headers.push_back(MessageHeader{}); headers.last().msgId = msg.msgId; headers.last().len = size_t(msg.bytes.length()); headers.last().containsFd = (msg.fd != -1) ; iovects.push_back({&headers.last(), sizeof (MessageHeader)}); iovec io; io.iov_base = const_cast(static_cast(msg.bytes.data())); io.iov_len = size_t(msg.bytes.size()); iovects.push_back(io); if(msg.fd != -1){ fds.push_back(msg.fd); } } assert(iovects.capacity() == iovects.size()); assert(headers.capacity() == headers.size()); struct msghdr messageHeader{}; messageHeader.msg_iov = iovects.data(); messageHeader.msg_iovlen = iovects.size(); QByteArray buf( int(CMSG_SPACE(size_t(fds.size()) * sizeof(int)) ), '\0'); if(! fds.isEmpty()){ messageHeader.msg_control = buf.data(); messageHeader.msg_controllen = size_t(buf.size()); struct cmsghdr *cmsg = CMSG_FIRSTHDR(&messageHeader); cmsg->cmsg_level = SOL_SOCKET; cmsg->cmsg_type = SCM_RIGHTS; cmsg->cmsg_len = CMSG_LEN(sizeof(int) * fds.size()); int *fdptr = reinterpret_cast(CMSG_DATA(cmsg) ); memcpy(fdptr, fds.data(), fds.size() * sizeof(int)); } try { os::sendmsg(m_sockFd, &messageHeader); } catch (const os::ExcOs& ex) { QString msg = (messages.isEmpty()) ? "EMPTY message" : messages[0].bytes; throw ExcFdComm(qtr("Failed to send «%1» - %2").arg(msg).arg(ex.what())); } } int SocketCommunication::sockFd() const { return m_sockFd; } void SocketCommunication::setSockFd(int fd) { m_sockFd = fd; } void SocketCommunication::setReceiveBufferSize(int s) { m_receiveBuf.resize(s); } /// set size of file descriptor buffer void SocketCommunication::setReceiveFdSize(int s) { m_receiveCtrlMsgBuf.resize( s * int(CMSG_SPACE(sizeof(int))) ); } SocketCommunication::Messages SocketCommunication::receiveMessages() { Messages messages; receiveMessages(&messages); return messages; } ================================================ FILE: src/common/fdcommunication.h ================================================ #pragma once #include #include "exccommon.h" namespace fdcommunication { class ExcFdComm : public QExcCommon { public: using QExcCommon::QExcCommon; }; class SocketCommunication { public: struct Message{ /// message-types < 0 are for internal use: -1 indicates an empty message /// (end of data: all instances of the other endpoint were closed). Message(int mesgType=-1, const QByteArray& b=QByteArray(), int fd = -1) : msgId(mesgType), bytes(b), fd(fd) {} bool operator==(const Message &rhs) const { return this->msgId == rhs.msgId && this->bytes == rhs.bytes && this->fd == rhs.fd; } int msgId; QByteArray bytes; int fd; }; typedef QVector Messages; SocketCommunication(); int sockFd() const; void setSockFd(int sockFd); void setReceiveBufferSize(int s); void setReceiveFdSize(int s); Messages receiveMessages(); void receiveMessages(Messages* messages); void sendMsg(const Message& message); void sendMessages(const Messages& messages); private: QByteArray m_receiveBuf; QByteArray m_receiveCtrlMsgBuf; int m_sockFd; }; } // namespace fdcommunication ================================================ FILE: src/common/fileeventhandler.cpp ================================================ #include #include #include #include #include #include #include #include #include "app.h" #include "excos.h" #include "fileeventhandler.h" #include "logger.h" #include "osutil.h" #include "os.h" #include "qfddummydevice.h" #include "qoutstream.h" #include "settings.h" #include "stdiocpp.h" #include "strlight_util.h" static QString buildFilecacheDir(){ return pathJoinFilename(QDir::tempPath(), QString(app::SHOURNAL_RUN) + "-cache-" + QString::number(os::getpid())); } /// meant to be called as real user within the original mount namespace FileEventHandler::FileEventHandler() : m_filecacheDir(buildFilecacheDir()), m_uid(os::getuid()), m_ourProcFdDirDescriptor(os::open("/proc/self/fd", O_DIRECTORY)), m_pathbuf(PATH_MAX + 1, '\0'), m_fdStringBuf(snprintf( nullptr, 0, "%d", std::numeric_limits::max()) + 1, '\0'), r_wCfg(Settings::instance().writeFileSettings()), r_rCfg(Settings::instance().readFileSettings()), r_scriptCfg(Settings::instance().readEventScriptSettings()), r_hashCfg(Settings::instance().hashSettings()) { FILE* f = stdiocpp::fopen( pathJoinFilename(m_filecacheDir.path().toUtf8(), QByteArray("file-events")), "w+"); m_fileEvents.setFile(f); this->fillAllowedGroups(); if(r_hashCfg.hashEnable){ // This is typically larger than maxCountOfReads*chunkSize // but does not have to be. m_hashControl.getXXHash().resizeBuf(1024*512); } } FileEventHandler::~FileEventHandler(){ fclose(m_fileEvents.file()); try { os::close(m_ourProcFdDirDescriptor); } catch (const os::ExcOs& e) { logCritical << __func__ << e.what(); } } void FileEventHandler::fillAllowedGroups() { auto groups = os::getgroups(); auto egid = os::getegid(); auto gid = os::getgid(); for(const auto& g : groups){ if(gid == egid || g != egid){ // only insert the 'real' groups m_groups.insert(g); } } } /// A file-write-event is considered allowed here, if the *real user* /// is allowed to write to a given file, be it because being the owner, /// the file is writable by everyone or he/she is /// part of the owning group. /// Background is that we are interested in write events, however, /// reporting file modifcations of a root-process should not be allowed. bool FileEventHandler::userHasWritePermission(const struct stat &st) { return (st.st_mode & S_IWUSR && st.st_uid == m_uid) || st.st_mode & S_IWOTH || (st.st_mode & S_IWGRP && m_groups.find(st.st_gid) != m_groups.end()); } /// See doc of write writeEventAllowed, replace 'write' with 'read' bool FileEventHandler::userHasReadPermission(const struct stat &st) { return (st.st_mode & S_IRUSR && st.st_uid == m_uid) || st.st_mode & S_IROTH || (st.st_mode & S_IRGRP && m_groups.find(st.st_gid) != m_groups.end()); } /// check whether to accept the file according to file extension and mime-type. /// If both set (not-empty) only one has to match, /// if both unset, accept all, /// else only take the set one into account. bool FileEventHandler::readFileTypeMatches(const Settings::ScriptFileSettings &scriptCfg, int fd, const StrLight& fpath) { if(! scriptCfg.includeExtensions.empty() && ! scriptCfg.includeMimetypes.empty()){ // both not empty, consider both (OR'd) return fileExtensionMatches(scriptCfg.includeExtensions, fpath) || mimeTypeMatches(fd, scriptCfg.includeMimetypes); } if(scriptCfg.includeExtensions.empty() && scriptCfg.includeMimetypes.empty()){ return true; } // one is empty, the other not if(! scriptCfg.includeExtensions.empty()){ return fileExtensionMatches(scriptCfg.includeExtensions, fpath); } assert(! scriptCfg.includeMimetypes.empty()); return mimeTypeMatches(fd, scriptCfg.includeMimetypes); } void FileEventHandler::readLinkOfFd(int fd, StrLight &output) { assert(m_ourProcFdDirDescriptor != -1); // uitoa, safe in this context (but not in general), // is a lot faster, so do not use snprintf here. // snprintf( &m_fdStringBuf[0], m_fdStringBuf.size() (+1?), "%d", fd); util_performance::uitoa(fd, m_fdStringBuf.data()); ssize_t path_len = ::readlinkat(m_ourProcFdDirDescriptor, m_fdStringBuf.data(), output.data(), output.capacity()); if (path_len == -1 ){ throw os::ExcReadLink("readlinkat failed for fd " + std::to_string(fd)); } output.resize(StrLight::size_type(path_len)); } bool FileEventHandler::fileExtensionMatches(const Settings::StrLightSet &validExtensions, const StrLight& fullPath) { strlight_util::findFileExtension_raw(fullPath, m_extensionBuf); if(m_extensionBuf.empty()){ return false; } return validExtensions.find(m_extensionBuf) != validExtensions.end(); } bool FileEventHandler::mimeTypeMatches(int fd, const Settings::MimeSet &validMimetypes) { QFdDummyDevice f(fd); const auto mimetype = m_mimedb.mimeTypeForData(&f).name(); os::lseek(fd, 0, SEEK_SET); return validMimetypes.find(mimetype) != validMimetypes.end(); } QString FileEventHandler::getTmpDirPath() const { return m_filecacheDir.path(); } void FileEventHandler::clearEvents() { m_fileEvents.clear(); } FileEvents &FileEventHandler::fileEvents() { return m_fileEvents; } /// @param enableReadActions: if false, do not read from fd, regardless of settings /// @throws ExcOs, CXXHashError void FileEventHandler::handleCloseWrite(int fd) { // first lookup the path, then stat, so no filename contains a trailing '(deleted)' readLinkOfFd(fd, m_pathbuf); const auto st = os::fstat(fd); if(st.st_nlink == 0){ // always ignore deleted files logDebug << "closedwrite-event ignored (file deleted):" << m_pathbuf; return; } if(! userHasWritePermission(st)){ logDebug << "closedwrite-event ignored (no write permission):" << m_pathbuf; return; } if(! r_wCfg.includePaths->isSubPath(m_pathbuf, true) ){ logDebug << "closedwrite-event ignored (no subpath of include_dirs): " << m_pathbuf; return; } if(r_wCfg.excludePaths->isSubPath(m_pathbuf, true) ){ logDebug << "closedwrite-event ignored (subpath of exclude_dirs): " << m_pathbuf; return; } if(r_wCfg.excludeHidden && pathIsHidden(m_pathbuf) && ! r_wCfg.includePathsHidden->isSubPath(m_pathbuf, true)){ logDebug << "closedwrite-event ignored (hidden file):" << m_pathbuf; return; } if(m_fileEvents.wEventCount() >= r_wCfg.maxEventCount){ logDebug << "closedwrite-event dropped:" << m_pathbuf; m_fileEvents.incrementDropCount(O_WRONLY); return; } HashValue hash; if(r_hashCfg.hashEnable){ hash = m_hashControl.genPartlyHash(fd, st.st_size, r_hashCfg.hashMeta); } m_fileEvents.write(O_WRONLY, m_pathbuf, st, hash); logDebug << "closedwrite-event recorded: " << m_pathbuf; // maybe_todo: reimplement that, if desired (?). // if(m_pArgparse->getCommandline()){ // info.cmdline = pidcontrol::findCmdlineOfPID(pid); // PID from fanotify event data... // } } bool FileEventHandler::generalReadSettingsSayLogIt(const bool userHasWritePerm, const StrLight& filepath) { if(! r_rCfg.enable){ return false; } if(r_rCfg.onlyWritable && ! userHasWritePerm){ logDebug << "general read event ignored: no write permission:" << filepath; return false; } if( ! r_rCfg.includePaths->isSubPath(filepath, true)){ logDebug << "general read event ignored: not a subpath of any included path:" << filepath; return false; } if( r_rCfg.excludePaths->isSubPath(filepath, true)){ logDebug << "general read event ignored: is a subpath of an excluded path:" << filepath; return false; } if(r_rCfg.excludeHidden && pathIsHidden(filepath) && ! r_rCfg.includePathsHidden->isSubPath(filepath, true)){ logDebug << "general read event ignored: hidden file:" << filepath; return false; } return true; } bool FileEventHandler::scriptReadSettingsSayLogIt(bool userHasWritePerm, const StrLight &fpath, const os::stat_t &st, int fd) { if(! r_scriptCfg.enable){ return false; } // repeat check here: fanotify-read-events are only unregistered, if // general read events are disabled... if(m_fileEvents.rStoredFilesCount() >= r_scriptCfg.maxCountOfFiles){ logDebug << "possible script-event ignored: already collected enough files:" << fpath; return false; } if(r_scriptCfg.onlyWritable && ! userHasWritePerm){ logDebug << "possible script-event ignored: no write permission:" << fpath; return false; } if(st.st_size > r_scriptCfg.maxFileSize){ logDebug << "possible script-event ignored: file too big:" << fpath; return false; } if( ! r_scriptCfg.includePaths->isSubPath(fpath, true)){ logDebug << "possible script-event ignored: file" << fpath << "is not a subpath of any included path"; return false; } if( r_scriptCfg.excludePaths->isSubPath(fpath, true)){ logDebug << "possible script-event ignored: file" << fpath << "is a subpath of an excluded path"; return false; } if(r_scriptCfg.excludeHidden && pathIsHidden(fpath) && ! r_scriptCfg.includePathsHidden->isSubPath(fpath, true)){ logDebug << "possible script-event ignored: hidden file:" << fpath; return false; } if(! readFileTypeMatches(r_scriptCfg, fd, fpath)){ logDebug << "script-event ignored: neither file-extension nor mime-type " "matches for " << fpath; return false; } return true; } bool FileEventHandler::pathIsHidden(const StrLight &fullPath) { return fullPath.find("/.") != StrLight::npos; } /// @param enableReadActions: if false, do not read from fd, regardless of settings void FileEventHandler::handleCloseRead(int fd) { // first lookup the path, then stat, so no filename contains a trailing '(deleted)' readLinkOfFd(fd, m_pathbuf); const auto st = os::fstat(fd); if(st.st_nlink == 0){ // always ignore deleted files logDebug << "read-event ignored (file deleted): " << m_pathbuf; return; } if(! userHasReadPermission(st)){ logDebug << "read-event ignored (read not allowed): " << m_pathbuf; return; } const bool userHasWritePerm = userHasWritePermission(st); const bool logGeneralReadEvent = generalReadSettingsSayLogIt(userHasWritePerm, m_pathbuf); bool logScriptEvent = scriptReadSettingsSayLogIt(userHasWritePerm, m_pathbuf, st, fd); if(! logGeneralReadEvent && ! logScriptEvent){ return; } if(m_fileEvents.rEventCount() >= r_rCfg.maxEventCount){ logDebug << "closedread-event dropped:" << m_pathbuf; m_fileEvents.incrementDropCount(O_RDONLY); return; } HashValue hash; if(r_hashCfg.hashEnable){ assert(os::ltell(fd) == 0); hash = m_hashControl.genPartlyHash(fd, st.st_size, r_hashCfg.hashMeta); } int storeFd; if(logScriptEvent){ assert(os::ltell(fd) == 0); storeFd = fd; } else { storeFd = -1; } m_fileEvents.write(O_RDONLY, m_pathbuf, st, hash, storeFd); logDebug << "closedread-event recorded (collect script:" << logScriptEvent << ")" << m_pathbuf; } ================================================ FILE: src/common/fileeventhandler.h ================================================ #pragma once #include #include #include #include #include #include #include #include "hashcontrol.h" #include "nullable_value.h" #include "fileevents.h" #include "settings.h" #include "os.h" #include "strlight.h" #include "util_performance.h" /// Collect desired file-event (read/write) metadata based on a file-descriptor. /// The Metadata is stored within binary files at a temporary directory /// (some read files may be stored there as a whole, based on user configuration). /// Events are filtered beforehand, e.g. for matching user or include/exclude paths. class FileEventHandler { public: FileEventHandler(); ~FileEventHandler(); void handleCloseWrite(int fd); void handleCloseRead(int fd); FileEvents& fileEvents(); void clearEvents(); QString getTmpDirPath() const; public: Q_DISABLE_COPY(FileEventHandler) DISABLE_MOVE(FileEventHandler) private: void fillAllowedGroups(); bool userHasWritePermission(const struct stat& st); bool userHasReadPermission(const struct stat& st); bool readFileTypeMatches(const Settings::ScriptFileSettings& scriptCfg, int fd, const StrLight &fpath); void readLinkOfFd(int fd, StrLight &output); bool fileExtensionMatches(const Settings::StrLightSet &validExtensions, const StrLight &fullPath); bool mimeTypeMatches(int fd, const Settings::MimeSet& validMimetypes); bool generalReadSettingsSayLogIt(bool userHasWritePerm, const StrLight &filepath); bool scriptReadSettingsSayLogIt(bool userHasWritePerm, const StrLight &fpath, const os::stat_t& st, int fd); bool pathIsHidden(const StrLight &fullPath); QTemporaryDir m_filecacheDir; FileEvents m_fileEvents; HashControl m_hashControl; std::unordered_set m_groups; uid_t m_uid; // cached real uid int m_ourProcFdDirDescriptor; // holds open fd on /proc/self/fd QMimeDatabase m_mimedb; StrLight m_pathbuf; StrLight m_fdStringBuf; StrLight m_extensionBuf; const Settings::WriteFileSettings& r_wCfg; const Settings::ReadFileSettings& r_rCfg; const Settings::ScriptFileSettings& r_scriptCfg; const Settings::HashSettings& r_hashCfg; }; ================================================ FILE: src/common/fileevents.cpp ================================================ #include "fileevents.h" #include #include #include "stdiocpp.h" #include "strlight.h" #include "os.h" #include "osutil.h" #include "logger.h" #include "user_kernerl.h" int FileEvent::flags() const { return m_close_event.flags; } uint64_t FileEvent::mtime() const { return m_close_event.mtime; } size_t FileEvent::size() const { return m_close_event.size; } uint64_t FileEvent::mode() const { return m_close_event.mode; } HashValue FileEvent::hash() const { return (m_close_event.hash_is_null) ? HashValue() : HashValue(m_close_event.hash); } off_t FileEvent::fileContentSize() const { return m_close_event.bytes; } off_t FileEvent::fileContentStart() const { return m_fileContentStart; } const char *FileEvent::path() const { return m_path.data(); } FILE *FileEvent::file() const { return m_file; } void FileEvent::setPath(const char *path) { m_path = path; } ///////////////////////////////////////////////////////////////////////////////////// /// Read null-terminated c-string from file into buf, return len static int freadCstring(FILE* file, char* buf){ int c; int pathIdx = 0; while((c=stdiocpp::fgetc_unlocked(file)) != EOF){ buf[pathIdx++] = char(c); if(c == '\0'){ break; } } if(pathIdx == 0){ throw QExcIo(QString("EOF reached without expected null-terminator for file %1") .arg(osutil::findPathOfFd(fileno(file)).constData())); } return pathIdx; } bool FileEvents::isReadEvent(int flags) { switch (flags) { case O_RDONLY: case O_RDWR: return true; default: return false; } } bool FileEvents::isWriteEvent(int flags) { switch (flags) { case O_WRONLY: case O_RDWR: return true; default: return false; } } FileEvents::FileEvents() : m_wbuf_lastReadDir(PATH_MAX, '\0'), m_wbuf_lastWrittenDir(PATH_MAX, '\0') { } /// Write the file-event to the logfile. If storefd != -1 and /// the file has a st_size greater than zero, the whole file is copied /// as well. void FileEvents::write(int flags, const StrLight &path, const struct stat &st, HashValue hash, int storefd) { bool isREvent = isReadEvent(flags); m_eventTmp.flags = flags; m_eventTmp.mtime = st.st_mtime; m_eventTmp.size = st.st_size; m_eventTmp.mode = st.st_mode; m_eventTmp.hash = (hash.isNull()) ? 0 : hash.value(); m_eventTmp.hash_is_null = hash.isNull(); m_eventTmp.bytes = (storefd == -1) ? 0 : st.st_size; auto oldOffset = stdiocpp::ftell(m_file); stdiocpp::fwrite_unlocked(&m_eventTmp , sizeof(m_eventTmp), 1, m_file ); if(m_eventTmp.bytes > 0){ assert(os::ltell(storefd) == 0); int targetfd = fileno_unlocked(m_file); // kernel-copy has no idea of our buffer - flush it stdiocpp::fflush(m_file); auto sent = os::sendfile(targetfd, storefd, m_eventTmp.bytes); stdiocpp::fseek(m_file, os::ltell(targetfd), SEEK_SET); if(sent != off_t(m_eventTmp.bytes)){ // should happpen very rarely - seek back and correct file size logInfo << qtr("Could only collect %1 of %2 bytes for file %3") .arg(sent).arg(m_eventTmp.bytes) .arg(osutil::findPathOfFd(storefd).constData()); m_eventTmp.bytes = sent; stdiocpp::fseek(m_file, oldOffset, SEEK_SET); stdiocpp::fwrite_unlocked(&m_eventTmp , sizeof(m_eventTmp), 1, m_file ); stdiocpp::fseek(m_file, 0, SEEK_END); } if(isREvent){ m_rStoredFilesCount++; } if(isWriteEvent(flags)){ m_wStoredFilesCount++; } } writeFilenameToFile(path, isREvent); if(isREvent){ m_rEventCount++; } else { m_wEventCount++; } } void FileEvents::incrementDropCount(int eventType) { switch (eventType) { case O_RDONLY: m_rDroppedCount++; break; case O_WRONLY: m_wDroppedCount++; break; default: throw QExcProgramming("bad event type: " + QString::number(eventType)); } } void FileEvents::clear() { stdiocpp::ftruncate_unlocked(m_file); m_rStoredFilesCount = 0; m_wStoredFilesCount = 0; m_rEventCount = 0; m_wEventCount = 0; } FileEvent *FileEvents::read() { if(stdiocpp::fread_unlocked(&m_fileEvent.m_close_event, sizeof(shournalk_close_event), 1, m_file) != 1){ return nullptr; } if(m_fileEvent.fileContentSize() > 0){ // remember offset where file content begins, the caller may use this m_fileEvent.m_fileContentStart = stdiocpp::ftell(m_file); stdiocpp::fseek(m_file, m_fileEvent.fileContentSize(), SEEK_CUR); } auto filename_len = freadCstring(m_file, m_pathTmp); // If last and current directory-path is equal, the producer // may have omitted it after the first time, // for read- and write-events respectively. In this case, // the path does not start with a '/' auto& lastDir = (isWriteEvent(m_fileEvent.m_close_event.flags)) ? m_rbuf_lastWrittenDir : m_rbuf_lastReadDir; if(m_pathTmp[0] == '/'){ m_fileEvent.m_path = QByteArray(m_pathTmp, filename_len); lastDir = splitAbsPath(m_fileEvent.m_path).first; } else { // use dir-path from last time and append current filename m_fileEvent.m_path = pathJoinFilename( lastDir, QByteArray::fromRawData(m_pathTmp, filename_len)); } return &m_fileEvent; } FILE *FileEvents::file() const { return m_file; } void FileEvents::setFile(FILE *file) { m_fileEvent.m_file = file; m_file = file; } uint FileEvents::wEventCount() const { return m_wEventCount; } uint FileEvents::wDroppedCount() const { return m_wDroppedCount; } uint FileEvents::rEventCount() const { return m_rEventCount; } uint FileEvents::rDroppedCount() const { return m_rDroppedCount; } uint FileEvents::rStoredFilesCount() const { return m_rStoredFilesCount; } uint FileEvents::wStoredFilesCount() const { return m_wStoredFilesCount; } void FileEvents::writeFilenameToFile(const StrLight &path, bool isREvent) { auto & lastDir = (isREvent) ? m_wbuf_lastReadDir : m_wbuf_lastWrittenDir; const char* filename; size_t len_with_nul; int slashIdx = path.lastIndexOf('/'); if(unlikely(slashIdx < 0)){ throw QExcProgramming(QString("Invalid path %1").arg(path.c_str())); } if(unlikely(slashIdx == 0)){ // a file written to the root directory. // KISS and always write full path. // size + 1 -> include nul. stdiocpp::fwrite_unlocked(path.c_str(), path.size()+ 1, 1, m_file ); lastDir.resize(0); // invalidate last cached dir return; } if(slashIdx == int(lastDir.size()) && memcmp(lastDir.constData(), path.constData(), slashIdx) == 0){ // optimization: don't write full path but only filename (this // is later handled when reading back for read- and write-events // respectively). filename = path.c_str() + slashIdx + 1; len_with_nul = path.size() - slashIdx; // including nul } else { // write full path and remeber current directory for next time filename = path.c_str(); len_with_nul = path.size() + 1; // including nul lastDir.resize(slashIdx); memcpy(lastDir.data(), path.c_str(), slashIdx); } stdiocpp::fwrite_unlocked(filename, len_with_nul, 1, m_file ); } ================================================ FILE: src/common/fileevents.h ================================================ #pragma once #include #include "shournalk_user.h" #include "nullable_value.h" #include "strlight.h" class FileEvent { public: int flags() const; /* One of O_RDONLY, O_WRONLY, O_RDWR */ uint64_t mtime() const; size_t size() const; uint64_t mode() const; HashValue hash() const; off_t fileContentSize() const; off_t fileContentStart() const; const char* path() const; FILE *file() const; private: void setPath(const char* path); shournalk_close_event m_close_event; QByteArray m_path; off_t m_fileContentStart; FILE* m_file; friend class FileEvents; friend class DbCtrlTest; }; /// Write file-events (in binary format) to a log-file and /// read them later on. class FileEvents { public: static bool isReadEvent(int flags); static bool isWriteEvent(int flags); public: FileEvents(); void write(int flags, const StrLight &path, const struct stat &st, HashValue hash, int storefd=-1); void incrementDropCount(int eventType); void clear(); FileEvent* read(); FILE *file() const; void setFile(FILE *file); uint rEventCount() const; uint rDroppedCount() const; uint rStoredFilesCount() const; uint wEventCount() const; uint wDroppedCount() const; uint wStoredFilesCount() const; private: Q_DISABLE_COPY(FileEvents) void writeFilenameToFile(const StrLight& path, bool isREvent); FILE* m_file{}; FileEvent m_fileEvent{}; shournalk_close_event m_eventTmp{}; StrLight m_wbuf_lastReadDir; StrLight m_wbuf_lastWrittenDir; QByteArray m_rbuf_lastReadDir; QByteArray m_rbuf_lastWrittenDir; char m_pathTmp[PATH_MAX]; uint m_rEventCount{0}; uint m_rDroppedCount{0}; uint m_rStoredFilesCount{0}; uint m_wEventCount{0}; uint m_wDroppedCount{0}; uint m_wStoredFilesCount{0}; }; ================================================ FILE: src/common/generic_container.h ================================================ #pragma once #include template class TypeHas_push_back { template // signature has to match *exactly* (indclduing e.g. a 'const' at the end for a // const object method static std::true_type testSignature(void (T::*)(const typename ContainerT::value_type&)); template static decltype(testSignature(&T::push_back)) test(std::nullptr_t); template static std::false_type test(...); public: using type = decltype(test(nullptr)); static const bool value = type::value; }; /// Allow adding to containers which either implement push_back or /// insert. The container must provide Container::value_type. template ::value, std::nullptr_t>::type = nullptr> void addToContainer(ContainerT& t, const typename ContainerT::value_type& val ){ t.push_back(val); } template ::value, std::nullptr_t>::type = nullptr> void addToContainer(ContainerT& t, const typename ContainerT::value_type& val ){ t.insert(val); } ================================================ FILE: src/common/groupcontrol.cpp ================================================ #include #include #include "groupcontrol.h" #include "os.h" /// @return all 'real' groups, the calling user is a member /// of os::Groups groupcontrol::generateRealGroups(){ // according to man getgroups(2) "it is unspecified whether the // effective group ID of the calling process is included in the returned // list". os::Groups groups = os::getgroups(); gid_t egid = os::getegid(); if(egid != os::getgid()){ auto egidIter = std::find(groups.begin(), groups.end(), egid); if(egidIter != groups.end()){ groups.erase(egidIter); } } return groups; } /// create a one-to-one-mapping of param groups for the /proc/$pid/gid_map. /// Be a little smart and merge consecutive groups into one IdMapEntry. groupcontrol::GidMapRanges_T groupcontrol::generateGidMapRanges(const os::Groups &groups_){ if(groups_.empty()){ qDebug() << "generateGidMapRanges called with empty groups"; return std::vector>(); } auto groups = groups_; std::sort(groups.begin(), groups.end()); std::vector> mapRanges; mapRanges.reserve(groups.size()); mapRanges.emplace_back(groups.at(0)); // at least one group exists (real gid) for(size_t i=1; i < groups.size(); i++){ S_IdMapEntry & previousRange = mapRanges.back(); if(groups[i] == previousRange.idInNs + 1){ // gid-range possible previousRange.count++; } else { mapRanges.emplace_back(groups[i]); } } return mapRanges; } /////////////////////// PRIVATE ///////////////////////// ================================================ FILE: src/common/groupcontrol.h ================================================ #pragma once #include #include "idmapentry.h" #include "os.h" namespace groupcontrol { typedef std::vector > GidMapRanges_T; os::Groups generateRealGroups(); GidMapRanges_T generateGidMapRanges(const os::Groups & groups); } ================================================ FILE: src/common/hashcontrol.cpp ================================================ #include "hashcontrol.h" /// xxhash parts of a file (or the whole file in case of a small one) according to the /// specified hashmeta-parameters. /// @return hash-value of null, if 0 bytes were read. /// @throws ExcOs, CXXHashError HashValue HashControl::genPartlyHash(int fd, qint64 filesize, const HashMeta &hashMeta, bool resetOffset) { const off64_t seektstep = filesize / hashMeta.maxCountOfReads; auto hashRes = m_hash.digestFile( fd, hashMeta.chunkSize, seektstep , hashMeta.maxCountOfReads); HashValue hashVal; if(hashRes.count_of_bytes > 0){ if(resetOffset){ os::lseek(fd, 0, SEEK_SET); } hashVal = hashRes.hash; } return hashVal; } CXXHash &HashControl::getXXHash() { return m_hash; } ================================================ FILE: src/common/hashcontrol.h ================================================ #pragma once #include "nullable_value.h" #include "cxxhash.h" #include "hashmeta.h" #include "os.h" class HashControl { public: HashValue genPartlyHash(int fd, qint64 filesize, const HashMeta& hashMeta, bool resetOffset=true); CXXHash& getXXHash(); private: CXXHash m_hash; }; ================================================ FILE: src/common/hashmeta.cpp ================================================ #include "hashmeta.h" HashMeta::HashMeta(size_type chunks, size_type maxCountOfR) : chunkSize(chunks), maxCountOfReads(maxCountOfR) {} bool HashMeta::isNull() const { return chunkSize == 0 && maxCountOfReads == 0; } bool HashMeta::operator==(const HashMeta &rhs) const { return chunkSize == rhs.chunkSize && maxCountOfReads == rhs.maxCountOfReads; } ================================================ FILE: src/common/hashmeta.h ================================================ #pragma once #include #include #include "database/db_globals.h" struct HashMeta { typedef int size_type; HashMeta() = default; HashMeta(size_type chunks, size_type maxCountOfR); size_type chunkSize {}; size_type maxCountOfReads {}; qint64 idInDb {db::INVALID_INT_ID} ; bool isNull() const; bool operator==(const HashMeta& rhs) const; }; ================================================ FILE: src/common/idmapentry.h ================================================ #pragma once #include #include template class S_IdMapEntry { public: T idInNs; T idOutOfNs; T count; S_IdMapEntry(T inNs_, T idOutOfNs_, T count_=1) : idInNs(inNs_), idOutOfNs(idOutOfNs_), count(count_){} S_IdMapEntry(T idInBoth) : S_IdMapEntry(idInBoth, idInBoth){} /* S_IdMapEntry(const S_IdMapEntry& other) : idInNs(other.idInNs), idOutOfNs(other.idOutOfNs), count(other.count) {} */ /// @return the string in the form the gid or uid map expects: /// 0 1000 1 (including trailing newline) std::string to_string() const { return std::to_string(idInNs) + " " + std::to_string(idOutOfNs) + " " + std::to_string(count) + '\n'; } }; ================================================ FILE: src/common/interrupt_handler.cpp ================================================ #include #include #include "logger.h" #include "interrupt_handler.h" #include "os.h" #include "exccommon.h" static thread_local bool g_withinInterProtect = false; static thread_local bool g_signalOccurred = false; static thread_local std::vector g_occurred_sigs{}; // map of signal and index into the g_occurred_sigs vector static thread_local std::unordered_map g_sig_indeces{}; #ifdef __cplusplus extern "C" { #endif void ip_dummySighandler(int signum){ auto it = g_sig_indeces.find(signum); if(it == g_sig_indeces.end()){ const char msg[] = "ip_dummySighandler: error: failed to find signal...\n"; os::write(2, msg, sizeof(msg)-1); return; } g_occurred_sigs[it->second] = true; g_signalOccurred = true; } #ifdef __cplusplus } #endif InterruptProtect::InterruptProtect() {} InterruptProtect::InterruptProtect(int signum) : InterruptProtect(std::vector{signum}) {} InterruptProtect::InterruptProtect(const std::vector &sigs){ this->enable(sigs); } bool InterruptProtect::signalOccurred() { return g_signalOccurred; } InterruptProtect::~InterruptProtect() { if(! g_withinInterProtect){ return; } try { this->disable(); } catch (const std::exception& e) { logCritical << __func__ << e.what(); } } void InterruptProtect::enable(const std::vector &sigs) { if(g_withinInterProtect){ throw QExcProgramming(QString(__func__) + ": only one instance allowed per thread"); } g_withinInterProtect = true; g_signalOccurred = false; m_sigs = sigs; m_oldActions.resize(sigs.size()); g_occurred_sigs.resize(sigs.size(), false); struct sigaction act{}; act.sa_handler = ip_dummySighandler; sigemptyset (&act.sa_mask); act.sa_flags = SA_RESTART; for(int idx=0; idx < int(sigs.size()); idx++){ auto s = sigs[idx]; g_sig_indeces[s] = idx; os::sigaction(s, &act, &m_oldActions[idx]); } } void InterruptProtect::disable() { if(! g_withinInterProtect){ throw QExcProgramming(QString(__func__) + ": not enabled"); } // restore previous handlers for(int idx=0; idx < int(m_sigs.size()); idx++){ try { os::sigaction(m_sigs[idx], &m_oldActions[idx], nullptr); } catch (const os::ExcOs& e) { logCritical << e.what(); } } g_withinInterProtect = false; if(! g_signalOccurred){ return; } for (auto& it: g_sig_indeces) { auto sig = it.first; auto idx = it.second; if(g_occurred_sigs.at(idx)){ logDebug << "sending sig" << sig; kill(getpid(), sig); } } } ================================================ FILE: src/common/interrupt_handler.h ================================================ #pragma once #include #include #include #include "util.h" /// Defer processing of signals /// until destruction. Automatically restart (some) /// system-calls during that time (SA_RESTART). /// Only one instance allowed at a time per thread! class InterruptProtect { public: InterruptProtect(); InterruptProtect(int signum); InterruptProtect(const std::vector &sigs); ~InterruptProtect(); void enable(const std::vector &sigs); void disable(); bool signalOccurred(); public: Q_DISABLE_COPY(InterruptProtect) DEFAULT_MOVE(InterruptProtect) private: std::vector m_sigs{}; std::vector m_oldActions{}; }; ================================================ FILE: src/common/limited_priority_queue.h ================================================ #pragma once #include #include #include template class limited_priority_queue : public std::priority_queue { public: typedef typename container::value_type value_type; public: void push(const T& val) { std::priority_queue::push(val); if(static_cast(this->size()) > m_maxSize){ this->pop(); } } void setMaxSize(const size_t &maxSize){ m_maxSize = maxSize; } template PopContainerT popAll(bool reverse=false){ PopContainerT ret(this->size()); if(reverse){ for(auto it = ret.rbegin(); it != ret.rend(); ++it ){ *it = this->top(); this->pop(); } } else { for(auto & el : ret){ el = this->top(); this->pop(); } } return ret; } protected: size_t m_maxSize{std::numeric_limits::max()}; }; ================================================ FILE: src/common/logger.cpp ================================================ #include #include #include #include #include #include #include #include #include "logger.h" #include "qoutstream.h" #include "app.h" #include "exccommon.h" #include "os.h" #include "osutil.h" #include "staticinitializer.h" #include "cflock.h" namespace { QString g_logPreamble; const QtMsgType DEFAULT_VERBOSITY = QtMsgType::QtWarningMsg; QtMsgType g_verbosityLvl = DEFAULT_VERBOSITY; int g_verbosityLvlOrdinal=logger::msgTypeToOrdinal(DEFAULT_VERBOSITY); void messageHandler(QtMsgType msgType, const QMessageLogContext &context, const QString &msg) { int typeOrdinal = logger::msgTypeToOrdinal(msgType); #ifndef NDEBUG if (msgType == QtDebugMsg) { if(typeOrdinal >= g_verbosityLvlOrdinal){ QErr() << g_logPreamble << " Dbg: " << "(" << QFileInfo(context.file).fileName() <<":" << context.line << ") " << " pid " << getpid() << ": " << msg << '\n' ; } // Don't log debug messages to file return; } #else Q_UNUSED(context) #endif const QString dateTime = QDateTime::currentDateTime().toString( "yyyy-MM-dd HH:mm:ss"); QString msgTypeStr = logger::msgTypeToStr(msgType); if(typeOrdinal >= g_verbosityLvlOrdinal){ QErr() << g_logPreamble << " "<= QT_VERSION_CHECK(5, 5, 0) case QtInfoMsg : return "info"; #endif case QtWarningMsg : return "warning"; case QtCriticalMsg : return "critical"; case QtFatalMsg : return "fatal"; } static StaticInitializer initOnFirstCall( [&msgType](){ logWarning << "msgTypeToStr" << "unknown messagetype" << msgType; }); return "warning"; } /// Unfortunately qt messagetype are not really in a meaningful order - do it ourselves. int logger::msgTypeToOrdinal(QtMsgType msgType) { switch (msgType) { case QtDebugMsg: return 0; #if QT_VERSION >= QT_VERSION_CHECK(5, 5, 0) case QtInfoMsg : return 1; #endif case QtWarningMsg : return 2; case QtCriticalMsg : return 3; case QtFatalMsg : return 4; } static StaticInitializer initOnFirstCall( [&msgType](){ logWarning << "msgTypeToOrdinal" << "unknown messagetype" << msgType; }); return 2; } /// str is epected to be valid! QtMsgType logger::strToMsgType(const char *str) { switch (str[0]) { case 'd': return QtMsgType::QtDebugMsg; case 'i': #if QT_VERSION >= QT_VERSION_CHECK(5, 5, 0) return QtMsgType::QtInfoMsg; #else return QtMsgType::QtWarningMsg; #endif case 'w': return QtMsgType::QtWarningMsg; case 'c': return QtMsgType::QtCriticalMsg; case 'f': return QtMsgType::QtFatalMsg; default: break; } static StaticInitializer initOnFirstCall( [&str](){ logWarning << "strToMsgType" << "unknown messagetype" << str << "using default"; }); return DEFAULT_VERBOSITY; } logger::LogRotate::LogRotate(QString fullpath) : m_fullpath(std::move(fullpath)) { } QTextStream &logger::LogRotate::stream() { return m_stream; } const QFile& logger::LogRotate::file() const { return m_file; } void logger::LogRotate::openLogfileOrThrow() { if(! m_file.open(QFile::OpenModeFlag::Append | QIODevice::Text)){ throw QExcIo(qtr("Failed to open logile at %1 - %2").arg(m_fullpath, m_file.errorString())); } } /// Open the log file in append mode, rotate logfiles race-free, if too big. /// @throws ExcOs, QExcIo void logger::LogRotate::setup() { assert(! m_fullpath.isEmpty()); m_file.setFileName(m_fullpath); openLogfileOrThrow(); if(os::fstat(m_file.handle()).st_size > 50000){ // race condition - make sure to only rename once: CFlock l(m_file.handle()); l.lockExclusive(); // renamed already (by another process)? if(! osutil::findPathOfFd(m_file.handle()).endsWith("_old")){ const std::string path = m_fullpath.toStdString(); os::rename(path, path + "_old"); } l.unlock(); m_file.close(); // open or create the new logfile: openLogfileOrThrow(); } m_stream.setDevice(&m_file); } void logger::LogRotate::cleanup() { m_file.close(); } void logger::LogRotate::setFullpath(const QString &p) { m_fullpath = p; } ================================================ FILE: src/common/logger.h ================================================ #pragma once #include #include #include #include #define logDebug qDebug() #if QT_VERSION < QT_VERSION_CHECK(5, 4, 0) // maybe_todo: do something else about the quotes - or suggest // user to upgrade their qt-version... #define logInfo qWarning() // no info yet... #define logWarning qWarning() #define logCritical qCritical() #elif QT_VERSION < QT_VERSION_CHECK(5, 5, 0) #define logInfo qWarning().noquote() // no info yet... #else #define logInfo qInfo().noquote() #define logWarning qWarning().noquote() #define logCritical qCritical().noquote() #endif namespace logger { class LogRotate{ public: LogRotate(QString fullpath=QString()); void setup(); void cleanup(); void setFullpath(const QString& p); QTextStream& stream(); const QFile& file() const; private: void openLogfileOrThrow(); QString m_fullpath; QFile m_file; QTextStream m_stream; }; const char* msgTypeToStr(QtMsgType msgType); int msgTypeToOrdinal(QtMsgType msgType); QtMsgType strToMsgType(const char* str); void setup(const QString &preamble); void enableLogToFile(const QString &filename); void disableLogToFile(); void setVerbosityLevel(QtMsgType lvl); void setVerbosityLevel(const char* str); QtMsgType getVerbosityLevel(); LogRotate& getLogRotate(); const QString &logDir(); } ================================================ FILE: src/common/oscpp/CMakeLists.txt ================================================ add_library(oscpp_lib cflock.cpp excos.cpp fdentries.cpp os.cpp osutil.cpp oscaps.cpp ) target_link_libraries(oscpp_lib PUBLIC Qt5::Core ${CMAKE_DL_LIBS} lib_util cap ) ================================================ FILE: src/common/oscpp/cflock.cpp ================================================ #include #include #include "cflock.h" #include "os.h" #include "osutil.h" #ifndef NDEBUG static bool checkFdFlockFlags(int fd, int operation){ int flags = os::getFdStatusFlags(fd); switch (operation) { case LOCK_EX: if(flags & O_WRONLY || flags & O_RDWR) { return true; } std::cerr << "LOCK_EX: fd opened RDONLY\n"; break; case LOCK_SH: if(!(flags & O_WRONLY)) { return true; } std::cerr << "LOCK_SH: fd opened WRONLY\n"; break; default: std::cerr << "Bad fd operation " << operation << "\n"; break; } return false; } #endif /// In order to catch further possible NFS idiosyncrasies (bugs?), better never lock /// blocking. See also e.g. shournal's commit 1918f88. static void doLockNB(int fd, int operation){ for(int i=0; ; i++){ try { os::flock(fd, operation | LOCK_NB); return; } catch (const os::ExcOs& ex) { if(ex.errorNumber() != EWOULDBLOCK){ throw; } if(i>9){ std::cerr << "doLockNB: gave up waiting for lock\n"; throw; } osutil::randomSleep(1 *1000, 3 *1000); } } } CFlock::CFlock(int fd) : m_fd(fd) {} CFlock::~CFlock() { if(m_isLockedSH || m_isLockedEX){ try { unlock(); } catch (const os::ExcOs& e) { // should never happen std::cerr << e.what() << "\n"; } } } void CFlock::lockExclusive() { assert(checkFdFlockFlags(m_fd, LOCK_EX)); if(m_isLockedSH){ throw QExcProgramming("Due to NFS issues, upgrading shared to exclusive " "locks is not supported. Please unlock() first."); } doLockNB(m_fd, LOCK_EX); m_isLockedSH = false; m_isLockedEX = true; } void CFlock::lockShared() { assert(checkFdFlockFlags(m_fd, LOCK_SH)); doLockNB(m_fd, LOCK_SH); m_isLockedEX = false; m_isLockedSH = true; } void CFlock::unlock() { os::flock(m_fd, LOCK_UN); m_isLockedEX = false; m_isLockedSH = false; } ================================================ FILE: src/common/oscpp/cflock.h ================================================ #pragma once /// Wrapper class for flock. /// Due to NFS emulating flock as fcntl(2) byte-range locks /// the fd open mode must match the locking operations: /// In order to place a shared lock, fd must be open for reading, /// In order to place an exclusive lock, fd must be open for writing. To /// place both types of lock, open a file read-write. class CFlock { public: /// fd should in general be opened read-write (see above)! CFlock(int fd); ~CFlock(); void lockExclusive(); void lockShared(); void unlock(); public: CFlock(const CFlock &) = delete ; void operator=(const CFlock &) = delete ; private: int m_fd; bool m_isLockedSH{false}; bool m_isLockedEX{false}; }; ================================================ FILE: src/common/oscpp/excos.cpp ================================================ #include #include #include #include #include "excos.h" #include "translation.h" os::ExcOsCommon::ExcOsCommon(std::string text) : m_descrip(std::move(text)) {} const char *os::ExcOsCommon::what() const noexcept { return m_descrip.c_str(); } os::ExcOsCommon::ExcOsCommon() = default; os::ExcOs::ExcOs(const std::string &preamble) : m_errorNumber(errno) { if(preamble.empty()){ m_descrip += "ExcOs occurred:"; } else { m_descrip += preamble; } m_descrip += " (" + std::to_string(errno) + "): " + translation::strerror_l(errno); #ifndef NDEBUG // maybe_todo: also outside of debug - performance? m_descrip += "\n" + generate_trace_string(); #endif } /// If errorNumber is zero, no preamble is autogenerated. os::ExcOs::ExcOs(const std::string &preamble, int errorNumber) : m_errorNumber(errorNumber) { if(preamble.empty()){ m_descrip += "ExcOs occurred:"; } else { m_descrip += preamble; } if(errorNumber != 0){ m_descrip += " (" + std::to_string(errorNumber) + "): " + translation::strerror_l(errorNumber); } } int os::ExcOs::errorNumber() const { return m_errorNumber; } /// @param status: depending on typeOfTerm, currently only the signal number (NOT_IMPLEMENTED) os::ExcProcessExitNotNormal::ExcProcessExitNotNormal(int status, TypeOfTerm typeOfTerm) : ExcOsCommon ("Process terminated not normally: " + std::to_string(status)), m_status(status), m_typeOfTermination(typeOfTerm) {} int os::ExcProcessExitNotNormal::status() const { return m_status; } os::ExcProcessExitNotNormal::TypeOfTerm os::ExcProcessExitNotNormal::typeOfTermination() const { return m_typeOfTermination; } ================================================ FILE: src/common/oscpp/excos.h ================================================ #pragma once #include #include namespace os { class ExcOsCommon : public std::exception { public: ExcOsCommon(std::string text); const char *what () const noexcept override; protected: ExcOsCommon(); std::string m_descrip; }; /// Exception with custom preamble which automatically /// determines errno and builds an error description string /// on what(). class ExcOs : public ExcOsCommon { public: ExcOs(const std::string & preamble=std::string()); ExcOs(const std::string & preamble, int errorNumber); int errorNumber() const; protected: int m_errorNumber; }; class ExcTooFewBytesWritten : public ExcOs { public: using ExcOs::ExcOs; }; class ExcReadLink : public ExcOs { public: using ExcOs::ExcOs; }; class ExcProcessExitNotNormal : public ExcOsCommon { public: enum TypeOfTerm { SIG, COREDUMP, NOT_IMPLEMENTED }; ExcProcessExitNotNormal(int status, TypeOfTerm typeOfTerm); int status() const; TypeOfTerm typeOfTermination() const; protected: int m_status; TypeOfTerm m_typeOfTermination; }; } // namespace os ================================================ FILE: src/common/oscpp/fdentries.cpp ================================================ #include #include #include "fdentries.h" #include "os.h" osutil::FdEntries::Iterator::Iterator(DIR *dir, int dirfd): m_iter_dir(dir), m_iter_fd(-1), m_iter_dirfd(dirfd) { if(dir != nullptr){ this->operator++(); } // else we are the end-iterator } osutil::FdEntries::Iterator osutil::FdEntries::Iterator::operator++() { struct dirent *ent; while ((ent = ::readdir (m_iter_dir)) != nullptr) { if(ent->d_name[0] == '.'){ continue; } int fd = std::stoi(ent->d_name); if(fd == m_iter_dirfd){ continue; } m_iter_fd = fd; return *this; } m_iter_fd = -1; return *this; } bool osutil::FdEntries::Iterator::operator!=(const FdEntries::Iterator &other) const { return m_iter_fd != other.m_iter_fd; } int osutil::FdEntries::Iterator::operator*() const { return m_iter_fd; } osutil::FdEntries::FdEntries() { m_dir = ::opendir ("/proc/self/fd"); if (m_dir == nullptr) { throw os::ExcOs("opendir failed: /proc/self/fd "); } m_dirLoc = telldir(m_dir); if(m_dirLoc == -1){ throw os::ExcOs("telldir failed"); } } osutil::FdEntries::~FdEntries() { if(closedir (m_dir) == -1){ std::cerr << __func__ << " closedir failed: " << strerror(errno) << "(" << errno <<")\n"; } } osutil::FdEntries::Iterator osutil::FdEntries::begin() const { ::seekdir(m_dir, m_dirLoc); return {m_dir, dirfd(m_dir)}; } osutil::FdEntries::Iterator osutil::FdEntries::end() const { return {nullptr, -1}; } ================================================ FILE: src/common/oscpp/fdentries.h ================================================ #pragma once #include #include "util.h" namespace osutil { /// Allow iterating for entries in /proc/self/fd. /// The dir-fd internally used is skipped. class FdEntries { public: class Iterator { friend class FdEntries; public: Iterator operator++(); bool operator!=(const Iterator & other) const; int operator*() const; private: Iterator(DIR * dir, int dirfd); DIR* m_iter_dir; int m_iter_fd; int m_iter_dirfd; // fd of *our* DIR stream }; public: FdEntries(); ~FdEntries(); Iterator begin() const; Iterator end() const; public: Q_DISABLE_COPY(FdEntries) DEFAULT_MOVE(FdEntries) private: DIR* m_dir; long m_dirLoc; }; } ================================================ FILE: src/common/oscpp/os.cpp ================================================ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "os.h" #include "excos.h" #include "cleanupresource.h" #include "osutil.h" const int os::OPEN_WRONLY = O_WRONLY; const int os::OPEN_RDONLY = O_RDONLY; const int os::OPEN_RDWR = O_RDWR; const int os::OPEN_CLOEXEC = O_CLOEXEC; const int os::OPEN_NONBLOCK = O_NONBLOCK; const int os::OPEN_CREAT = O_CREAT; const int os::OPEN_EXCL = O_EXCL; const int os::OPEN_TRUNC = O_TRUNC; static bool& retryOnInterrupt(){ thread_local static bool retryIt = false; return retryIt; } void os::setRetryOnInterrupt(bool val) { bool & valRef = retryOnInterrupt(); valRef = val; } /// @throws ExcOs os::stat_t os::fstat(int fd) { struct stat stat_; if(::fstat(fd, &stat_) == -1){ throw ExcOs("fstat " + std::to_string(fd) + " failed"); } return stat_; } /// @throws ExcOs os::stat_t os::stat(const char *filename) { os::stat_t st; if(::stat(filename, &st) == -1){ throw ExcOs("stat " + std::string(filename) + " failed"); } return st; } /// @throws ExcOs void os::getresgid(gid_t *rgid, gid_t *egid, gid_t *sgid) { if(::getresgid(rgid, egid, sgid) == -1) { throw ExcOs("getresgid failed"); } } /// @throws ExcOs void os::getresuid(uid_t *ruid, uid_t *euid, uid_t *suid) { if(::getresuid(ruid, euid, suid) == -1) { throw ExcOs("getresuid failed"); } } /// @throws ExcOs void os::setgid(gid_t gid) { if(::setgid(gid) == -1){ throw ExcOs("setgid failed"); } } /// @throws ExcOs void os::setuid(uid_t uid) { if(::setuid(uid) == -1){ throw ExcOs("setuid failed"); } } std::string os::getHomeDir() { char *homedir = getenv("HOME"); // fallback if (homedir == nullptr) { homedir = ::getpwuid(getuid())->pw_dir; } return homedir; } std::string os::getCacheDir() { char *cacheDir = getenv("XDG_CACHE_HOME"); std::string cacheDirStr; // fallback if (cacheDir == nullptr) { cacheDirStr = os::getHomeDir() + "/.cache"; } else { cacheDirStr = cacheDir; } return cacheDirStr; } /// @throws ExcOs void os::close(int fd) { if(::close(fd) == -1){ throw ExcOs("close failed for fd " + std::to_string(fd)); } } /// @throws ExcOs int os::open(const char* filename, int flags, bool clo_exec, mode_t mode){ if(clo_exec){ flags |= O_CLOEXEC; } // quoting the man: // the mode argument must be supplied when // O_CREAT or O_TMPFILE is specified in flags; if neither O_CREAT // nor O_TMPFILE is specified, then mode is ignored. // So we always pass the mode. int fd = ::open(filename, flags, mode); if(fd == -1) { throw ExcOs("open " + std::string(filename) + " failed"); } return fd; } /// @param throwIfLessBytesWritten: if true, throw if the number of written bytes /// is less than requested (in param n) /// @throws ExcOs, ExcTooFewBytesWritten ssize_t os::write(int fd, const void *buf, size_t n, bool throwIfLessBytesWritten) { auto ret = ::write(fd, buf, n); if(ret == -1){ throw ExcOs("write failed"); } if(throwIfLessBytesWritten && ret < static_cast(n)){ throw ExcTooFewBytesWritten("Too few bytes written for file " + osutil::findPathOfFd(fd), 0); } return ret; } /// @overload /// @throws ExcOs ssize_t os::write(int fd, const std::string &buf, bool throwIfLessBytesWritten) { return os::write(fd, buf.c_str(), buf.size(), throwIfLessBytesWritten); } /// @overload /// @throws ExcOs ssize_t os::write(int fd, const QByteArray &buf, bool throwIfLessBytesWritten) { return os::write(fd, buf.data(), static_cast(buf.size()), throwIfLessBytesWritten); } /// Return a pipe-array, where idx 0 holds read- idx 1 /// holds the write end /// @throws ExcOs os::Pipes_t os::pipe(int flags, bool clo_exec) { if(clo_exec){ flags |= O_CLOEXEC; } Pipes_t fds; if(::pipe2(fds.data(), flags) == -1){ throw ExcOs("pipe failed, used flags: " + std::to_string(flags)); } return fds; } /// @throws ExcOs pid_t os::fork() { auto pid = ::fork(); if(pid == -1){ throw ExcOs("fork failed"); } return pid; } /// @throws ExcOs void os::unlinkat(int dirfd, const char *pathname, int flags) { if(::unlinkat(dirfd, pathname, flags) == -1){ throw ExcOs(std::string("unlinkat failed for ") + pathname); } } /// @throws ExcOs void os::umount(const std::string &specialFile) { int res = ::umount(specialFile.c_str()); if(res == -1){ throw ExcOs("umount failed"); } } /// @throws ExcOs void os::setpriority(int which, id_t who, int prio) { if(::setpriority(which, who, prio) == -1){ throw ExcOs("setpriority failed"); } } /// @throws ExcOs void os::setegid(gid_t gid) { if(::setegid(gid) == -1){ throw ExcOs("setegid failed"); } } /// @throws ExcOs void os::seteuid(uid_t uid) { if(::seteuid(uid) == -1){ throw ExcOs("seteuid failed"); } } uid_t os::getuid() { return ::getuid(); } gid_t os::getgid() { return ::getgid(); } /// @throws ExcOs uid_t os::getsuid() { uid_t ruid, euid, suid; os::getresuid(&ruid, &euid, &suid); return suid; } /// @throws ExcOs gid_t os::getsgid() { gid_t rgid, egid, sgid; os::getresgid(&rgid, &egid, &sgid); return sgid; } void os::mkfifo(const char *pathname, mode_t mode){ if(::mkfifo(pathname, mode) == -1){ throw os::ExcOs(std::string("mkfifo ") + pathname + " failed"); } } /// aquivalent of 'mkdir -p' to create necessary /// parts of a path recursively void os::mkpath(std::string s, mode_t mode) { size_t pre=0, pos; std::string dir; int mdret; if(s[s.size()-1]!='/'){ s+='/'; } while((pos=s.find_first_of('/',pre))!=std::string::npos){ dir=s.substr(0,pos++); pre=pos; if(dir.empty()){ continue; } if((mdret=mkdir(dir.c_str(),mode)) && errno!=EEXIST){ throw ExcOs("mkpath failed"); } } } int __os::openat(int dirfd, const char* filename, int flags, bool clo_exec, mode_t mode) { if(clo_exec){ flags |= O_CLOEXEC; } int fd = ::openat(dirfd, filename, flags, mode); if(fd == -1){ throw os::ExcOs("openat " + std::string(filename) + " failed"); } return fd; } /// Be careful, according to man, "It is unspecified whether the effective group /// ID of the calling process is included in the returned list." /// @throws ExcOs os::Groups os::getgroups() { os::Groups groups; int ngroups = ::getgroups(0, groups.data()); if(ngroups == -1){ throw ExcOs("getgroups"); } groups.resize(static_cast(ngroups)); ngroups = ::getgroups(static_cast(groups.size()), groups.data()); if(ngroups == -1){ throw ExcOs("getgroups"); } return groups; } uid_t os::geteuid() { return ::geteuid(); } gid_t os::getegid() { return ::getegid(); } /// returns a list of all group-ids on the system (/etc/group) /// @throws ExcOs std::vector os::queryGroupIds() { std::vector ids; group* grp; ::setgrent(); errno = 0; while((grp = getgrent()) != nullptr){ ids.push_back(grp->gr_gid); } if(errno != 0){ throw ExcOs("getgrent"); } endgrent(); return ids; } /// @throws ExcKernelVersionParse, ExcOs os::KernelVersion os::getKernelVersion() { utsname uname_; if(uname(&uname_) == -1){ throw ExcOs("uname"); } KernelVersion version; os::KernelVersion::size_type verIdx=0; std::string release = uname_.release; std::string currentNumber; for(const char c : release){ if(std::isdigit(c)){ currentNumber += c; } else{ version[static_cast(verIdx++)] = std::stoi(currentNumber); currentNumber.clear(); if(verIdx == version.size()){ break; } } } if(verIdx != version.size()){ throw ExcKernelVersionParse(); } return version; } int os::unshare(int flags) { int ret = ::unshare(flags); if(ret == -1){ throw ExcOs("unshare failed"); } return ret; } /// returns the names of the directory contents /// @throws ExcOs std::vector os::ls(const std::string &dirname_, os::DirFilter filter) { DIR *dir; struct dirent *ent; if ((dir = ::opendir (dirname_.c_str())) == nullptr) { throw ExcOs("opendir failed: " + dirname_); } auto closeLater = finally([&dir] { closedir (dir); }); std::vector files; while ((ent = ::readdir (dir)) != nullptr) { if((filter & DirFilter::NoDot && strcmp(ent->d_name, "." ) == 0) || (filter & DirFilter::NoDotDot && strcmp(ent->d_name, ".." ) == 0)){ continue; } files.emplace_back(ent->d_name); } return files; } pid_t os::getpid() { return ::getpid(); } /// @param cleanStatusOnSuccess:: If the child terminated normally, clean status, so /// it only contains the 8 least significant bits (WEXITSTATUS). /// @throws ExcOs, ExcProcessExitNotNormal pid_t os::waitpid(pid_t pid, int *status, int options, bool cleanStatusOnSuccess) { int internalStatus = 1; if(status == nullptr){ status = &internalStatus; } pid_t ret = ::waitpid (pid, status, options) ; if(ret == -1){ throw ExcOs("waitpid failed for pid " + std::to_string(pid) ); } if (! WIFEXITED (*status)){ // process did not call exit and did not return from main normally // find out what happended int extractedStatus=-1; ExcProcessExitNotNormal::TypeOfTerm typeOfTerm = ExcProcessExitNotNormal::NOT_IMPLEMENTED; if(WIFSIGNALED(*status)){ extractedStatus = WTERMSIG(*status); if(WCOREDUMP(*status)){ typeOfTerm = ExcProcessExitNotNormal::COREDUMP; } else { typeOfTerm = ExcProcessExitNotNormal::SIG; } } // There are some other cases, which could be checked: // WIFSTOPPED/WIFCONTINUED throw ExcProcessExitNotNormal(extractedStatus, typeOfTerm); } if(cleanStatusOnSuccess){ *status = WEXITSTATUS(*status); } return ret; } /// @throws ExcOs void os::exec (const char *filename, char * const argv[], char * const envp[]) { if(envp == nullptr){ envp = environ; } ::execvpe(filename, argv, envp); // only get here on error throw ExcOs("executing " + std::string(filename) + " failed" ); } /// @throws ExcOs void os::exec(const std::vector &args, char * const envp[]) { if (args.empty()) { throw std::invalid_argument( "exec called with empty args"); } std::vector pointerVec(args.size() + 1 ); // + 1 because of terminating NULL for(unsigned i = 0; i < args.size() ; ++i) { pointerVec[i] = const_cast(args[i].c_str()); } pointerVec.back() = nullptr; char** result = pointerVec.data(); os::exec(result[0], result, envp); } /// @throws ExcOs void os::setgroups(const os::Groups &groups) { if(::setgroups(groups.size(), groups.data()) == -1){ std::stringstream result; std::copy(groups.begin(), groups.end(), std::ostream_iterator(result, " ")); throw ExcOs("setgroups failed. Used groups: " + result.str() ); } } /// @throws ExcOs off_t os::lseek (int fd, off_t offset, int whence) { off_t ret = ::lseek(fd, offset, whence); if(ret == -1){ throw os::ExcOs("lseek failed. fd: " + std::to_string(fd) + " offset: " + std::to_string(offset)); } return ret; } /// Like ftell for a file descriptor /// @throws ExcOs off_t os::ltell(int fd) { return os::lseek(fd, 0, SEEK_CUR); } void os::mount(const char *source, const char *target, const char *fstype, unsigned long rwflag, const void *data) { if(::mount(source, target, fstype, rwflag, data) == -1){ throw ExcOs("Mount from " + strFromCString(source) + " to " + strFromCString(target) + " failed"); } } /// @throws ExcOs void os::mount(const std::string &source, const std::string& target, const char *fstype, unsigned long rwflag, const void *data) { os::mount(source.c_str(), target.c_str(), fstype, rwflag, data); } /// @throws ExcOs void *os::dlsym(void *handle, const char *symbol) { auto sym_ = ::dlsym(handle, symbol); if(sym_ == nullptr){ char* errStr = dlerror(); if(errStr == nullptr){ throw ExcOs("dlsym returned null, but dlerror was also null..."); } throw ExcOs("dlsym failed: " + std::string(errStr)); } return sym_; } /// @throws ExcOs void os::setns(int fd, int nstype) { if(::setns(fd, nstype) == -1 ){ throw ExcOs("Failed to enter namespace " + std::to_string(nstype)); } } pid_t os::setsid() { pid_t sid = ::setsid(); if(sid == static_cast(- 1) ){ throw ExcOs("setsid failed"); } return sid; } bool os::exists(const std::string &name) { struct stat buffer; return (::stat (name.c_str(), &buffer) == 0); } /// Shortcut for fcntl(F_SETFD, ... /// @throws ExcOs void os::setFdDescriptorFlags(int fd, int flags) { if(::fcntl(fd, F_SETFD, flags) == -1){ throw ExcOs(std::string(__func__) + " failed for fd "+ std::to_string(fd) + " (flags " + std::to_string(flags) + ")" ); } } void os::setFdStatusFlags(int fd, int flags){ if(::fcntl(fd, F_SETFL, flags) == -1){ throw ExcOs(std::string(__func__) + " failed for fd "+ std::to_string(fd) + " (flags " + std::to_string(flags) + ")" ); } } int os::dup(int oldfd){ int newfd = ::dup(oldfd); if(newfd == -1){ throw ExcOs(std::string(__func__) + " failed for fd "+ std::to_string(oldfd) ); } return newfd; } void os::dup2(int oldfd, int newfd) { if(::dup2(oldfd, newfd) == -1){ throw ExcOs(std::string(__func__) + " failed for fds "+ std::to_string(oldfd) + ", " + std::to_string(newfd) ); } } void os::dup3(int oldfd, int newfd, int flags) { if(::dup3(oldfd, newfd, flags) == -1){ throw ExcOs(std::string(__func__) + " failed for fds "+ std::to_string(oldfd) + ", " + std::to_string(newfd) ); } } /// @throws ExcOs void os::fchdir(int fd) { if(::fchdir(fd) == -1){ throw ExcOs("fchdir failed"); } } /// @throws ExcOs void os::fchmod(int fd, mode_t mode) { if(::fchmod(fd, mode) == -1){ throw ExcOs("fchmod failed"); } } void os::sigaction(int signum, const struct sigaction *act, struct sigaction *oldact) { if(::sigaction(signum, act, oldact) == -1){ throw ExcOs(std::string(__func__) + "failed"); } } /// @throws ExcOs /// @return the signal number int os::sigwait(const sigset_t *set) { int sig; int ret = ::sigwait(set, &sig); if(ret != 0){ // not using errno here! throw ExcOs("sigwait failed", ret); } return sig; } /// @throws ExcOs void os::sigfillset(sigset_t *set) { if(::sigfillset(set) != 0){ throw ExcOs("sigfillset failed"); } } /// Always returns the old handler /// @throws ExcOs sighandler_t os::signal(int sig, sighandler_t handler) { auto oldhandler = ::signal(sig, handler); if ( oldhandler == SIG_ERR) { throw ExcOs("signal failed"); } return oldhandler; } void os::symlink(const char *target, const char *linkpath) { if(::symlink(target, linkpath) == -1){ throw ExcOs("symlink failed"); } } /// @throws ExcOs void os::chdir(const char *path) { if(::chdir(path) == -1){ throw ExcOs("chdir failed"); } } void os::chdir(const std::string &path) { return os::chdir(path.data()); } /// shortcut for fcntl(fd , F_GETFL) /// @throws ExcOs int os::getFdStatusFlags(int fd) { int statusflags = fcntl(fd , F_GETFL); if(statusflags == -1){ throw ExcOs("failed to get status flags from fd " + std::to_string(fd)); } return statusflags; } /// shortcut for fcntl(fd , F_GETFD) /// @throws ExcOs int os::getFdDescriptorFlags(int fd) { int statusflags = fcntl(fd , F_GETFD); if(statusflags == -1){ throw ExcOs("failed to get descriptor flags from fd " + std::to_string(fd)); } return statusflags; } /// @throws ExcOs ssize_t os::read(int fd, void *buf, size_t nbytes, bool retryOnInterrupt) { while (true) { auto read = ::read(fd, buf, nbytes); if(read == -1){ if(retryOnInterrupt && errno == EINTR){ continue; } throw ExcOs("read failed"); } return read; } } void os::readlinkat(int dirfd, const char *filename, std::string &output){ folly::resizeWithoutInitialization(output, PATH_MAX); const ssize_t path_len = ::readlinkat(dirfd, filename, &output[0], PATH_MAX); if (path_len == -1 ){ throw ExcReadLink("readlinkat failed for file " + std::string(filename)); } folly::resizeWithoutInitialization(output, static_cast(path_len)); } void os::rmdir(const char *path){ if (::rmdir(path) == -1 ){ throw ExcReadLink("rmdir failed for " + std::string(path)); } } /// @return the number of bytes send /// @throws ExcOs size_t os::sendmsg(int fd, const msghdr *message, int flags) { while (true){ ssize_t ret = ::sendmsg(fd, message, flags); if (ret == -1) { if(retryOnInterrupt() && errno == EINTR){ continue; } throw ExcOs(std::string(__func__) + " failed"); } return static_cast(ret); } } /// offset of in_fd is *not* modified /// @return number of sent bytes off_t os::sendfile(int out_fd, int in_fd, size_t count, off_t offset) { ssize_t sizeToSend = count; while (true) { auto sent = ::sendfile(out_fd, in_fd, &offset, sizeToSend); if(sent == -1){ throw ExcOs(std::string(__func__) + " failed"); } sizeToSend -= static_cast(sent); assert(sizeToSend >= 0); if(sizeToSend <= 0 || sent == 0){ break; } } return count - sizeToSend; } /// returns the number of bytes received /// @throws ExcOs size_t os::recvmsg(int fd, msghdr *message, int flags) { while (true){ ssize_t ret = ::recvmsg(fd, message, flags); if (ret == -1) { if(retryOnInterrupt() && errno == EINTR){ continue; } throw ExcOs(std::string(__func__) + " failed"); } return static_cast(ret); } } os::SocketPair_t os::socketpair(int domain, int type_, int protocol) { SocketPair_t pair; if(::socketpair(domain, type_, protocol, pair.data()) == -1){ throw ExcOs(std::string(__func__) + " failed"); } return pair; } void os::unsetenv(const char *name) { if(::unsetenv(name) == -1){ throw ExcOs(std::string(__func__) + " failed"); } } /// Return a rather random array of signals which are catchable and would by default /// cause a process to end. const std::vector& os::catchableTermSignals() { static const std::vector sigs {SIGHUP, SIGINT, SIGQUIT, SIGTERM, SIGPIPE}; return sigs; } void os::flock(int fd, int operation) { if(::flock(fd, operation) == -1){ throw os::ExcOs("flock failed"); } } ================================================ FILE: src/common/oscpp/os.h ================================================ #pragma once #include #include #include #include #include #include #include #include #include #include #include #include #include "excos.h" #include "util.h" /// Private functions - internal use namespace __os { int openat(int dirfd, const char* filename, int flags, bool clo_exec, mode_t mode); } /// Simple wrappers for several os calls /// which throw exceptions on error. namespace os { void setRetryOnInterrupt(bool val); // read/write by owner, read only by group and others static const int DEFAULT_CREAT_FLAGS = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; extern const int OPEN_WRONLY; extern const int OPEN_RDONLY; extern const int OPEN_RDWR; extern const int OPEN_CLOEXEC; extern const int OPEN_NONBLOCK; extern const int OPEN_EXCL; extern const int OPEN_CREAT; extern const int OPEN_TRUNC; class ExcKernelVersionParse : public std::exception { }; typedef std::array Pipes_t; typedef std::array SocketPair_t; typedef std::vector Groups; typedef std::array KernelVersion; // major minor patch typedef struct stat stat_t; enum DirFilter { NoDot=0x2000, NoDotDot=0x4000, NoDotAndDotDot=NoDot | NoDotDot }; const std::vector &catchableTermSignals(); void chdir(const char *path); void chdir(const std::string& path); template void chmod(const Str_t& path, mode_t mode); void close(int fd); void *dlsym (void *handle, const char *symbol); pid_t fork(); stat_t fstat(int fd); stat_t stat(const char* filename); int dup(int oldfd); void dup2(int oldfd, int newfd); void dup3(int oldfd, int newfd, int flags); bool exists(const std::string& name); void fchdir(int fd); void fchmod(int fd, mode_t mode); int getFdStatusFlags(int fd); int getFdDescriptorFlags(int fd); std::string getHomeDir(); std::string getCacheDir(); void getresgid(gid_t *rgid, gid_t *egid, gid_t *sgid); void getresuid(uid_t *ruid, uid_t *euid, uid_t *suid); uid_t getuid(); uid_t geteuid(); uid_t getsuid(); gid_t getgid(); gid_t getegid(); gid_t getsgid(); Groups getgroups(); pid_t getpid(); template Str_t getUserName(); [[ noreturn ]] void exec(const char *filename, char *const argv[], char *const envp[]=nullptr); [[ noreturn ]] void exec(const std::vector &args, char *const envp[]=nullptr); template [[ noreturn ]] void exec(const ContainerT &args, char *const envp[]=nullptr); void flock(int fd, int operation); std::vector ls(const std::string & dirname_, DirFilter filter=DirFilter::NoDotAndDotDot); off_t lseek (int fd, off_t offset, int whence); off_t ltell(int fd); void mkfifo(const char *pathname, mode_t mode); void mkpath(std::string s, mode_t mode=0755); void mount (const std::string & source, const std::string &target, const char *fstype, unsigned long int rwflag, const void *data=nullptr); void mount (const char* source, const char* target, const char *fstype, unsigned long int rwflag, const void *data=nullptr); int open(const char* filename, int flags, bool clo_exec=true, mode_t mode=DEFAULT_CREAT_FLAGS); template int open(const Str_t& filename, int flags, bool clo_exec=true, mode_t mode=DEFAULT_CREAT_FLAGS); // int openat(int dirfd, const char *filename, int flags, bool clo_exec=true, // mode_t mode=DEFAULT_CREAT_FLAGS); template int openat(int dirfd, const Str_t& filename, int flags,bool clo_exec=true, mode_t mode=DEFAULT_CREAT_FLAGS ); int unshare (int flags); Pipes_t pipe(int flags=0, bool clo_exec=true); template Str_t readlink (const char* filename); template Str_t readlink (const Str_t & filename); template Str_t readlinkat (int dirfd, const Str_t & filename); void readlinkat (int dirfd, const char* filename, std::string & output); template void readlinkat (int dirfd, const Str_t & filename, Str_t & output); ssize_t read (int fd, void *buf, size_t nbytes, bool retryOnInterrupt=false); template Str_t readStr(int fd, size_t nbytes, bool retryOnInterrupt=false); size_t recvmsg (int fd, struct msghdr *message, int flags=0); template void remove(const Str_t & path); template void rename(const Str_t & old, const Str_t & new_); void rmdir(const char *path); size_t sendmsg (int fd, const struct msghdr *message, int flags=0); off_t sendfile(int out_fd, int in_fd, size_t count, off_t offset=0); template off_t sendfile(const Str_t& out_path, const Str_t& in_path, size_t count); void setFdDescriptorFlags(int fd, int flags); void setFdStatusFlags(int fd, int flags); void setgid (gid_t gid); void setgroups (const Groups & groups); void setegid(gid_t gid); void setpriority(int which, id_t who, int prio); template void setenv(const Str_t& name, const Str_t& value, bool overwrite=true); void seteuid(uid_t uid); void setuid (uid_t uid); void setns (int fd, int nstype); pid_t setsid(); void sigaction(int signum, const struct sigaction *act, struct sigaction *oldact); int sigwait(const sigset_t *set); void sigfillset(sigset_t *set); sighandler_t signal(int sig, sighandler_t handler); void symlink(const char *target, const char *linkpath); SocketPair_t socketpair (int domain, int type_, int protocol=0); void unlinkat(int dirfd, const char *pathname, int flags); void umount (const std::string& specialFile); void unsetenv(const char* name); pid_t waitpid (pid_t pid, int* status=nullptr, int options=0, bool cleanStatusOnSuccess=true); ssize_t write (int fd, const void *buf, size_t n, bool throwIfLessBytesWritten=true); ssize_t write (int fd, const std::string &buf, bool throwIfLessBytesWritten=true); ssize_t write (int fd, const QByteArray &buf, bool throwIfLessBytesWritten=true); std::vector queryGroupIds(); KernelVersion getKernelVersion(); } // namespace os template void os::chmod(const Str_t& path, mode_t mode){ if(::chmod(strDataAccess(path), mode) == -1){ throw ExcOs("chmod failed"); } } template void os::exec(const ContainerT &args, char * const envp[]) { if (args.isEmpty()) { throw std::invalid_argument( "exec called with empty args"); } os::exec(args[0], (char**)args.data(), envp); } template Str_t os::readlinkat (int dirfd, const Str_t & filename){ Str_t path; os::readlinkat(dirfd, filename, path); return path; } template void os::readlinkat (int dirfd, const Str_t & filename, Str_t & output){ output.resize(PATH_MAX); char* buf = strDataAccess(output); const char* filename_cstr = strDataAccess(filename); ssize_t path_len = ::readlinkat(dirfd, filename_cstr, buf, PATH_MAX); if (path_len == -1 ){ throw ExcReadLink("readlinkat failed for file " + std::string(filename_cstr)); } output.resize(static_cast(path_len)); } /// @throws ExcReadLink template Str_t os::readlink(const char* filename) { Str_t path; path.resize(PATH_MAX); char* buf = strDataAccess(path); ssize_t path_len = ::readlink(filename, buf, PATH_MAX); if(path_len == -1){ throw ExcReadLink("readlink failed for file " + std::string(filename)); } #if QT_VERSION >= QT_VERSION_CHECK(5, 7, 0) path.resize(static_cast(path_len)); #else // accept the Warning, at least in QT <=5.3 QByteArray has no size_type path.resize(path_len); #endif return path; } /// @throws ExcReadLink template Str_t os::readlink(const Str_t &filename) { return os::readlink(filename.data()); } /// @throws ExcOs template int os::open(const Str_t& filename, int flags, bool clo_exec, mode_t mode){ return os::open(strDataAccess(filename), flags, clo_exec, mode); } /// @throws ExcOs template int os::openat(int dirfd, const Str_t& filename, int flags, bool clo_exec, mode_t mode){ return __os::openat(dirfd, strDataAccess(filename), flags, clo_exec, mode); } /// @return the username of the *real* user template Str_t os::getUserName() { return ::getpwuid(getuid())->pw_name; } /// @throws ExcOs template Str_t os::readStr(int fd, size_t nbytes, bool retryOnInterrupt) { Str_t buf; buf.resize(nbytes); ssize_t readBytes = os::read(fd, buf.data(), buf.size(), retryOnInterrupt); buf.resize(static_cast(readBytes)); return buf; } /// @throws ExcOs template void os::remove(const Str_t & path){ if(::remove(strDataAccess(path)) == -1){ throw ExcOs("remove failed"); } } /// @throws ExcOs template void os::rename(const Str_t & old, const Str_t & new_){ if(::rename(strDataAccess(old), strDataAccess(new_)) == -1){ throw ExcOs("rename failed"); } } /// @throws ExcOs template void os::setenv(const Str_t &name, const Str_t &value, bool overwrite) { // in contrast to putenv, setenv makes copies of the passed // string, so we are safe, when value-string goes out of scope if(::setenv(strDataAccess(name), strDataAccess(value), int(overwrite)) == -1){ throw ExcOs("setenv failed"); } } template off_t os::sendfile(const Str_t& out_path, const Str_t& in_path, size_t count){ int out_fd=-1, in_fd=-1; try { out_fd = os::open(out_path, os::OPEN_WRONLY | os::OPEN_CREAT); in_fd = os::open(in_path, os::OPEN_RDONLY); auto ret = os::sendfile(out_fd, in_fd, count); close(out_fd); close(in_fd); return ret; } catch (const os::ExcOs&) { if(out_fd != -1) close(out_fd); if(in_fd != -1) close(in_fd); throw ; } } ================================================ FILE: src/common/oscpp/oscaps.cpp ================================================ #include #include #include #include "oscaps.h" #include "excos.h" static void cap_set_flag_wrapper(cap_t caps, cap_flag_t typeOfFlag, const os::Capabilites::CapFlags &flags, cap_flag_value_t setOrClear){ if (cap_set_flag(caps, typeOfFlag, int(flags.size()), flags.data(), setOrClear) == -1) { throw os::ExcOs(__func__); } } os::Capabilites::Capabilites(cap_t caps) : m_caps(caps) {} void os::Capabilites::flush() { // in future maybe something else but only flush to proc, which // should be set in constructor (enum) this->flushToProc(); } os::Capabilites::~Capabilites() { if(m_caps != nullptr){ if(::cap_free(m_caps) == -1){ perror(__func__); } } } void os::Capabilites::setFlags(cap_flag_t typeOfFlag, const CapFlags &flags, bool autoflush) { cap_set_flag_wrapper(m_caps, typeOfFlag, flags, CAP_SET); if(autoflush){ this->flush(); } } void os::Capabilites::clearFlags(cap_flag_t typeOfFlag, const CapFlags &flags, bool autoflush) { cap_set_flag_wrapper(m_caps, typeOfFlag, flags, CAP_CLEAR); if(autoflush){ this->flush(); } } void os::Capabilites::clear(bool autoflush) { if( cap_clear(m_caps) == -1){ throw os::ExcOs(__func__); } if(autoflush){ this->flush(); } } /// see also: cap_set_proc void os::Capabilites::flushToProc() { if (::cap_set_proc(m_caps) == -1){ throw os::ExcOs(__func__); } } /// see also cap_get_proc os::Capabilites::Ptr_t os::Capabilites::fromProc() { cap_t caps = ::cap_get_proc(); if(caps == nullptr){ throw os::ExcOs(__func__); } return Ptr_t(new os::Capabilites(caps)); } ================================================ FILE: src/common/oscpp/oscaps.h ================================================ #pragma once #include #include #include #include "util.h" namespace os { /// Simple wrapper around libcap /// When leaving autoflush to the default value of true, the set flags are applied /// immediately to the process (throws ExsOs on error). class Capabilites{ public: typedef std::shared_ptr Ptr_t; typedef std::vector CapFlags; ~Capabilites(); void setFlags(cap_flag_t typeOfFlag ,const CapFlags& flags, bool autoflush=true); void clearFlags(cap_flag_t typeOfFlag ,const CapFlags& flags, bool autoflush=true); void clear(bool autoflush=true); void flushToProc(); static Ptr_t fromProc(); public: Q_DISABLE_COPY(Capabilites) DEFAULT_MOVE(Capabilites) private: Capabilites(cap_t caps); void flush(); cap_t m_caps; }; } ================================================ FILE: src/common/oscpp/osutil.cpp ================================================ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "osutil.h" #include "os.h" #include "pidcontrol.h" #include "qoutstream.h" #include "fdentries.h" #ifdef __cplusplus extern "C" { #endif void intertSighandler(int){} #ifdef __cplusplus } #endif int osutil::countOpenFds() { int count = 0; for(const int fd : osutil::FdEntries()){ Q_UNUSED(fd) count++; } return count; } rlim_t osutil::getMaxCountOpenFiles() { struct rlimit rlim; getrlimit(RLIMIT_NOFILE, &rlim); return rlim.rlim_cur; } /// @return true, if fd existed within this process bool osutil::fdIsOpen(int fd) { const std::string fdpath = "/proc/self/fd/" + std::to_string(fd); return os::exists(fdpath); } /// @return true, if st1 and st2 refer to the same file (device/inode) bool osutil::sameFile(const os::stat_t& st1, const os::stat_t& st2) { return st1.st_dev == st2.st_dev && st1.st_ino == st2.st_ino; } /// Get the file access mode, file status flags and *some* 'file creation flags' using /// /proc/$pid/fdinfo/$fd. /// The returned flags include O_CLOEXEC. /// @param fdInfoDir: an open directory descritor pointing to an fdinfo-dir. /// @param fdNb: the file descritor /// See also: man 5 proc int osutil::retrieveFdFlags(int fdInfoDir, const std::string& fdNb) { // Note that fcntl(fd, F_GETFL) does *not* return O_CLOEXEC (and possibly others?). // That flag would have to be obtained *indirectly* by // fcntl(fd , F_GETFD), which, as of March 2019, only has the FD_CLOEXEC-flag // (which has a different value than O_CLOEXEC). std::string octalFlags = parseGenericKeyValFile(fdInfoDir, fdNb, "flags:"); return std::stoi( octalFlags, nullptr, 8 ); } /// Get the file access mode, file status flags and *some* 'file creation flags' using fcntl. /// The returned flags include O_CLOEXEC (if set). int osutil::retrieveFdFlags(int fd) { // Note that fcntl(fd, F_GETFL) does *not* return O_CLOEXEC. // That flag can be obtained *indirectly* by // fcntl(fd , F_GETFD), which, as of March 2019, only has the FD_CLOEXEC-flag // (which has a different value than O_CLOEXEC). int statusFlags = os::getFdStatusFlags(fd); int descrFlags = os::getFdDescriptorFlags(fd); if(IsBitSet(descrFlags, FD_CLOEXEC)){ setBitIn(statusFlags, O_CLOEXEC); } return statusFlags; } /// Reopen an open file decriptor of *this* process by resolving the symlink /// /proc/self/fd/$fd points to and passing that path (string) to open(2). /// Make sure that the new file descriptor really refers to the /// *same* file (it might not be the same path though but instead /// another hardlink, which is ignored here). /// @return the new file descriptor /// @throws ExcOs, especially, if the reopened file has a different device- /// inode-combination. int osutil::reopenFdByPath(int oldFd, int openflags, bool clo_exec, bool restoreOffset) { const auto oldStat = os::fstat(oldFd); // Race condition in next line... const int newFd = os::open(osutil::findPathOfFd(oldFd), openflags, clo_exec); // Note: the following is *not* a race-free variant of above call, because after unsharing // the mount-namespace, such an open call results to an fd still belonging to the original // mnt_id. // const int newFd = os::open("/proc/self/fd/" + std::to_string(oldFd), openflags, clo_exec); auto closeOnErr = finally([&newFd] { close(newFd); }); const auto newStat = os::fstat(newFd); if(! osutil::sameFile(oldStat, newStat)){ throw os::ExcOs("reopen failed, the new path refers to a " "different file", 0); } if(restoreOffset){ os::lseek(newFd, os::ltell(oldFd), SEEK_SET); } closeOnErr.setEnabled(false); return newFd; } std::string osutil::parseGenericKeyValFile(int dirFd, const std::string &filename, const std::string &key) { int fd = os::openat(dirFd, filename.c_str(), O_RDONLY); // closes fd in destrcutor __gnu_cxx::stdio_filebuf filebuf(fd, std::ios::in); std::istream is(&filebuf); std::string line; while(getline(is, line)){ std::string currentKey; std::stringstream wordStream(line); if( !(wordStream >> currentKey)){ continue; } if(currentKey != key){ continue; } std::string val; if( wordStream >> val){ return val; } break; } return std::string(); } std::string osutil::fcntlflagsToString(int flags) { std::string o; if (flags & O_WRONLY){ o += "O_WRONLY "; } else if (flags & O_RDWR){ o += "O_RDWR "; } else { o += "O_RDONLY "; } if (flags & O_CREAT) o += "O_CREAT "; if (flags & O_CLOEXEC) o += "O_CLOEXEC "; if (flags & O_DIRECTORY) o += "O_DIRECTORY "; if (flags & O_EXCL) o += "O_EXCL "; if (flags & O_NOCTTY) o += "O_NOCTTY "; if (flags & O_NOFOLLOW) o += "O_NOFOLLOW "; #ifdef O_TMPFILE if (flags & O_TMPFILE) o += "O_TMPFILE "; #endif if (flags & O_APPEND) o += "O_APPEND "; if (flags & O_ASYNC) o += "O_ASYNC "; if (flags & O_DIRECT) o += "O_DIRECT "; if (flags & O_DSYNC) o += "O_DSYNC "; if (flags & O_LARGEFILE) o += "O_LARGEFILE "; if (flags & O_NOATIME) o += "O_NOATIME "; if (flags & O_NONBLOCK) o += "O_NONBLOCK "; if (flags & O_PATH) o += "O_PATH "; if (flags & O_DSYNC) o += "O_DSYNC "; if (flags & O_SYNC) o += "O_SYNC "; if (flags & O_TRUNC) o += "O_TRUNC "; return o; } /// Merely a debug function void osutil::printOpenFds(bool onlyRegular) { QIErr() << "open fds:\n"; for(const int fd : osutil::FdEntries()){ auto st = os::fstat(fd); if( onlyRegular && ! S_ISREG(st.st_mode)){ continue; } QByteArray fdPath = QByteArray("/proc/self/fd/") + QByteArray::number(fd); auto resolvedPath = os::readlink(fdPath); QIErr() << fd << ": " << resolvedPath; } } void osutil::randomSleep(int msMin, int msMax) { std::mt19937_64 eng{std::random_device{}()}; std::uniform_int_distribution<> dist{msMin, msMax}; std::this_thread::sleep_for(std::chrono::milliseconds{dist(eng)}); } /// For most efficient usage assign a bufsize a little larger than the (probable) /// file size. QByteArray osutil::readWholeFile(int fd, int bufSize) { assert(bufSize > 0); QByteArray buf; buf.resize(bufSize); int offset=0; while(true){ char* dataPtr = buf.data() + offset; auto readCount = os::read(fd, dataPtr, static_cast(bufSize), true); if(readCount < bufSize){ // EOF buf.resize(offset + static_cast(readCount)); return buf; } offset += bufSize; buf.resize(buf.size() + bufSize); } } /// @param fd: typically STDOUT_FILENO or similar bool osutil::isTTYForegoundProcess(int fd) { return getpgrp() == tcgetpgrp(fd); } /// Wait until a typical 'TERM'-signal occurs. During wait the signal handlers /// are overridden and restored afterwards. void osutil::waitForSignals() { QVarLengthArray oldHandlers; // wait for typical signals to exit for(int s : os::catchableTermSignals()){ oldHandlers.push_back(os::signal(s, [](int){})); } sigset_t sigs; os::sigfillset(&sigs); os::sigwait(&sigs); for (int i=0; i < oldHandlers.size(); ++i) { os::signal(os::catchableTermSignals()[static_cast(i)], oldHandlers[i]); } } /// Set a signal handler doing nothing for the specified signales. /// Note that this is *not* equivalent to SIG_IGN: /// SIG_IGN is inherited on execve, our signal handler is not. void osutil::setInertSighandler(const std::vector &sigs) { struct sigaction act{}; act.sa_handler = intertSighandler; sigemptyset (&act.sa_mask); act.sa_flags = SA_RESTART; for(auto s : sigs){ os::sigaction(s, &act, nullptr); } } /// be verbose in case os::close fails void osutil::closeVerbose(int fd) { try { os::close(fd); } catch (const os::ExcOs& e) { std::cerr << e.what() << generate_trace_string() << "\n"; } } /// Filedescriptors are usually given out using low integers, this function allows /// for finding the highest fd starting at startFd. If startFd==-1, return the /// the highest possible free fd (per-process max.-fd-count is e.g. 1024). /// @return: A fd-number if a free fd could be found. The fd is not opened, so there is /// a race-condition here. If no fd in the given range could be found, return -1. int osutil::findHighestFreeFd(int startFd, int minFd){ int fd = (startFd == -1) ? static_cast(osutil::getMaxCountOpenFiles() -1) : startFd; for(; fd >= minFd; --fd) { if(fd == 255 ){ // that one is usually reserved continue; } if(! osutil::fdIsOpen(fd)){ return fd; } } return -1; } /// @param path: if empty, it will be filled with tmp.xxx at system's tempdir int osutil::mktmp(QByteArray &path, int flags){ if(path.isEmpty()){ path = QDir::tempPath().toUtf8(); path = pathJoinFilename(path, QByteArray("tmp.XXXXXX")); } int fd = ::mkostemp(path.data(), flags); if (fd < 0) { throw os::ExcOs("osutil::mktmp failed"); } return fd; } int osutil::mktmp(int flags){ QByteArray p(QDir::tempPath().toUtf8()); p = pathJoinFilename(p, QByteArray("tmp.XXXXXX")); return osutil::mktmp(p, flags); } int osutil::unnamed_tmp(int flags){ // tmpfs and possibly other filesystems do not suppot O_TMPFILE, for // the sake of simplicity just use mkostemp. #if false //#ifdef O_TMPFILE int fd = os::open(p, O_RDWR | O_TMPFILE | O_EXCL | o_flags, true, S_IRUSR | S_IWUSR); #else QByteArray p; int fd = osutil::mktmp(p, flags); if(remove(p) < 0){ fprintf(stderr, "%s: failed to delete the just created file %s - %s\n", __func__, p.constData(), strerror(errno)); } #endif return fd; } ================================================ FILE: src/common/oscpp/osutil.h ================================================ #pragma once #include #include #include #include #include #include #include #include "os.h" #include "cleanupresource.h" #include "util.h" namespace osutil { int countOpenFds(); void closeVerbose(int fd); std::string fcntlflagsToString(int flags); bool fdIsOpen(int fd); /// Shells usually start at low numbers for internal file descriptors (usually 10), /// we try to find the highest possible free fd /// If startFd != -1, start searching from that. int findHighestFreeFd(int startFd=-1, int minFd=11); template Str_t findPathOfFd(int fd); rlim_t getMaxCountOpenFiles(); bool isTTYForegoundProcess(int fd); std::string parseGenericKeyValFile(int dirFd, const std::string &filename, const std::string &key); void printOpenFds(bool onlyRegular=false); void randomSleep(int msMin, int msMax); QByteArray readWholeFile(int fd, int bufSize); int retrieveFdFlags(int fd); int retrieveFdFlags(int fdInfoDir, const std::string &fdNb); int reopenFdByPath(int oldFd, int openflags, bool clo_exec=true, bool restoreOffset=true); bool sameFile(const os::stat_t& st1, const os::stat_t& st2); int mktmp(QByteArray& path, int flags=os::OPEN_CLOEXEC); int mktmp(int flags=os::OPEN_CLOEXEC); int unnamed_tmp(int flags=os::OPEN_CLOEXEC); void waitForSignals(); void setInertSighandler(const std::vector& sigs); } // namespace fdcontrol /// Find path where an open fd of OUR process (currently) points to. /// @return path /// @throws ExcReadLink template Str_t osutil::findPathOfFd(int fd){ char procfdPath[PATH_MAX]; snprintf(procfdPath, sizeof(procfdPath), "/proc/self/fd/%d", fd); Str_t path = os::readlink(procfdPath); return path; } ================================================ FILE: src/common/pathtree.cpp ================================================ #include #include #include "pathtree.h" #include "util.h" #include "logger.h" /// Write the next available filename from the null-terminated path to out. /// Warning: must not contain leading '/'. /// @return the current ptr, if out was written, else nullptr (end of string) static const char* nextFilename(const char* path, StrLight& out){ assert(*path != '/'); const char* begin = path; for(; *path != '\0'; ++path){ if(*path == '/'){ // we are done: out.setRawData(begin, path - begin); // slash '/' not of interest: go on. ++path; return path; } } if(begin == path){ // may happen if '/' is exanimed return nullptr; } out.setRawData(begin, path - begin); assert(*path == '\0'); return path; } /// Construct a new dir iterator pointing at a directory entries at path. PathTree::iterator::iterator(_DirMap::const_iterator begin, _DirMap::const_iterator end, const StrLight& path) : d(std::make_shared()) { if(begin == end){ return; } d->currentPath = path.deepCopy(); appendPath(begin.key()); d->dirStack.push_back( { begin, end, begin.key().size() }); if(! begin.value()->isEnd){ // move to first *really* inserted path ++(*this); } } PathTree::iterator::iterator() = default; bool PathTree::iterator::operator==(const PathTree::iterator &rhs) const { bool ourDirsEmpty = (d == nullptr) ? true : d->dirStack.empty(); bool otherDirsEmpty = (rhs.d == nullptr) ? true : rhs.d->dirStack.empty(); if(ourDirsEmpty || otherDirsEmpty){ // cannot compare dirStack return ourDirsEmpty == otherDirsEmpty; } return d->currentPath == rhs.d->currentPath; } bool PathTree::iterator::operator!=(const PathTree::iterator &rhs) const { return ! (*this == rhs); } /// Iterate exactly over the inserted directories skipping /// possible other paths, e.g. if /home/user/foo is set, /// /home and /home/user will be skipped. PathTree::iterator &PathTree::iterator::operator++() { assert(! d->dirStack.empty()); while (true) { nextDir(); if(d->dirStack.empty()){ return *this; } auto & currentDir = d->dirStack.back(); if(currentDir.it != currentDir.end && currentDir.it.value()->isEnd){ return *this; } } } /// go to next dir, prefering going as deep as possible /// first, then to the sibling directories and finally /// walk up the tree again, jumping over already visited dirs. void PathTree::iterator::nextDir() { assert(! d->dirStack.empty()); auto & dirInfo = d->dirStack.back(); // go into depth first, if possible auto & subDirs = dirInfo.it.value()->children; if(cdSubDirIfExist(subDirs.begin(), subDirs.end())){ return; } if(nextSiblingIfExist(dirInfo)){ return; } nextEntryInParentDirs(); } bool PathTree::iterator::cdSubDirIfExist(_DirMap::iterator begin, _DirMap::iterator end) { if(begin == end){ return false; } appendPath(begin.key()); d->dirStack.push_back( { begin, end, begin.key().size() }); return true; } /// Go up as many parent directories necessary until the next valid /// entry is found. The dirstack will be empty, in case we're done void PathTree::iterator::nextEntryInParentDirs() { while (true) { auto & currentDirInfo = d->dirStack.back(); stripPath(currentDirInfo.sizeDirName); d->dirStack.pop_back(); if(d->dirStack.empty()){ // we are done -> empty stack == iterator.end() return; } auto & upperDir = d->dirStack.back(); if(nextSiblingIfExist(upperDir)){ return; } // got this dir as well -> go up even more } } /// increment the passed dir and adjust the path appropriately if we could switch /// to the next sibling (same directory level, no parent- or subdir) bool PathTree::iterator::nextSiblingIfExist(PathTree::iterator::CurrentDirInfo &dirInfo) { ++dirInfo.it; if(dirInfo.it == dirInfo.end){ return false; } stripPath(dirInfo.sizeDirName); appendPath(dirInfo.it.key()); dirInfo.sizeDirName = dirInfo.it.key().size(); return true; } void PathTree::iterator::appendPath(const StrLight &dirname) { if( d->currentPath.empty() || d->currentPath.back() != '/'){ d->currentPath += '/'; } d->currentPath += dirname; } void PathTree::iterator::stripPath(size_t lastDirSize) { assert(lastDirSize <= d->currentPath.size()); d->currentPath.resize(d->currentPath.size() - lastDirSize); if(! d->currentPath.empty() && d->currentPath.back() == '/'){ d->currentPath.pop_back(); } } const PathTree::iterator PathTree::begin() const { return iterator(m_rootDirMapDummy.begin(), m_rootDirMapDummy.end(), "/"); } const PathTree::iterator PathTree::end() const { return iterator(); } /// @return an iterator pointing on the directory-node corresponding to path. /// Subsequentially incrementing it results in an iteration of all sub-paths /// as well as path, if it exists. Note: path may also be an intermediate /// directory (dir->isEnd == false) PathTree::iterator PathTree::iter(const StrLight &path) const { auto dir =findDir(path); if(dir == nullptr ){ return end(); } _DirMap::const_iterator itOfChildInParent; _DirMap::const_iterator dummyEnd; if(dir == m_rootDir){ itOfChildInParent = m_rootDirMapDummy.begin(); dummyEnd = m_rootDirMapDummy.end(); } else { // need to determine the iterators from parent. auto parentDirWeak = dir->parent; assert(! is_uninitialized(parentDirWeak)); auto parentDir = parentDirWeak.lock(); itOfChildInParent = parentDir->children.find(dir->name); assert(itOfChildInParent != parentDir->children.end()); // Do not iterate over possible siblings as well: set next one as end() // (even if it is no real end) dummyEnd = itOfChildInParent; ++dummyEnd; } return iterator(itOfChildInParent, dummyEnd, splitAbsPath(path).first); } /// @return: An iterator for all subpaths of param path (so path is *not* /// traversed). PathTree::iterator PathTree::subpathIter(const StrLight &path) const { auto dir =findDir(path); if(dir == nullptr ){ return end(); } return iterator(dir->children.begin(), dir->children.end(), path); } PathTree::iterator PathTree::erase(PathTree::iterator it) { assert(! it.d->dirStack.empty()); assert( it.d->dirStack.back().it != it.d->dirStack.back().end); auto dir = it.d->dirStack.back().it.value(); assert(dir->isEnd); dir->isEnd = false; // before possibly deleting empty in-between paths (isEnd=false) // move to next 'real dir' -> otherwise iterators might have been invalidated. ++it; // go up the current tree and erase all empty dirs (stop on first non-empty) while (true) { if(! dir->children.empty()){ // our dir has children, so do not erase it! return it; } // our dir has no children, so it is safe for our parent to delete it auto parentDirWeak = dir->parent; if(is_uninitialized(parentDirWeak)){ // reached root / return it; } auto parentDir = dir->parent.lock(); auto itOfDirInParent = parentDir->children.find(dir->name); assert(itOfDirInParent != parentDir->children.end()); parentDir->children.erase(itOfDirInParent); dir = parentDir; } } ////////////////////////////////////////////////////////////////////////////////////////////// PathTree::PathTree() { commonConstructor(); } void PathTree::commonConstructor() { m_rootDir = std::make_shared<_Dir>("/"); m_rootDirMapDummy = {{"", m_rootDir}}; m_rootNodeIsContained = false; } const std::unordered_set &PathTree::allPaths() const { return m_allPaths; } void PathTree::printDbg() { if(m_rootDir->children.empty()){ std::cerr << __func__ << " tree is empty\n"; } else { printRec(m_rootDir); } } void PathTree::clear() { m_rootDir->children.clear(); m_rootDir->isEnd = false; m_allPaths.clear(); m_orderedPathlenghts.clear(); } bool PathTree::isEmpty() const { return m_rootDir->children.empty(); } void PathTree::insert(const StrLight &path){ assert( path.find("//") == StrLight::npos); auto currenDir = m_rootDir; const char* cpath = path.c_str(); // ignore leading / ++cpath; StrLight filename; while ( (cpath = nextFilename(cpath, filename)) != nullptr ) { currenDir = mkDirIfNotExist(currenDir, filename); } currenDir->isEnd = true; m_allPaths.insert(path.deepCopy()); if(std::find(m_orderedPathlenghts.begin(), m_orderedPathlenghts.end(), path.size()) == m_orderedPathlenghts.end()){ m_orderedPathlenghts.push_back(path.size()); std::sort( m_orderedPathlenghts.begin(), m_orderedPathlenghts.end()); } } bool PathTree::contains(const StrLight &path) const { StrLight pathLight; pathLight.setRawData(path.c_str(), path.size()); auto dir = findDir(pathLight); if(dir == nullptr){ return false; } return dir->isEnd; } /// Check if path is a parent path of any other path within this /// tree. Example: /// /home/user/foo exists in this tree and it is queried for path /// /home/user -> true is returned /// /// If allowEquals is true, true is also returned, if /// the searched path is contained but has no children (equals /// to the searched path). bool PathTree::isParentPath(const StrLight &path, bool allowEquals) const { StrLight pathLight; pathLight.setRawData(path.c_str(), path.size()); auto dir = findDir(pathLight); if(dir == nullptr){ return false; } if(! dir->children.empty()){ return true; } // no children exist return dir->isEnd && allowEquals; } /// @return true, if param path is subpath of any previously inserted paths /// or the same, if allowEquals=true bool PathTree::isSubPath(const StrLight &path, bool allowEquals) const { // maybe_todo: continously calculate the hash (not always from beginning). if(! m_orderedPathlenghts.empty() && m_orderedPathlenghts.front() == 1){ // We contain the root node. Any path is a subpath execpt / assert(m_allPaths.find("/") != m_allPaths.end()); if(allowEquals){ return true; } return path != '/'; } m_rawbuftmp.setRawData(path.constData(), path.size()); for(size_t s : m_orderedPathlenghts){ if(s < path.size()){ // If we didn't have a / at the next position, we would cut the // path at a wrong position -> continue if(path[s] != '/'){ continue; } // A candiate path with the same size exists. No need to check // allowEquals, because the path continues m_rawbuftmp.setRawSize(s); if(m_allPaths.find(m_rawbuftmp) != m_allPaths.end()){ return true; } } else if( s > path.size()){ // m_orderedPathlenghts is ordered ascending -> the // next paths will be even longer: return false; } else { // s == path.size // The next m_orderedPathlength will be greater, so we can only // be a 'sub'-path, if allowEquals is true. if( allowEquals){ m_rawbuftmp.setRawSize(s); if(m_allPaths.find(m_rawbuftmp) != m_allPaths.end()){ return true; } } return false; } } return false; } void PathTree::printRec(const PathTree::_DirPtr &node, const StrLight &dir) const { for(const auto & n : node->children){ auto fullPath = dir + '/' + n->name; printf("%s %.*s\n", __func__, int(fullPath.size()), fullPath.constData()); printRec(n, fullPath); } } /// @return the new or existing dir PathTree::_DirPtr PathTree::mkDirIfNotExist(PathTree::_DirPtr &parent, const StrLight &name) { auto it = parent->children.find(name); if(it == parent->children.end()){ auto name_copy = name.deepCopy(); auto newDir = std::make_shared<_Dir>(name_copy); newDir->parent = parent; parent->children[name_copy] = newDir; return newDir; } return it.value(); } /// @return The node exactly matching the passed path or nullptr PathTree::_DirPtr PathTree::findDir(const StrLight &path) const { auto currentDir = m_rootDir; const char* cpath = path.constData(); // ignore leading / ++cpath; StrLight filename; while ( (cpath = nextFilename(cpath, filename)) != nullptr ) { auto it = currentDir->children.find(filename); if(it == currentDir->children.end()){ return nullptr; } currentDir = it.value(); } return currentDir; } /* void PathTree::recursiveCopy(_DirPtr& dst, const _DirPtr& src) { dst->isEnd = src->isEnd; dst->name = src->name; dst->children.reserve(src->children.size()); for(auto& subSrc : src->children){ auto newDir = std::make_shared<_Dir>(); dst->children[subSrc.first] = newDir; newDir->parent = dst; recursiveCopy(newDir, subSrc.second); } } */ void PathTree::recursiveClear(PathTree::_DirPtr &dir) { for(auto& sub : dir->children){ recursiveClear(sub); } dir->children.clear(); } ================================================ FILE: src/common/pathtree.h ================================================ #pragma once #include #include #include #include #include "strlight.h" /// Add a set of absolute file paths and /// later check, if a given path is a sub- /// or parent path of one of those. /// No filesystem-activity involved! /// Please make sure the paths are clean beforehand /// ( no //, no traling /, no relative paths ../../ etc.) class PathTree { private: struct _Dir; typedef std::shared_ptr<_Dir> _DirPtr; typedef QHash _DirMap; struct _Dir { _Dir(const StrLight& name) : isEnd(false), name(name){} _DirMap children; std::weak_ptr<_Dir> parent; // break reference cycles ! bool isEnd; StrLight name; }; public: class iterator { public: bool operator==(const iterator& rhs) const; bool operator!=(const iterator& rhs) const; iterator& operator++ (); StrLight & operator*() { return d->currentPath; } private: struct CurrentDirInfo { _DirMap::const_iterator it; // current position _DirMap::const_iterator end; size_t sizeDirName; // putting brace {} here makes compilation fail. Why? }; typedef std::vector DirStack; iterator(_DirMap::const_iterator begin, _DirMap::const_iterator end, const StrLight &path); iterator(); void nextDir(); bool cdSubDirIfExist(_DirMap::iterator begin, _DirMap::iterator end); void nextEntryInParentDirs(); bool nextSiblingIfExist(CurrentDirInfo& dirInfo); void appendPath(const StrLight& dirname); void stripPath(size_t lastDirSize); struct PrivateData{ DirStack dirStack; // last element always points to the current // dir of iteration StrLight currentPath; }; std::shared_ptr d; friend class PathTree; }; public: const iterator begin() const; const iterator end() const ; iterator iter(const StrLight& path) const; iterator subpathIter(const StrLight& path) const; iterator erase(iterator it); public: PathTree(); ~PathTree() = default; PathTree(const PathTree&) = delete ; PathTree& operator=( const PathTree& ) = delete ; void clear(); bool isEmpty() const; void insert(const StrLight& path); template void insert(Iterator first, Iterator last); bool contains(const StrLight & path) const; bool isParentPath(const StrLight &path, bool allowEquals=false) const ; bool isSubPath(const StrLight &path, bool allowEquals=false) const; void printDbg(); const std::unordered_set& allPaths() const; private: static const char sep = '/'; void commonConstructor(); _DirPtr m_rootDir; _DirMap m_rootDirMapDummy; mutable StrLight m_rawbuftmp; std::unordered_set m_allPaths; std::vector m_orderedPathlenghts; bool m_rootNodeIsContained; void printRec(const _DirPtr &node, const StrLight &dir="") const; _DirPtr mkDirIfNotExist(_DirPtr& parent, const StrLight &name); _DirPtr findDir(const StrLight &path) const; // static void recursiveCopy(_DirPtr &dst, const _DirPtr &src); static void recursiveClear(_DirPtr &dir); }; template void PathTree::insert(Iterator first, Iterator last) { for(Iterator it=first; it != last; ++it) { this->insert(*it); } } ================================================ FILE: src/common/pidcontrol.cpp ================================================ #include #include #include #include #include #include #include #include #include "pidcontrol.h" #include "logger.h" #include "os.h" #include "osutil.h" /// returns an empty string, if opening fails /// (or the file belonging to pid was empty) std::string pidcontrol::parseCmdlineOfPID(pid_t pid) { std::string cmdline; const std::string pathToPid = "/proc/" + std::to_string(pid) + "/cmdline"; std::ifstream f; f.open(pathToPid, std::fstream::in); if(f.is_open() ) { // recombine the string-set to one string. From man proc: // The command-line arguments appear // in this file as a set of strings separated by null bytes // ('\0'), with a further null byte after the last string. char ch; cmdline.reserve(128); bool previousChWasBSlash0 = false; while (f >> std::noskipws >> ch) { if(ch == '\0'){ if(previousChWasBSlash0){ break; } cmdline.push_back(' '); previousChWasBSlash0 = true; } else { cmdline.push_back(ch); previousChWasBSlash0 = false; } } if(! cmdline.empty()){ cmdline.pop_back(); } } return cmdline; } /// Read the status file at /proc/$PID/status and return /// the real user id found in it (but check for null). /// See also man 5 proc. /// @param procDirFd: *must* be an open directory descriptor /// at /proc/$pid NullableValue pidcontrol::parseRealUidOf(int procDirFd){ std::string uid = osutil::parseGenericKeyValFile(procDirFd, "status", "Uid:"); if(uid.empty()){ return {}; } return {qVariantTo_throw(QByteArray::fromStdString(uid))}; } ================================================ FILE: src/common/pidcontrol.h ================================================ #pragma once #include "nullable_value.h" #include namespace pidcontrol { std::string parseCmdlineOfPID(pid_t pid); NullableValue parseRealUidOf(int procDirFd); } // namespace pidcontrol ================================================ FILE: src/common/qfddummydevice.cpp ================================================ #include #include "qfddummydevice.h" #include "os.h" /// @param becomeOwner: if true, close the fd in destructor QFdDummyDevice::QFdDummyDevice(int fd, bool becomeOwner) : m_fd(fd), m_owner(becomeOwner) {} QFdDummyDevice::~QFdDummyDevice() { if(m_owner){ try { os::close(m_fd); } catch (const os::ExcOs& e) { std::cerr << __func__ << " " << e.what() << "\n"; } } } qint64 QFdDummyDevice::readData(char *data, qint64 maxlen) { return os::read(m_fd, data,static_cast(maxlen)); } qint64 QFdDummyDevice::writeData(const char *data, qint64 len) { return os::write(m_fd, data, static_cast(len)); } ================================================ FILE: src/common/qfddummydevice.h ================================================ #pragma once #include /// Dummy wrapper because QFile::open(int fd,...) cannot handle an already /// open fd... class QFdDummyDevice : public QIODevice { public: QFdDummyDevice(int fd, bool becomeOwner=false); ~QFdDummyDevice() override; public: QFdDummyDevice(const QFdDummyDevice&) = delete; void operator=(const QFdDummyDevice&) = delete; protected: int m_fd; bool m_owner; qint64 readData(char *data, qint64 maxlen) override; qint64 writeData(const char *data, qint64 len) override; }; ================================================ FILE: src/common/qfilethrow.cpp ================================================ #include "qfilethrow.h" #include "util.h" /// @throws QExcIo void QFileThrow::flush() { if(! QFile::flush()){ throw QExcIo(qtr("Failed to flush %1: %2") .arg(this->fileName(), this->errorString())); } } bool QFileThrow::open(QIODevice::OpenMode flags) { if(! QFile::open(flags)){ throw QExcIo(qtr("Failed to open %1: %2") .arg(this->fileName(), this->errorString())); } return true; } bool QFileThrow::open(FILE *f, QIODevice::OpenMode ioFlags, QFileDevice::FileHandleFlags handleFlags) { if(! QFile::open(f, ioFlags, handleFlags)){ throw QExcIo(qtr("Failed to open file: %1") .arg(this->errorString())); } return true; } bool QFileThrow::open(int fd, QIODevice::OpenMode ioFlags, QFileDevice::FileHandleFlags handleFlags) { if(! QFile::open(fd, ioFlags, handleFlags)){ throw QExcIo(qtr("Failed to open fd %1: %2") .arg(fd).arg(this->errorString())); } return true; } /// @throws QExcIo /// @return: *always* true, only bool because of 'override' bool QFileThrow::seek(qint64 offset) { if(! QFile::seek(offset)){ throw QExcIo(qtr("Failed to seek %1: %2") .arg(this->fileName(), this->errorString())); } return true; } qint64 QFileThrow::readData(char *data, qint64 maxSize){ auto ret = QFile::readData(data, maxSize); if(ret == -1){ throw QExcIo(qtr("Failed to read from file %1: %2") .arg(this->fileName(), this->errorString())); } return ret; } qint64 QFileThrow::readLineData(char *data, qint64 maxlen){ auto ret = QFile::readLineData(data, maxlen); if(ret == -1){ throw QExcIo(qtr("Failed to readLine from file %1: %2") .arg(this->fileName(), this->errorString())); } return ret; } /// @throws QExcIo qint64 QFileThrow::writeData(const char *data, qint64 len) { auto bytesWritten = QFile::writeData(data, len); if(bytesWritten == -1){ throw QExcIo(qtr("Failed to write to file %1: %2") .arg(this->fileName(), this->errorString())); } if( bytesWritten != len){ throw QExcIo(qtr("Unexpected written size for file %1 - " "expected %2, actual: %3") .arg(this->fileName()).arg(len).arg(bytesWritten)); } return bytesWritten; } ================================================ FILE: src/common/qfilethrow.h ================================================ #pragma once #include class QFileThrow : public QFile { public: using QFile::QFile; void flush(); bool open(QFile::OpenMode flags) override; bool open(FILE *f, OpenMode ioFlags, FileHandleFlags handleFlags=DontCloseHandle); bool open(int fd, OpenMode ioFlags, FileHandleFlags handleFlags=DontCloseHandle); bool seek(qint64 offset) override; qint64 readData(char *data, qint64 maxlen) override; qint64 readLineData(char *data, qint64 maxlen) override; qint64 writeData(const char *data, qint64 len) override; }; ================================================ FILE: src/common/qoptargparse/CMakeLists.txt ================================================ add_library(lib_qoptargparse excoptargparse.cpp qoptargparse.cpp qoptarg.cpp qoptsqlarg.cpp qoptvarlenarg.cpp qoptargtrigger.cpp ) target_link_libraries(lib_qoptargparse PUBLIC Qt5::Core lib_util ) ================================================ FILE: src/common/qoptargparse/excoptargparse.cpp ================================================ #include "excoptargparse.h" ExcOptArgParse::ExcOptArgParse(const QString &text) : QExcCommon(text, false) { } ================================================ FILE: src/common/qoptargparse/excoptargparse.h ================================================ #pragma once #include "exccommon.h" class ExcOptArgParse : public QExcCommon { public: ExcOptArgParse(const QString & text); }; ================================================ FILE: src/common/qoptargparse/qoptarg.cpp ================================================ #include #include #include "qoptarg.h" #include "compat.h" #include "exccommon.h" #include "excoptargparse.h" #include "util.h" #include "conversions.h" /// @param shortName short name, one minus is added to the front (-e) /// @param name long name, two minus signs are added to the front (--exec) /// @param description /// @param hasValue --verbose might be a flag, --size 2 has the value 2. /// QOptArg::QOptArg(const QString &shortName, const QString &name, QString description, bool hasValue) : m_name("--" + name), m_description(std::move(description)), m_hasValue(hasValue), m_argIdx(-1), m_internalOnly(false), m_isFinalizeFlag(false), m_isByteSizeArg(false), m_isRelativeDateTime(false), m_relativeDateTimeSubtract(false) { if(name.isEmpty()){ throw QExcIllegalArgument("argname must not be empty"); } if(name.startsWith('-')){ throw QExcProgramming("please pass names without leading minus"); } if(! shortName.isEmpty()){ if(shortName.startsWith('-')){ throw QExcProgramming("please pass short names without leading minus"); } m_shortName = '-' + shortName; } } /// @param optTrigger /// @param defaultTriggerStr Trigger string which shall be used, /// in case no trigger is entered (by the user) /// QOptArg::QOptArg(const QString &shortName, const QString &name, const QString &description, const QOptArgTrigger &optTrigger, const QString& defaultTriggerStr) : QOptArg(shortName, name, description, true) { m_optTrigger = optTrigger; m_defaultTriggerStr = defaultTriggerStr; } const QString &QOptArg::shortName() const { return m_shortName; } const QString& QOptArg::name() const { return m_name; } QString QOptArg::description() const { return m_description ; } bool QOptArg::hasValue() const { return m_hasValue; } bool QOptArg::wasParsed() const { return m_argIdx != -1; } const QOptArgTrigger &QOptArg::optTrigger() const { return m_optTrigger; } /// Meant to be overidden by subclasses. /// Called right before a potential trigger word is further /// processed. QString QOptArg::preprocessTrigger(const char *str) const { return str; } const QString &QOptArg::parsedTrigger() const { return m_parsedTrigger; } void QOptArg::setParsedTrigger(const QString &parsedTrigger) { m_parsedTrigger = parsedTrigger; } /// See also: setAllowedOptions() /// @param maxCount: throw, in case more options than maxCount were parsed. QStringList QOptArg::getOptions(int maxCount) const { if(m_allowedOptions.empty()){ throw QExcProgramming(QString("%1 called without previous setAllowedOptions") .arg(__func__)); } if(! m_hasValue){ throwgetValueCalledOnFlag(__func__); } QStringList valList; for(int i=0; i < m_vals.len; i++){ QStringList newVals = QString(m_vals.argv[i]) .split(m_allowedOptionsDelimeter, Qt::SkipEmptyParts); for(const QString& str : newVals){ if(m_allowedOptions.find(str) == m_allowedOptions.end()){ throw ExcOptArgParse(qtr("'%1' is not a supported option for '%2'. ") .arg(str, m_name)); } } valList += newVals; if(valList.size() > maxCount){ throw ExcOptArgParse(qtr("Only %1 option(s) allowed for argument %2") .arg(maxCount).arg(m_name)); } } return valList; } /// Note: this argument must have been marked as 'bytesize' beforehand QVariantList QOptArg::getVariantByteSizes(const QVariantList &defaultValues) { assert(m_isByteSizeArg); auto sizeStrs = getVariantValues(defaultValues); QVariantList sizes; Conversions userStrConv; for(const auto& s : sizeStrs){ try { sizes.push_back(userStrConv.bytesFromHuman(s.toString())); } catch (const ExcConversion& e) { throw ExcOptArgParse(e.descrip() + " (arg " + m_name + ')' ); } } return sizes; } QVariantList QOptArg::getVariantRelativeDateTimes(const QVariantList &defaultValues) { assert(m_isRelativeDateTime); auto dateTimeStrs = getVariantValues(defaultValues); QVariantList dateTimes; Conversions userStrConv; for(const auto& s : dateTimeStrs){ try { dateTimes.push_back(userStrConv.relativeDateTimeFromHuman(s.toString(), m_relativeDateTimeSubtract)); } catch (const ExcConversion& e) { throw ExcOptArgParse(e.descrip() + " (arg " + m_name + ')' ); } } return dateTimes; } /// See also: getOptions(), where the check is performed lazily. void QOptArg::setAllowedOptions(const std::unordered_set &options, const QString &delimeter) { m_allowedOptions = options; if(delimeter.isEmpty()){ throw QExcIllegalArgument(QString("%1: empty delimeter passed.").arg(__func__)); } m_allowedOptionsDelimeter = delimeter; } const QOptArg::RawValues_t &QOptArg::vals() const { return m_vals; } void QOptArg::setVals(const RawValues_t &vals) { m_vals = vals; } int QOptArg::argIdx() const { return m_argIdx; } void QOptArg::setArgIdx(int argIdx) { m_argIdx = argIdx; } const QString &QOptArg::defaultTriggerStr() const { return m_defaultTriggerStr; } /// see setter bool QOptArg::internalOnly() const { return m_internalOnly; } /// An internal argument is not displayed in the help void QOptArg::setInternalOnly(bool internalOnly) { m_internalOnly = internalOnly; } /// If *this* argument is parsed, param arg must be parsed /// as well. void QOptArg::addRequiredArg(const QOptArg *arg) { assert(arg->name() != this->name()); m_requiredArs.append(arg); } const QVector& QOptArg::requiredArs() const { return m_requiredArs; } void QOptArg::throwgetValueCalledOnFlag(const char *functionname) const { throw QExcProgramming(QString("%1() was called although argument %2 " "was marked as flag (no value)").arg(functionname, m_name)); } /// @param subtractIt: if true, the parsed date is subtracted from current one, /// else it is added. void QOptArg::setIsRelativeDateTime(bool isRelativeDateTime, bool subtractIt) { m_isRelativeDateTime = isRelativeDateTime; m_relativeDateTimeSubtract = subtractIt; m_description += qtr(" Supported units include %1") .arg(Conversions::relativeDateTimeUnitDescriptions()); } /// adds a description, that this argument also accepts bytesizes after given /// numbers lik KiB, MiB, etc. void QOptArg::setIsByteSizeArg(bool isByteSizeArg) { m_isByteSizeArg = isByteSizeArg; m_description += qtr(" You may provide a unit such as KiB, MiB, etc.."); } bool QOptArg::isFinalizeFlag() const { return m_isFinalizeFlag; } /// Currently only supported for flags (arguments without values). /// If true, the parser will stop processing args, if argument is passed. /// This can be used to delegate parsing control to a 'sub-parser'. /// Therefor, if finalize is true, an exception will be thrown, if no furhter /// arguments are available after the respective flag. /// Default is false. void QOptArg::setFinalizeFlag(bool f) { if(m_hasValue){ throw QExcProgramming("Finalize flag currently only supported for " "flags (arguments without value)"); } m_isFinalizeFlag = f; } const QString &QOptArg::allowedOptionsDelimeter() const { return m_allowedOptionsDelimeter; } const std::unordered_set& QOptArg::allowedOptions() const { return m_allowedOptions; } ================================================ FILE: src/common/qoptargparse/qoptarg.h ================================================ #pragma once #include #include #include #include "compat.h" #include "util.h" #include "qoptargtrigger.h" #include "excoptargparse.h" class QOptArg { public: struct RawValues_t{ RawValues_t() : argv(nullptr), len(0) {} char** argv; int len; }; QOptArg(const QString& shortName, const QString & name, QString description, bool hasValue=true ); QOptArg(const QString& shortName, const QString & name, const QString& description, const QOptArgTrigger & optTrigger, const QString& defaultTriggerStr); virtual ~QOptArg() = default; const QString& shortName() const; const QString& name() const; virtual QString description() const; bool hasValue() const; // after parse: bool wasParsed() const; const QOptArgTrigger& optTrigger() const; virtual QString preprocessTrigger(const char* str) const; const QString& parsedTrigger() const; virtual void setParsedTrigger(const QString &parsedTrigger); template T getValue(const T& defaultValue=T()) const; template ContainerT getValues(const ContainerT& defaultValues={}); template ContainerT getValuesByDelim(const QString& delim=",", const ContainerT& defaultValues={}, const int minValueSize=1, const int maxValueSize=std::numeric_limits::max()); template QVariantList getVariantValues(const QVariantList& defaultValues={}); QStringList getOptions(int maxCount=std::numeric_limits::max()) const; QVariantList getVariantByteSizes(const QVariantList& defaultValues={}); QVariantList getVariantRelativeDateTimes(const QVariantList& defaultValues={}); void setAllowedOptions(const std::unordered_set& options, const QString&delimeter=","); const std::unordered_set& allowedOptions() const; const QString& allowedOptionsDelimeter() const; const RawValues_t& vals() const; void setVals(const RawValues_t &vals); int argIdx() const; void setArgIdx(int argIdx); const QString& defaultTriggerStr() const; bool internalOnly() const; void setInternalOnly(bool internalOnly); void addRequiredArg(const QOptArg* arg); const QVector& requiredArs() const; bool isFinalizeFlag() const; void setFinalizeFlag(bool f); void setIsByteSizeArg(bool isByteSizeArg); void setIsRelativeDateTime(bool isRelativeDateTime, bool subtractIt); protected: [[noreturn]] void throwgetValueCalledOnFlag(const char* functionname) const; QString m_shortName; QString m_name; QString m_description; bool m_hasValue; QOptArgTrigger m_optTrigger; QString m_defaultTriggerStr; int m_argIdx; bool m_internalOnly; RawValues_t m_vals; QVector m_requiredArs; // after parse: QString m_parsedTrigger; std::unordered_set m_allowedOptions; QString m_allowedOptionsDelimeter; bool m_isFinalizeFlag; bool m_isByteSizeArg; bool m_isRelativeDateTime; bool m_relativeDateTimeSubtract; }; /// Get the first value and try to convert it /// to the target type (throws on error). If the value /// is empty (not parsed), the default one is returned. /// @throws ExcCfg template T QOptArg::getValue(const T& defaultValue) const{ if(! m_hasValue){ throwgetValueCalledOnFlag(__func__); } if(m_vals.len == 0){ return defaultValue; } T t; try { qVariantTo_throw(m_vals.argv[0], &t, false); } catch (const ExcQVariantConvert& ex) { throw ExcOptArgParse(ex.descrip() + " (arg " + m_name + ')' ); } return t; } /// Try to convert all values /// to the target type (throws on error). If the values /// are empty (not parsed), the default ones are returned. /// @throws ExcCfg template ContainerT QOptArg::getValues(const ContainerT& defaultValues){ if(! m_hasValue){ throwgetValueCalledOnFlag(__func__); } if(m_vals.len == 0){ return defaultValues; } ContainerT container; for(int i=0; i < m_vals.len; i++){ typename ContainerT::value_type t; try { qVariantTo_throw(m_vals.argv[i], &t, false); } catch (const ExcQVariantConvert& ex) { throw ExcOptArgParse(ex.descrip() + " (arg " + m_name + ')' ); } container.push_back(t); } return container; } /// for a *single* argument string, whose values are separated by a delimter (e.g. comma) /// argFoo 1,2,3 template ContainerT QOptArg::getValuesByDelim(const QString& delim, const ContainerT& defaultValues, const int minValueSize, const int maxValueSize){ if(! m_hasValue){ throwgetValueCalledOnFlag(__func__); } if(m_vals.len == 0){ return defaultValues; } ContainerT container; const auto splittedVals = QString(m_vals.argv[0]).split(delim, Qt::SkipEmptyParts); if(splittedVals.size() < minValueSize || splittedVals.size() > maxValueSize){ throw ExcOptArgParse(qtr("argument %1 requires at least %2 and at most %3 " "parameters, separated by '%4' but %5 were given.") .arg(m_name).arg(minValueSize).arg(maxValueSize) .arg(delim).arg(splittedVals.size())); } for(const QString & val : splittedVals){ typename ContainerT::value_type t; try { qVariantTo_throw(val, &t, false); } catch (const ExcQVariantConvert& ex) { throw ExcOptArgParse(ex.descrip() + " (arg " + m_name + ')' ); } container.push_back(t); } return container; } /// Same as getValues(), but returns a QVariantList. /// The template parameter is there to convert the values into /// the target type right here. template QVariantList QOptArg::getVariantValues(const QVariantList& defaultValues) { if(! m_hasValue){ throwgetValueCalledOnFlag(__func__); } if(m_vals.len == 0){ return defaultValues; } QVariantList l; for(int i=0; i < m_vals.len; i++){ T t; try { qVariantTo_throw(m_vals.argv[i], &t, false); } catch (const ExcQVariantConvert& ex) { throw ExcOptArgParse(ex.descrip() + " (arg " + m_name + ')' ); } l.push_back(t); } return l; } ================================================ FILE: src/common/qoptargparse/qoptargparse.cpp ================================================ #include #include #include #include #include "qoptargparse.h" #include "qoptsqlarg.h" #include "qoptvarlenarg.h" #include "excoptargparse.h" #include "cleanupresource.h" #include "qoutstream.h" #include "cpp_exit.h" #include "qformattedstream.h" using RawValues_t = QOptArg::RawValues_t; namespace { /// Consume the next arguments according to the found trigger (if any). /// Store the found values in arg void consumeOptArgs(int argc, char *argv[], int& i, QOptArg& arg){ auto & optionalTrigger = arg.optTrigger(); QString preprocessedTrigger = arg.preprocessTrigger(argv[i]); auto foundTrigger = optionalTrigger.trigger().find(preprocessedTrigger); if(foundTrigger != optionalTrigger.trigger().end()) { arg.setParsedTrigger(preprocessedTrigger); // trigger given and consumed -> head to next arg ++i; } else { // The trigger is optional, probably it was leaved out (or mistyped). // In that case, the default trigger is used. // Note that the default trigger *must* be part of the allowed // trigger-set. foundTrigger = optionalTrigger.trigger().find(arg.defaultTriggerStr()); assert(foundTrigger != optionalTrigger.trigger().end()); } // Depending on the trigger a different count of values can be consumed RawValues_t v; v.argv = &argv[i]; for(v.len = 0; v.len < foundTrigger.value(); v.len++){ if(i >= argc){ // Out of args - delegate it to business logic break; } // This parser does *not* support empty commandline arguments. if(argv[i][0] == '\0'){ throw ExcOptArgParse(qtr("%1 has an empty value").arg(arg.name())); } ++i; } arg.setVals(v); } /// A var-len argument starts with the count of following arguments, which shall be consumed void consumeVarLenArg(int argc, char *argv[], int& i, QOptVarLenArg* arg){ const char* nArgStr = argv[i]; int nArgs; try { qVariantTo_throw(nArgStr, &nArgs, false); } catch (const ExcQVariantConvert&) { throw ExcOptArgParse(qtr("The argument %1 expects an integer as first value " "but %2 was given").arg(arg->name(), nArgStr)); } // increment for nArgStr and all following values RawValues_t v; v.argv = &argv[++i]; v.len = nArgs; i += nArgs; if(i > argc){ throw ExcOptArgParse(qtr("%1: too few arguments left (%2 required)") .arg(arg->name()).arg(nArgs)); } arg->setVals(v); } } // namespace QOptArgParse::QOptArgParse() = default; void QOptArgParse::addArg(QOptArg *arg) { assert(m_args.find(arg->name()) == m_args.end()); m_args.insert({arg->name(), arg}); if(! arg->shortName().isEmpty()){ // short names are optional in which case they are empty m_argsShort.insert({arg->shortName(), arg}); } } /// Parse the commandline for all previously added arguments (and for /// -h, --help, after which the application EXITS). /// Parsing starts at argv[0], so in case it was received from the parent process, /// rather increment it first (++argv; argc--;) /// @throws ExcOptArgParse void QOptArgParse::parse(int argc, char *argv[]) { // to know which args we got, create a copy and delete // elements on match auto argsCopy = m_args; if(argc > 0){ QByteArray first(argv[0]); if(first == "-h" || first == "--help"){ printHelp(); cpp_exit(0); } } // generate the vector here and not in addArg to allow // for adding requirements *after* an argument was added // to the parser QVector argsWithRequirements; for(int i=0; i < argc; ){ tsl::ordered_map::iterator argIter; const QString argStr = argv[i]; if(argStr.startsWith("--")){ // search in long names argIter = argsCopy.find(argStr); } else { // search in short names but then try to refind it in argsCopy (long names), // to know, which args we got argIter = m_argsShort.find(argStr); if(argIter == m_argsShort.end()){ argIter = argsCopy.end(); } else { argIter = argsCopy.find(argIter.value()->name()); } } if(argIter == argsCopy.end()){ if(m_args.find(argStr) != m_args.end()){ // maybe_todo: add mulit arg, if required and perform dynamic_cast to that // subclass throw ExcOptArgParse(argStr + qtr(" was passed multiple times")); } // We are done. Store rest-ptr m_rest.argv = &argv[i]; m_rest.len = argc - i; break; } // remember that we got this arg auto deleteArgLater = finally([&argsCopy, &argIter] { argsCopy.erase(argIter); }); QOptArg* arg = argIter.value(); arg->setArgIdx(i); if(! arg->requiredArs().isEmpty()){ argsWithRequirements.push_back(arg); } if(! arg->hasValue()){ // a simple flag if(arg->isFinalizeFlag()){ // We are done. Store rest-ptr ++i; m_rest.argv = &argv[i]; m_rest.len = argc - i; if(m_rest.len == 0){ throw ExcOptArgParse(qtr("'%1' passed without further arguments").arg(arg->name())); } break; } ++i; continue; } if(++i >= argc){ throw ExcOptArgParse(qtr("Missing value for %1").arg(arg->name())); } if(argv[i][0] == '\0'){ // This parser does *not* support empty commandline arguments. throw ExcOptArgParse(qtr("%1 has an empty value").arg(arg->name())); } auto* varLenArg = dynamic_cast(arg); // each of below cases has to point i to the next argument to be parsed! if(varLenArg != nullptr){ consumeVarLenArg(argc, argv, i, varLenArg); } else if(! arg->optTrigger().isEmpty()){ // special "feature" of this parser: consume the next argument(s) according // to the given, xor to the default trigger word. consumeOptArgs(argc, argv, i, *arg); } else { RawValues_t v; v.argv = &argv[i]; v.len = 1; arg->setVals(v); ++i; } } for(const QOptArg* argWithReq : argsWithRequirements){ for(const QOptArg* requiremnt : argWithReq->requiredArs()){ if(! requiremnt->wasParsed()){ throw ExcOptArgParse(qtr("'%1' is required by '%2' but was not parsed.") .arg(requiremnt->name(), argWithReq->name())); } } } } QOptArg::RawValues_t &QOptArgParse::rest() { return m_rest; } void QOptArgParse::setHelpIntroduction(const QString &txt) { m_helpIntroduction = txt; } void QOptArgParse::printHelp() { QFormattedStream s(stdout); struct winsize termWinSize; ioctl(STDOUT_FILENO, TIOCGWINSZ, &termWinSize); if(termWinSize.ws_col > 10 && termWinSize.ws_col < 80 ){ s.setMaxLineWidth(termWinSize.ws_col); } else { s.setMaxLineWidth(80); } s << m_helpIntroduction << "\n-h, --help :" << qtr("Print this help and exit") << "\n"; const QString indent = " "; for(const auto &nameArgPair : m_args){ if(nameArgPair.second->internalOnly()){ continue; } QString shortNameStr = nameArgPair.second->shortName(); if(! shortNameStr.isEmpty()){ shortNameStr += ", "; } QString value; if(nameArgPair.second->hasValue()){ if(nameArgPair.second->allowedOptions().empty()){ // first two characters are -- value =nameArgPair.second->name()[2]; } else { for(const QString& str : nameArgPair.second->allowedOptions()){ value += str + nameArgPair.second->allowedOptionsDelimeter(); } value.resize(value.size() - nameArgPair.second->allowedOptionsDelimeter().size()); } } s << shortNameStr << nameArgPair.second->name(); s.setLineStart(indent); s << value << ": " << nameArgPair.second->description() << "\n"; s.setLineStart(""); } } ================================================ FILE: src/common/qoptargparse/qoptargparse.h ================================================ #pragma once #include #include #include "qoptarg.h" #include "ordered_map.h" /// Currently no support for having the same argument multiple times class QOptArgParse { public: QOptArgParse(); void addArg(QOptArg* arg ); void parse(int argc, char *argv[]); QOptArg::RawValues_t& rest(); void setHelpIntroduction(const QString& txt); private: tsl::ordered_map m_args; tsl::ordered_map m_argsShort; QOptArg::RawValues_t m_rest; QString m_helpIntroduction; void printHelp(); }; ================================================ FILE: src/common/qoptargparse/qoptargtrigger.cpp ================================================ #include "qoptargtrigger.h" #include QOptArgTrigger::QOptArgTrigger() = default; QOptArgTrigger::QOptArgTrigger(QOptArgTrigger::TriggerEntries trigger) : m_trigger(std::move(trigger)) { } const QOptArgTrigger::TriggerEntries &QOptArgTrigger::trigger() const { return m_trigger; } bool QOptArgTrigger::isEmpty() const { return m_trigger.isEmpty(); } void QOptArgTrigger::setTrigger(const TriggerEntries &trigger) { m_trigger = trigger; } ================================================ FILE: src/common/qoptargparse/qoptargtrigger.h ================================================ #pragma once #include #include #include #include "util.h" /// Allow the consumption of multiple commandline arguments, in case /// a trigger word is given. Example: /// If the trigger word <-between> is given, two values shall be consumed, /// if <-greater> is given, one value shall be consumed. class QOptArgTrigger { public: // store for each trigger, how many values shall be consumed typedef QHash TriggerEntries; QOptArgTrigger(); QOptArgTrigger(TriggerEntries pTrigger); const TriggerEntries& trigger() const; void setTrigger(const TriggerEntries &trigger); bool isEmpty() const; private: TriggerEntries m_trigger; }; ================================================ FILE: src/common/qoptargparse/qoptsqlarg.cpp ================================================ #include #include "qoptsqlarg.h" #include "staticinitializer.h" using TriggerDefinitions = QOptArgTrigger::TriggerEntries; namespace { const QOptArgTrigger& allArgTrigger(){ static QOptArgTrigger allArgTrigger; static StaticInitializer loader( [](){ TriggerDefinitions triggerDefs; for(const E_CompareOperator& op : QOptSqlArg::cmpOpsAll()){ int countOfConsumeVals; switch (op) { case E_CompareOperator::BETWEEN: countOfConsumeVals=2 ;break; default: countOfConsumeVals=1 ;break; } triggerDefs.insert(CompareOperator(op).asTerminal(), countOfConsumeVals ); } allArgTrigger.setTrigger(triggerDefs); }); return allArgTrigger; } } // namespace const QOptSqlArg::CompareOperators &QOptSqlArg::cmpOpsAll() { static const QOptSqlArg::CompareOperators ops = { E_CompareOperator::GT, E_CompareOperator::GE, E_CompareOperator::LT, E_CompareOperator::LE, E_CompareOperator::EQ, E_CompareOperator::NE, E_CompareOperator::LIKE, E_CompareOperator::BETWEEN }; return ops; } const QOptSqlArg::CompareOperators &QOptSqlArg::cmpOpsAllButLike() { static const QOptSqlArg::CompareOperators ops = { E_CompareOperator::GT, E_CompareOperator::GE, E_CompareOperator::LT, E_CompareOperator::LE, E_CompareOperator::EQ, E_CompareOperator::NE, E_CompareOperator::BETWEEN }; return ops; } const QOptSqlArg::CompareOperators &QOptSqlArg::cmpOpsText() { static const QOptSqlArg::CompareOperators ops = { E_CompareOperator::EQ, E_CompareOperator::NE, E_CompareOperator::LIKE }; return ops; } const QOptSqlArg::CompareOperators &QOptSqlArg::cmpOpsEqNe() { static const QOptSqlArg::CompareOperators ops = { E_CompareOperator::EQ, E_CompareOperator::NE }; return ops; } QOptSqlArg::QOptSqlArg(const QString &shortName, const QString &name, const QString &description, const CompareOperators &supportedOperators, const E_CompareOperator &defaultOperator) : QOptArg(shortName, name, description, allArgTrigger(), CompareOperator(defaultOperator).asTerminal()), m_parsedOperator(defaultOperator), m_supportedOperators(supportedOperators) { if(supportedOperators.empty()){ throw QExcIllegalArgument("supportedOperators is empty"); } if(! supportedOperators.contains(defaultOperator)){ throw QExcIllegalArgument("supportedOperators does not contain defaultOperator"); } } void QOptSqlArg::setParsedTrigger(const QString &parsedTrigger) { QOptArg::setParsedTrigger(parsedTrigger); CompareOperator op = CompareOperator(); if(! op.fromTerminal(parsedTrigger)){ throw ExcOptArgParse(qtr("Failed to convert %1 to a sql comparison operator") .arg(parsedTrigger)); } if(! m_supportedOperators.contains(op.asEnum())){ throw ExcOptArgParse(qtr("%1 is not a supported sql comparison operator for %2") .arg(parsedTrigger, name())); } m_parsedOperator = op.asEnum(); } /// All sql parameters are processed internally as lower strings. QString QOptSqlArg::preprocessTrigger(const char *str) const { return QString(str).toLower(); } QString QOptSqlArg::description() const { QStringList operators; for(const auto& op : m_supportedOperators){ operators.push_back(CompareOperator(op).asTerminal()); } return m_description + qtr(" Supported operators: %1. Default operator: %2") .arg(operators.join(", "), m_defaultTriggerStr); } E_CompareOperator QOptSqlArg::parsedOperator() const { return m_parsedOperator; } ================================================ FILE: src/common/qoptargparse/qoptsqlarg.h ================================================ #pragma once #include #include "qoptarg.h" #include "compareoperator.h" class QOptSqlArg : public QOptArg { public: typedef QVector CompareOperators; static const CompareOperators& cmpOpsAll(); static const CompareOperators& cmpOpsAllButLike(); static const CompareOperators& cmpOpsText(); static const CompareOperators& cmpOpsEqNe(); QOptSqlArg(const QString& shortName, const QString & name, const QString& description, const CompareOperators& supportedOperators, const E_CompareOperator& defaultOperator=E_CompareOperator::EQ); void setParsedTrigger(const QString &parsedTrigger) override; QString preprocessTrigger(const char* str) const override; QString description() const override; E_CompareOperator parsedOperator() const; private: E_CompareOperator m_parsedOperator; CompareOperators m_supportedOperators; }; ================================================ FILE: src/common/qoptargparse/qoptvarlenarg.cpp ================================================ #include "qoptvarlenarg.h" QOptVarLenArg::QOptVarLenArg(const QString &shortName, const QString &name, const QString &description) : QOptArg(shortName, name, description, true) {} ================================================ FILE: src/common/qoptargparse/qoptvarlenarg.h ================================================ #pragma once #include "qoptarg.h" class QOptVarLenArg : public QOptArg { public: QOptVarLenArg(const QString& shortName, const QString & name, const QString& description); }; ================================================ FILE: src/common/qresource_helper.cpp ================================================ #include "qresource_helper.h" #include "compat.h" /// /// \brief qresource_helper::data_safe uncompress data as neeeded /// \param r /// \return /// QByteArray qresource_helper::data_safe(QResource &r) { QByteArray data = Qt::resourceIsCompressed(r) ? qUncompress(r.data(), int(r.size())) : QByteArray(reinterpret_cast(r.data())); return data; } ================================================ FILE: src/common/qresource_helper.h ================================================ #pragma once #include namespace qresource_helper { QByteArray data_safe(QResource& r); } ================================================ FILE: src/common/qsimplecfg/CMakeLists.txt ================================================ add_library(lib_qsimplecfg cfg.cpp section.cpp exccfg.cpp ) target_link_libraries(lib_qsimplecfg PUBLIC Qt5::Core lib_util oscpp_lib lib_orderedmap ) ================================================ FILE: src/common/qsimplecfg/cfg.cpp ================================================ #include #include #include #include #include #include #include #include "cfg.h" #include "exccfg.h" #include "util.h" #include "qformattedstream.h" #include "cflock.h" #include "excos.h" #include "os.h" #include "interrupt_handler.h" namespace { void setStreamCommentMode(QFormattedStream& s){ s.setMaxLineWidth(80); s.setLineStart("# "); } void unsetStreamCommentMode(QFormattedStream& s){ s.setMaxLineWidth(std::numeric_limits::max()); s.setLineStart(""); } void writeMultiLineKey(QFormattedStream& stream, const QString& keyname, const QString& value){ assert(stream.streamChunkSep() == '\n'); auto oldMaxLineWidth = stream.maxLineWidth(); stream.setMaxLineWidth(std::numeric_limits::max()); const QString TRIPLE_QUOTE = "'''"; stream << keyname + " = " + TRIPLE_QUOTE; auto oldLineStart = stream.lineStart(); // repsecting oldLineStart makes this function compatible // with comment and normal mode. stream.setLineStart(oldLineStart + " "); stream << value; stream.setLineStart(oldLineStart); stream << TRIPLE_QUOTE; stream.setMaxLineWidth(oldMaxLineWidth); } } // namespace /// Parse the config file at filepath. Create it, if necessary. /// Note that the content of multi-line strings between triple quotes /// is parsed "as is", except for an optional final \n, if the closing triple /// quotes are in the next line. Example: /// ''' /// text /// ''' /// -> no \n after (but before it there is one). /// So it does not matter, whether the closing triple quotes are in the same /// or the next line. /// @throws ExcCfg qsimplecfg::Cfg::Cfg() : m_allowEraseSections(true) {} void qsimplecfg::Cfg::parse(const QString &filepath) { createDirsToFilename(filepath); QFile file(filepath); if(! file.open(QIODevice::OpenModeFlag::ReadOnly | QIODevice::OpenModeFlag::Text)){ throw ExcCfg(qtr("Failed to open %1 - %2"). arg(filepath, file.errorString())); } parse(file); } /// @overload /// @param file: parse the already for reading opened file (whose offset should typically be zero). void qsimplecfg::Cfg::parse(QFile &file) { QTextStream in(&file); try{ parse(&in); } catch(ExcCfg & ex){ ex.setDescrip(ex.descrip() + qtr(". Please correct the file at %1").arg(QFileInfo(file).absoluteFilePath())); throw; } } void qsimplecfg::Cfg::store(const QString &filepath) { createDirsToFilename(filepath); QFile file(filepath); if(! file.open(QIODevice::OpenModeFlag::WriteOnly | QIODevice::OpenModeFlag::Text)){ throw ExcCfg(qtr("Failed to open %1 - %2"). arg(filepath, file.errorString())); } store(file); } /// Save config at given filepath. **Not** safe against races. /// @throws ExcCfg void qsimplecfg::Cfg::store(QFile &file) { QFormattedStream stream(&file); stream.setStreamChunkSep('\n'); unsetStreamCommentMode(stream); if(! m_initialComments.isEmpty()){ setStreamCommentMode(stream); stream << m_initialComments; unsetStreamCommentMode(stream); } stream << "\n\n"; for(const auto& nameSect : m_nameSectionHash){ stream << '[' + nameSect.second.sect->sectionName() + "]"; writeSectionCommentsToStream(nameSect.second.sect, stream); writeSectionToStream(nameSect.second.sect, stream); stream << "\n\n"; } } /// Get a parsed section or create a new one with given name. /// The order in which operator[] is called, /// determines the order in which it will be stored to disk on Cfg::store(). /// Parsed sections ( see parse() ) which were not requested via this function, /// will *not* be store()'ed. The idea is that the config scheme is autogenerated /// by requesting the sections and keys. /// Note that calling this function a second time with the same section name, /// after another section was created, does *not* change the order. /// @return the Section_Ptr is never null. qsimplecfg::Cfg::Section_Ptr qsimplecfg::Cfg::operator[](const QString §Name) { auto parsedIt = m_parsedNameSectionHash.find(sectName); if(parsedIt != m_parsedNameSectionHash.end()){ SectWithMeta sectMeta; sectMeta.sect = parsedIt->second; m_nameSectionHash.insert({sectName, sectMeta}); m_parsedNameSectionHash.erase(parsedIt); return sectMeta.sect; } // section was not parsed or requested a second time: get or create: auto & sectMeta = m_nameSectionHash[sectName]; if(sectMeta.sect == nullptr){ sectMeta.sect = make_shared_section(sectName); } return sectMeta.sect; } void qsimplecfg::Cfg::handleParseKeyValue(QStringRef &line, size_t *pLineNumber, QTextStream *stream,const Section_Ptr& section) { int equalIdx = line.indexOf('='); if(equalIdx == -1){ throw ExcCfg(qtr("Line %1 - %2: Unexpected content (missing =)"). arg(*pLineNumber).arg(line.toString())); } QStringRef key = line.left(equalIdx).trimmed(); QStringRef value = line.mid(equalIdx + 1).trimmed(); if(! value.startsWith("'''")){ // simple case: not a multi-line string section->insert(key.toString(), value.toString()); return; } // ignore leading ''' value = value.mid(3); // still possible that string ends in same line: int tripleIdx = value.indexOf("'''"); if(tripleIdx != -1){ if(tripleIdx != value.length()-3){ throw ExcCfg(qtr("Line %1 - %2: content after closing triple quotes '''"). arg(*pLineNumber).arg(line.toString())); } section->insert(key.toString(), value.left(value.size() - 3).toString()); return; } m_keyValBuf = value.toString(); // mutli line string: keep going through file until the // next ''' size_t startingLine = *pLineNumber; while (true) { if(! readLineInto(*stream, &m_keyValReadBuf)){ break; } (*pLineNumber)++; QStringRef currentLine(&m_keyValReadBuf); currentLine = currentLine.trimmed(); tripleIdx = currentLine.indexOf("'''"); if(tripleIdx == -1){ // keep \n's for later split m_keyValBuf += '\n' + currentLine.toString() ; continue; } if(tripleIdx != currentLine.length()-3){ throw ExcCfg(qtr("Line %1 - %2: content after closing triple quotes '''"). arg(*pLineNumber).arg(currentLine.toString())); } if(tripleIdx != 0){ m_keyValBuf += '\n' + currentLine.left(currentLine.size() - 3).toString(); } section->insert(key.toString(), m_keyValBuf); return; } throw ExcCfg(qtr("Line %1 - %2: missing closing triple quotes '''"). arg(startingLine).arg(line.toString())); } void qsimplecfg::Cfg::writeKeyValue(const QString &key, const QString &val, const QString &sep, QFormattedStream &stream) { if(sep.contains('\n') || val.contains('\n')){ writeMultiLineKey(stream, key, val); } else { stream << key + " = " + val; } } void qsimplecfg::Cfg::writeSectionToStream(const qsimplecfg::Cfg::Section_Ptr §, QFormattedStream &stream) { for(const auto & keyValMeta : sect->keyValHash()){ QString valStr; if(keyValMeta.second.insertDefault){ valStr = QVariantListToString( keyValMeta.second.defaultValues, keyValMeta.second.separator); } else { if(keyValMeta.second.rawStr.isNull()){ // No value was parsed and default shall // not be inserted -> do not write this // key to file. continue; } valStr = keyValMeta.second.rawStr.trimmed(); } writeKeyValue(keyValMeta.first, valStr, keyValMeta.second.separator, stream); } } void qsimplecfg::Cfg::writeSectionCommentsToStream(const qsimplecfg::Cfg::Section_Ptr §, QFormattedStream &stream) { setStreamCommentMode(stream); if(! sect->comments().isEmpty()){ stream << sect->comments() + '\n'; } for(const auto& keyValMeta : sect->keyValHash()){ const auto & key = keyValMeta.first; const auto & valMeta = keyValMeta.second; if(! valMeta.insertDefaultToComments){ continue; } const QString defaultValStr = QVariantListToString(valMeta.defaultValues, valMeta.separator); writeKeyValue(key, defaultValStr, valMeta.separator, stream); } unsetStreamCommentMode(stream); } /// @throws ExcCfg void qsimplecfg::Cfg::createDirsToFilename(const QString &filename) { assert(! filename.isEmpty()); QFileInfo fileInfo(filename); if(! QDir().mkpath(fileInfo.absolutePath())){ throw ExcCfg(qtr("Failed to create directories for path %1") .arg(fileInfo.absolutePath()) ); } } QString qsimplecfg::Cfg::QVariantListToString(const QVariantList &l, const QString &sep) { QString str; for(const auto& v : l){ str += qVariantTo_throw(v) + sep; } return str; } void qsimplecfg::Cfg::setInitialComments(const QString &comments) { m_initialComments = comments; } /// Return all sections and their keys which were not read after having been /// inserted. qsimplecfg::NotReadSectionKeys qsimplecfg::Cfg::generateNonReadSectionKeyPairs() { NotReadSectionKeys allNotRead; for(auto it = m_nameSectionHash.begin(); it != m_nameSectionHash.end(); ++it){ auto notReadKeys = it.value().sect->notReadKeys(); if(! notReadKeys.empty()){ allNotRead.push_back({it.key(), notReadKeys}); } } return allNotRead; } /// Rename a parsed section. Warning: it is *not* allowed, to call this function /// after having accessed a section via operator[], because that would destroy /// the order of the sections. /// @return true, if the old section existed. bool qsimplecfg::Cfg::renameParsedSection(const QString &oldName, const QString &newName) { assert(m_nameSectionHash.empty()); auto oldIt = m_parsedNameSectionHash.find(oldName); if(oldIt == m_parsedNameSectionHash.end()){ return false; } Section_Ptr sect = oldIt->second; sect->setSectionName(newName); m_parsedNameSectionHash.erase(oldIt); m_parsedNameSectionHash[newName] = sect; return true; } /// @return those sections which were parsed but not accessed via operator[] const qsimplecfg::Cfg::ParsedNameSectionHash& qsimplecfg::Cfg::getParsedButNotReadNameSectionHash() const { return m_parsedNameSectionHash; } /// Get a parsed section. Warning: it is *not* allowed, to call this function /// after having accessed a section via operator[], because that would destroy /// the order of the sections. So call this function after Cfg::parse but before /// accessing any section via operator[]. /// @return the parsed section or null qsimplecfg::Cfg::Section_Ptr qsimplecfg::Cfg::getParsedSectionIfExist(const QString §Name) { assert(m_nameSectionHash.empty()); auto it = m_parsedNameSectionHash.find(sectName); if(it == m_parsedNameSectionHash.end()){ return nullptr; } return it->second; } /// @overload void qsimplecfg::Cfg::parse(QTextStream *in) { m_parsedNameSectionHash.clear(); m_nameSectionHash.clear(); m_initialComments.clear(); bool withinSection=false; Section_Ptr currentSect; QString currentSectName; size_t currentLine = 0; QString lineBuf; lineBuf.reserve(8192); while (true) { if(! readLineInto(*in, &lineBuf)){ break; } QStringRef line(&lineBuf); line = line.trimmed(); currentLine++; if(line.startsWith('#')){ // No point in reading comments. } else if(line.isEmpty()){ } else if(line.startsWith('[')){ if(! line.endsWith(']')){ throw ExcCfg(qtr("Line %1 - %2: section start [ without closing end ] detected"). arg(currentLine).arg(line.toString())); } withinSection = true; currentSectName = line.mid(1, line.size() - 2).toString(); if(currentSectName.isEmpty()){ throw ExcCfg(qtr("Line %1 - %2: empty section detected"). arg(currentLine).arg(line.toString())); } auto currentSectIt = m_parsedNameSectionHash.find(currentSectName); if(currentSectIt != m_parsedNameSectionHash.end()){ throw ExcCfg(qtr("Line %1 - %2: section name already defined (in upper line)"). arg(currentLine).arg(line.toString())); } currentSect = make_shared_section(currentSectName); m_parsedNameSectionHash.insert({currentSectName, currentSect}); } else { if(! withinSection){ throw ExcCfg(qtr("Line %1 - %2: Content before first section"). arg(currentLine).arg(line.toString())); } handleParseKeyValue(line, ¤tLine, in, currentSect ); } } } ================================================ FILE: src/common/qsimplecfg/cfg.h ================================================ #pragma once #include #include #include #include #include #include "section.h" class QTextStream; class CfgTest; // unit test class QFormattedStream; namespace qsimplecfg { /// First Element of each pair is section-name, second a set of not read keys typedef QVector > > NotReadSectionKeys ; /// A simple parser for ini-like config files. /// The scheme of the config file, that is: the order of sections /// and keys and their default values is autogenerated by subsequent /// calls to Cfg::operator[] and Section::getValue. *Only* those sections /// and keys are written (back) to the config-file, which were accessed /// by these methods. The order written to file is exactly the access order. /// Scheme updates work by renaming sections (Cfg::renameParsedSection) and /// keys (Section::renameParsedKey) after Cfg::parse but before accessing /// a value via the above described methods. To rename a key, obtain /// the section via Cfg::getParsedSectionIfExist and *not* Cfg::operator[]. /// /// No subsections are supported but: /// - initial file comment /// - comments after a section header: /// [sectionname] /// # comment1 /// # comment2 /// - values over multiple lines with triple quotes ''': /// key = '''foo1 /// foo2''' /// - comments are always re-written by the application on store(), /// but ignored when parsing the file. class Cfg { public: typedef std::shared_ptr
Section_Ptr; typedef std::unordered_map ParsedNameSectionHash; Cfg(); void parse(const QString& filepath); void parse(QFile& file); void store(const QString& filepath); void store(QFile& file); Section_Ptr operator[](const QString §Name); void setInitialComments(const QString &comments); NotReadSectionKeys generateNonReadSectionKeyPairs(); bool renameParsedSection(const QString& oldName, const QString& newName); const ParsedNameSectionHash& getParsedButNotReadNameSectionHash() const; Section_Ptr getParsedSectionIfExist(const QString& sectName); private: // maybe_todo: store plain section instead, if sure, that no metadata // about it needs to be stored... struct SectWithMeta { Section_Ptr sect; }; typedef tsl::ordered_map SectionHash; ParsedNameSectionHash m_parsedNameSectionHash; // parsed from file SectionHash m_nameSectionHash; // accessed by user via operator[] QString m_initialComments; QString m_keyValReadBuf; QString m_keyValBuf; bool m_allowEraseSections; void parse(QTextStream *in); void handleParseKeyValue(QStringRef &line, size_t* pLineNumber, QTextStream* stream, const Section_Ptr §ion); void writeKeyValue(const QString& key, const QString& val, const QString& sep, QFormattedStream& stream); void writeSectionToStream(const Section_Ptr& sect, QFormattedStream& stream); void writeSectionCommentsToStream(const Section_Ptr& sect, QFormattedStream& stream); static void createDirsToFilename(const QString& filename); static std::shared_ptr
make_shared_section(const QString& sectName) { // since section is private, but make_shared requires it to be public, the dummy inheritance is // one soultion struct allow_mk_shared : public Section { allow_mk_shared(const QString& sectName) : Section(sectName) {} }; return std::make_shared(sectName); } static QString QVariantListToString(const QVariantList& l, const QString& sep); // unit test: friend class ::CfgTest; }; } // namespace qsimplecfg ================================================ FILE: src/common/qsimplecfg/exccfg.cpp ================================================ #include "exccfg.h" qsimplecfg::ExcCfg::ExcCfg(const QString &preamble) : QExcCommon (preamble, false) {} ================================================ FILE: src/common/qsimplecfg/exccfg.h ================================================ #pragma once #include #include "exccommon.h" namespace qsimplecfg { class ExcCfg : public QExcCommon { public: ExcCfg(const QString & preamble); }; } // namespace qsimplecfg ================================================ FILE: src/common/qsimplecfg/section.cpp ================================================ #include "section.h" #include "conversions.h" qsimplecfg::Section::Section(const QString §ionName) : m_sectionName(sectionName) {} void qsimplecfg::Section::insert(const QString &key, const QString &value) { m_parsedKeyValHash.insert({key, value}); m_NotReadKeys.insert(key); } /// @see getValue(). qint64 qsimplecfg::Section::getFileSize(const QString &key, const qint64 &defaultValue, bool insertDefaultIfNotExist) { try { Conversions userStrConv; return userStrConv.bytesFromHuman( this->getValue( key, userStrConv.bytesToHuman(defaultValue), insertDefaultIfNotExist)); } catch (const ExcConversion& ex) { throw qsimplecfg::ExcCfg(ex.descrip() + " (key " + key + ')' ); } } const qsimplecfg::Section::KeyMetaValHash &qsimplecfg::Section::keyValHash() { return m_keyValHash; } void qsimplecfg::Section::setComments(const QString &comments) { m_comments = comments; if(! comments.isEmpty() && comments[comments.size() - 1] != QChar::LineFeed){ m_comments.push_back(QChar::LineFeed); } } const QString& qsimplecfg::Section::comments() const { return m_comments; } /// Affects subsequent calls to getValue(s): if true, default values will be written /// to comments on Cfg::store, else not. void qsimplecfg::Section::setInsertDefaultToComments(bool insertDefaultToComments) { m_insertDefaultToComments = insertDefaultToComments; } void qsimplecfg::Section::removeFromNotReadKeysIfExist(const QString &key) { auto it = m_NotReadKeys.find(key); if(it != m_NotReadKeys.end()){ m_NotReadKeys.erase(it); } } qsimplecfg::Section::ValueWithMeta& qsimplecfg::Section::generateValueWithMeta(const QString &key, const QString& separator, const QVariantList& defaultValues, bool insertDefaultIfNotExist) { removeFromNotReadKeysIfExist(key); ValueWithMeta & valWithMeta = m_keyValHash[key]; valWithMeta.insertDefaultToComments = m_insertDefaultToComments; valWithMeta.separator = separator; valWithMeta.defaultValues = defaultValues; auto parsedValIt = m_parsedKeyValHash.find(key); if(parsedValIt == m_parsedKeyValHash.end()){ valWithMeta.rawStr = QString(); valWithMeta.insertDefault = insertDefaultIfNotExist; } else { valWithMeta.rawStr = parsedValIt->second; } return valWithMeta; } void qsimplecfg::Section::setSectionName(const QString §ionName) { m_sectionName = sectionName; } const QString& qsimplecfg::Section::sectionName() const { return m_sectionName; } /// Rename a parsed key. Warning: it is *not* allowed, to call this function /// after having accessed a key via getValue(), because that would destroy /// the order of the keys. So call this function after Cfg::parse but before /// accessing any value. bool qsimplecfg::Section::renameParsedKey(const QString &oldName, const QString &newName) { assert(m_keyValHash.empty()); auto oldIt = m_parsedKeyValHash.find(oldName); if(oldIt == m_parsedKeyValHash.end()){ return false; } const QString value = oldIt->second; m_parsedKeyValHash[newName] = value; m_parsedKeyValHash.erase(oldIt); return true; } /// return those keys which were not read via getValue[s]() after insert(); const std::unordered_set& qsimplecfg::Section::notReadKeys() const { return m_NotReadKeys; } ================================================ FILE: src/common/qsimplecfg/section.h ================================================ #pragma once #include #include #include #include #include #include #include "compat.h" #include "ordered_map.h" #include "exccfg.h" #include "util.h" #include "generic_container.h" namespace qsimplecfg { /// A config section consisting of optional initial comments /// and key-value pairs. Example /// # Some comment /// key1 = val1 /// key2 = val2 /// /// If insertDefaultToComments is true, add a comment of the form /// key = defaultValue when getValue is called, as hint for the user. /// Note that comments are ignored when parsing the config file, they are /// set by the application (and written to the file on store). class Section { public: qint64 getFileSize(const QString & key, const qint64& defaultValue={}, bool insertDefaultIfNotExist=false ); template T getValue(const QString & key, const T& defaultValue=T(), bool insertDefaultIfNotExist=false); template ContainerT getValues(const QString & key, const ContainerT & defaultValue=ContainerT(), bool insertDefaultIfNotExist=false, const QString & separator=",", Qt::SplitBehavior splitbehaviour=Qt::SkipEmptyParts ); void setComments(const QString &comments); const QString & comments() const; void setInsertDefaultToComments(bool insertDefaultToComments); const std::unordered_set& notReadKeys() const; const QString §ionName() const; bool renameParsedKey(const QString& oldName, const QString& newName); public: ~Section() = default; Q_DISABLE_COPY(Section) DISABLE_MOVE(Section) private: struct ValueWithMeta { QString rawStr; // isNull() == true, if not parsed. QString separator; QVariantList defaultValues; bool insertDefault {false}; bool insertDefaultToComments {true}; }; private: friend class Cfg; typedef tsl::ordered_map KeyMetaValHash; // methods to be called from class Cfg: Section(const QString& sectionName); void setSectionName(const QString §ionName); void insert(const QString & key, const QString& value); const KeyMetaValHash& keyValHash(); QString generateComments(); private: void removeFromNotReadKeysIfExist(const QString& key); template T convertValueOrThrow(const QString& valueStr, const QString& keyname); ValueWithMeta& generateValueWithMeta(const QString& key, const QString &separator, const QVariantList &defaultValues, bool insertDefaultIfNotExist); QString m_comments; std::unordered_map m_parsedKeyValHash; // parsed from file KeyMetaValHash m_keyValHash; // accessed by user via getValue(s) bool m_insertDefaultToComments { true }; std::unordered_set m_NotReadKeys; QString m_sectionName; }; /// Find the 'value' correspondig to 'key' and try to convert it /// to the target type (throws on error). If the value is not /// found or is empty, the default one is returned, which is also inserted /// into the section, if param insertDefaultIfNotExist is true. /// The default value *must* be convertible to a string using qVariantTo<>. /// If possible, the default is stored as hint in the comments. /// @throws ExcCfg template T Section::getValue(const QString & key, const T& defaultValue, bool insertDefaultIfNotExist){ #ifndef NDEBUG QString assertTmpResult; assert(qVariantTo(defaultValue, &assertTmpResult) ); #endif auto & valWithMeta = generateValueWithMeta(key, QString(), { QVariant::fromValue(defaultValue) }, insertDefaultIfNotExist); if(valWithMeta.rawStr.isNull()){ // not parsed, return default; return defaultValue; } return convertValueOrThrow(valWithMeta.rawStr, key); } /// Similar to getValue, but support for multiple values stored within the /// same key. The single-value container, whose elements are of ValT is then /// filled. The value is *always* trimmed. /// @throws ExcCfg template ContainerT Section::getValues(const QString &key, const ContainerT &defaultValue, bool insertDefaultIfNotExist, const QString &separator, Qt::SplitBehavior splitbehaviour) { QVariantList defaultVariantValues; for(const auto & val : defaultValue){ #ifndef NDEBUG QString assertTmpResult; assert(qVariantTo(val, &assertTmpResult) ); #endif defaultVariantValues.push_back(QVariant::fromValue(val)); } auto & valWithMeta = generateValueWithMeta(key, separator, defaultVariantValues, insertDefaultIfNotExist); if(valWithMeta.rawStr.isNull()){ // not parsed, return default; return defaultValue; } QStringList list = valWithMeta.rawStr.split(separator, splitbehaviour); ContainerT container; for(const QString & el : list){ auto parsedVal = convertValueOrThrow(el, key); addToContainer(container, parsedVal); } return container; } ////////////////////////////////// private ////////////////////////////////// template T Section::convertValueOrThrow(const QString &valueStr, const QString &keyname) { try { return qVariantTo_throw(valueStr, false); } catch (const ExcQVariantConvert& ex) { throw qsimplecfg::ExcCfg( qtr("%1 (key %2) in section %3") .arg(ex.descrip(), keyname, m_sectionName)); } } } // namespace qsimplecfg ================================================ FILE: src/common/safe_file_update.h ================================================ #pragma once #include #include #include #include "interrupt_handler.h" #include "logger.h" #include "os.h" #include "osutil.h" #include "qfilethrow.h" #include "util.h" /// Safeley read and update (config) files without locking, even on NFS(!). /// For 'normal' filesystems (e.g. ext4) renaming files within the same filesystem /// is an 'atomic' operation. However, on NFS this is not the case (see e.g. /// https://serverfault.com/questions/817887/rename-on-nfs-atomicity ). However, link- /// or directory-creation is referred being atomic (https://unix.stackexchange.com/a/125946). /// Therefore, we use the following procedure: /// The basic procedure is: /// * Updaters 'atomically' create a lock_dir, do the rename, sync, and remove the lock /// * Readers are prepared for non-existing files and stale reads, whereupon they try to /// gain the lock themselves. Once they got the lock, they try to read the config file /// again, to differentiate file-in-update from file-not-exist. class SafeFileUpdate { public: explicit SafeFileUpdate(const QString& filepath): m_filepath(filepath), m_lockfilepath(filepath.toLocal8Bit() + "__lock"), m_file(filepath) {} ~SafeFileUpdate() { // Do not throw from destructor try { if(m_isLocked){ doUnlock(); } } catch (const std::exception& ex ) { std::cerr << ex.what() << "\n"; } } QFileThrow& file() { return m_file; } template bool read(F func){ const QIODevice::OpenMode openflags = QIODevice::OpenModeFlag::ReadOnly | QIODevice::OpenModeFlag::Text; auto finallyClose = finally([this]{ if(m_file.isOpen()){ m_file.close(); } }); try { m_file.open(openflags); func(); return true; } catch (const QExcIo& ex) { switch (ex.errorNumber()) { case ENOENT: break; case ESTALE: break; default: logWarning << "unhandled error in" <<__func__ << "file" << m_filepath; throw; } } doLock(); // logDebug << "got lock for reading"; QFileInfo fileInfo(m_filepath); if(! fileInfo.exists()){ // We got the lock, but the file still does not exist. // In the case of config files this usually means, that we'll // initially create it later. We keep the lock. logDebug << "file does not exist:" << m_filepath; return false; } // We got the lock and the file exists. Reading it should now succeed try { // reopen the file anyway, as the descriptor might be 'stale'. if(m_file.isOpen()){ m_file.close(); } m_file.open(openflags); func(); } catch (const std::exception&) { logDebug << "second attempt failed altough we got the lock. Oh oh..."; doUnlock(); throw; } return true; } template void write(F func){ bool renameSuccess = false; int fd = -1; if(! m_isLocked){ doLock(); } // logDebug << "got lock for update"; QByteArray tmpFilepath = pathJoinFilename( QFileInfo(m_filepath).absolutePath().toLocal8Bit(), QByteArray("tmp.XXXXXX")); auto finalActions = finally([this, &fd, &renameSuccess, &tmpFilepath]{ if(fd != -1 && ! renameSuccess){ os::remove(tmpFilepath); } if(m_file.isOpen()){ m_file.close(); } this->doUnlock(); }); fd = osutil::mktmp(tmpFilepath); m_file.open(fd, QIODevice::OpenModeFlag::ReadWrite, QFileDevice::AutoCloseHandle); func(); // Also flushes the file before sync m_file.close(); auto dstPath = m_filepath.toLocal8Bit(); os::rename(tmpFilepath, dstPath); renameSuccess = true; sync(); } public: Q_DISABLE_COPY(SafeFileUpdate) private: QString m_filepath; QByteArray m_lockfilepath; QFileThrow m_file; bool m_isLocked{false}; InterruptProtect m_interruptProtect; void doLock(){ if(m_isLocked){ throw QExcProgramming(QString(__func__) + ": already locked " + m_filepath); } QFileInfo fileInfo(m_filepath); if(! QDir().mkpath(fileInfo.absolutePath())){ throw QExcIo(qtr("Failed to create directories for path %1") .arg(fileInfo.absolutePath()) ); } m_interruptProtect.enable(os::catchableTermSignals()); // try to lock the path by creating a dir. Link creation // is atomically on NFS. for(int i=0; i < 10; i++){ if(mkdir(m_lockfilepath.data(), 0755) == 0){ m_isLocked = true; return; } if(errno != EEXIST){ throw QExcIo("failed to create lockfile " + m_lockfilepath); } sleep(1); } throw QExcIo("Gave up creating lock-directory " + m_lockfilepath + ". If it's not a load-problem, please remove the stale directory."); } void doUnlock(){ if(!m_isLocked){ throw QExcProgramming(QString(__func__) + ": not locked " + m_filepath); } if(rmdir(m_lockfilepath) != 0){ logCritical << __func__ << "failed to remove lockpath:" << strerror(errno); } m_interruptProtect.disable(); m_isLocked = false; } }; ================================================ FILE: src/common/settings.cpp ================================================ #include #include #include #include #include #include #include "settings.h" #include "cfg.h" #include "exccfg.h" #include "os.h" #include "util.h" #include "pathtree.h" #include "logger.h" #include "app.h" #include "translation.h" #include "cflock.h" #include "qfilethrow.h" #include "conversions.h" #include "safe_file_update.h" using Section_Ptr = qsimplecfg::Cfg::Section_Ptr; using qsimplecfg::ExcCfg; using StringSet = Settings::StringSet; using std::numeric_limits; const char* Settings::SECT_READ_NAME {"File read-events"}; const char* Settings::SECT_READ_KEY_ENABLE {"enable"}; const char* Settings::SECT_READ_KEY_INCLUDE_PATHS {"include_paths"}; const char* Settings::SECT_SCRIPTS_NAME {"File read-events storage settings"}; const char* Settings::SECT_SCRIPTS_ENABLE {"enable"}; const char* Settings::SECT_SCRIPTS_INCLUDE_PATHS {"include_paths"}; const char* Settings::SECT_SCRIPTS_INCLUDE_FILE_EXTENSIONS {"include_file_extensions"}; Settings &Settings::instance() { static Settings s; return s; } void Settings::setUserCfgDir(const QString &p) { m_userCfgDir = p; } void Settings::setUserDataDir(const QString &p) { m_userDataDir = p; } const QStringList &Settings::defaultIgnoreCmds() { // commands ending with an asterisk will be // later inserted into the to-ignore-commands // No args may be added in that case. static const QStringList vals = {"mount*", QString(app::SHOURNAL) + '*', QString(app::SHOURNAL_RUN) + '*' }; return vals; } QString Settings::cfgAppDir() { if(! m_userCfgDir.isEmpty()){ return m_userCfgDir; } // don't make path static -> mutliple test cases... return pathJoinFilename(QStandardPaths::writableLocation(QStandardPaths::ConfigLocation), QCoreApplication::applicationName()); } QString Settings::cfgFilepath() { // don't make path static -> mutliple test cases... return cfgAppDir() + "/config.ini"; } QString Settings::dataDir() { if(! m_userDataDir.isEmpty()){ return m_userDataDir; } return QStandardPaths::writableLocation(QStandardPaths::DataLocation); } // was in use until shournal 2.1, then migrated // from .cache/shournal to .config/shournal static QString legacyCfgVersionFilePath(){ // don't make path static -> mutliple test cases... const QString path = QStandardPaths::writableLocation(QStandardPaths::CacheLocation) + "/config-file-version"; return path; } /// For lines not ending with an asterisk: /// The first word (whitespace!) is considered the command, whose full /// path is found, the rest is appended as arguments. /// If no command could be found, the returned string is empty /// Example /// "bash -c" -> /bin/bash -c /// "bash" -> /bin/bash static QString ignoreCmdLineToFullCmdAndArgs(const QString& str){ int spaceIdx = str.indexOf(QChar::Space); if(spaceIdx == -1){ return QStandardPaths::findExecutable(str); } QString cmd = str.left(spaceIdx); cmd = QStandardPaths::findExecutable(cmd); if(cmd.isEmpty()) return cmd; // space still there...: return cmd + str.mid(spaceIdx); } /// Adds a given command string from defaults or config file to the respective set. /// @param warnIfNotFound: if false, no warnings are printed if a command is not found. /// It is to prevenet printing warnings for default commands, which are not installed on /// the target system. /// If a command ends with an asterisk, it will be ignored, regardless /// of its arguments. Else the commands are only ignored it the argument match exactly void Settings::addIgnoreCmd(QString cmd, bool warnIfNotFound, const QString & ignoreCmdsSectName){ const QString lineCopy = cmd; // do not simplify as argmument in the config parser! cmd=cmd.simplified(); const QString ignoreCmdsErrPreamble = ignoreCmdsSectName + qtr(": invalid command in line ") + '<'; if(cmd.endsWith('*')){ cmd.remove(cmd.size()-1, 1); cmd=cmd.trimmed(); if(cmd.contains(' ')){ logWarning << ignoreCmdsErrPreamble << lineCopy << "> - " << qtr("The command contains whitespaces. " "Note that arguments are not (yet) supported " "when wildcards are used."); return; } QString fullPath = QStandardPaths::findExecutable(cmd); if(fullPath.isEmpty()){ if(warnIfNotFound){ logWarning << ignoreCmdsErrPreamble << lineCopy << "> - not found:" << cmd; } } else { m_ignoreCmdsRegardlessOfArgs.insert(fullPath.toStdString()); } } else { cmd = ignoreCmdLineToFullCmdAndArgs(cmd); if(cmd.isEmpty()){ if(warnIfNotFound){ logWarning << ignoreCmdsErrPreamble << lineCopy << "> - not found." ; } } else { m_ignoreCmds.insert(cmd.toStdString()); } } } /// @param hiddenPaths: if not null, store hidden paths in the passed tree, instead /// of the returned one. std::shared_ptr Settings::loadPaths(Section_Ptr& section, const QString& keyName, bool eraseSubpaths, const std::unordered_set & defaultValues, PathTree* hiddenPaths){ auto rawPaths = section->getValues >(keyName, defaultValues, false, "\n"); auto tree = std::make_shared(); for(const auto& p : rawPaths){ QString canonicalPath = p; if(canonicalPath.startsWith("$CWD")){ if(m_workingDir.isEmpty()){ logWarning << qtr("section %1: %2: $CWD is set but the working-" "directory could not be determined. Maybe it does " "not exist?") .arg(section->sectionName(), keyName); continue; } canonicalPath.replace("$CWD", m_workingDir); } else if(canonicalPath.startsWith("$HOME")){ canonicalPath.replace("$HOME", m_userHome); } else if(canonicalPath.startsWith("~")) { canonicalPath.replace("~", m_userHome); } canonicalPath = QDir(canonicalPath).canonicalPath(); if(canonicalPath.isEmpty()){ logWarning << qtr("section %1: %2: path does not exist: %3") .arg(section->sectionName(), keyName, p); continue; } PathTree* currentTree =(hiddenPaths != nullptr && canonicalPath.contains("/.")) ? hiddenPaths : tree.get(); auto canoicalPathLight = toStrLight(canonicalPath); // avoid adding needless parent/subpaths if(eraseSubpaths){ if(currentTree->isSubPath(canoicalPathLight)){ logDebug<< keyName << "ignore" << canonicalPath << "because it is a subpath"; continue; } // the new path might be a parent path: // erase its children (if any) auto subPathIt = currentTree->subpathIter(canoicalPathLight); while(subPathIt != currentTree->end() ){ logDebug << keyName << "ignore" << *subPathIt << "because it is a subpath"; subPathIt = currentTree->erase(subPathIt); } } currentTree->insert(canoicalPathLight); } return tree; } /// Until shournal 2.1 the version file was located at .cache, then migrated /// to .config/shournal /// @return QVersionNumer.isNull==true In case of no or an invalid version. static QVersionNumber readLegacyConfigFileVersion(){ const QString path = legacyCfgVersionFilePath(); QFile f(path); if(! f.open(QFile::OpenModeFlag::ReadOnly)){ return {}; } CFlock l(f.handle()); l.lockShared(); auto ver = QVersionNumber::fromString(QTextStream(&f).readLine()); l.unlock(); // unlock explicitly: in case f is removed, it is closed beforehand, // so the fd passed to CFlock is already invalid. if(ver.isNull()){ logWarning << QString("Bad version string in file %1. Deleting it...") .arg(path); f.remove(); } return ver; } /// Remove all paths from excludePaths which are not sub-paths /// of any tree in includePathtrees. Print a warning in this case. static void cleanExcludePaths(const QVector& includePathtrees, std::shared_ptr& excludePaths, const QString& sectionName){ for(auto it=excludePaths->begin(); it != excludePaths->end();){ bool isSubPath = false; for(const PathTree* includePaths : includePathtrees){ if(includePaths->isSubPath(*it)){ isSubPath = true; break; } } if(isSubPath){ ++it; } else { logWarning << qtr("section %1: ignore exclude-path %2 - it is not a sub-path " "of any include-path").arg(sectionName).arg((*it).c_str()); it = excludePaths->erase(it); } } } // static void cleanExcludePaths(const PathTree& includePaths, PathTree& excludePaths, // const QString& sectionName){ // cleanExcludePaths( {&includePaths}, excludePaths, sectionName); // } static void cleanExcludePaths(const std::shared_ptr& includePaths, const PathTree* optionalIncludePaths, std::shared_ptr& excludePaths, const QString& sectionName){ if(optionalIncludePaths != nullptr){ cleanExcludePaths( {includePaths.get(), optionalIncludePaths}, excludePaths, sectionName); } cleanExcludePaths( {includePaths.get()}, excludePaths, sectionName); } bool Settings::loadSections(const QVersionNumber& parsedCfgVersion){ m_cfg.setInitialComments(qtr( "Configuration file for %1. Uncomment lines " "to change defaults. Multi-line-values (e.g. paths) " "are framed by leading and trailing " "triple-quotes ''' .\n" "When loading paths, the following symbols may be " "specified:\n" "$HOME or ~ for your home directory\n" "$CWD for the current working directory\n" "In several sections, the key 'exclude_hidden'\n" "can be set to true - in this case, a " "file event is excluded, if it is below " "*any* hidden directory or is hidden itself. A " "explicitly included hidden file is not affected.\n" "Please do not store custom comments in this file, " "as those are lost each time shournal is updated to " "a new version.").arg(app::SHOURNAL) ); bool updateNeeded = false; updateNeeded |= loadSectWrite(parsedCfgVersion); updateNeeded |= loadSectRead(parsedCfgVersion); loadSectScriptFiles(); loadSectIgnoreCmd(); loadSectMount(); loadSectHash(); return updateNeeded; } bool Settings::loadSectWrite(const QVersionNumber& parsedCfgVersion) { auto sectWriteEvents = m_cfg["File write-events"]; bool updateNeeded = false; sectWriteEvents->setComments(qtr( "Configure, which paths shall be observed for " "*write*-events. Put each desired path into " "a separate line. " "Default is to observe all paths.\n" )); m_wSettings.excludeHidden = sectWriteEvents->getValue("exclude_hidden", true); PathTree* hiddenPaths = (m_wSettings.excludeHidden) ? m_wSettings.includePathsHidden.get() : nullptr; m_wSettings.includePaths = loadPaths( sectWriteEvents, "include_paths", true, {"/"}, hiddenPaths); m_wSettings.excludePaths = loadPaths( sectWriteEvents, "exclude_paths", true, {}); cleanExcludePaths(m_wSettings.includePaths, hiddenPaths, m_wSettings.excludePaths, sectWriteEvents->sectionName()); bool insertMaxEventCount; uint32_t maxEventCount; if(parsedCfgVersion < QVersionNumber{2,4}){ // Backwards compatibility: old versions did not impose // an event limit updateNeeded = true; logDebug << "updating cfg-file to" << QVersionNumber{2,4}.toString(); insertMaxEventCount = true; maxEventCount = 0; } else { insertMaxEventCount = false; maxEventCount = 5000; } m_wSettings.maxEventCount = sectWriteEvents->getValue( "max_event_count", maxEventCount, insertMaxEventCount); m_wSettings.maxEventCount = (m_wSettings.maxEventCount == 0) ? numeric_limits::max() : m_wSettings.maxEventCount; return updateNeeded; } bool Settings::loadSectRead(const QVersionNumber& parsedCfgVersion) { auto sectReadEvents = m_cfg[SECT_READ_NAME]; bool updateNeeded = false; sectReadEvents->setComments(qtr( "Configure, which paths shall be observed for " "read- or exec-events. Put each desired path into " "a separate line. " "Per default read file-events are only logged, " "if you have *also* write permission (assuming other " "read files are not of interest)." )); m_rSettings.enable = sectReadEvents->getValue(SECT_READ_KEY_ENABLE, true); m_rSettings.onlyWritable = sectReadEvents->getValue("only_writable", true); m_rSettings.excludeHidden = sectReadEvents->getValue("exclude_hidden", true); PathTree* hiddenPaths = (m_rSettings.excludeHidden) ? m_rSettings.includePathsHidden.get() : nullptr; m_rSettings.includePaths = loadPaths( sectReadEvents, SECT_READ_KEY_INCLUDE_PATHS, true, {"$HOME"}, hiddenPaths); m_rSettings.excludePaths = loadPaths( sectReadEvents, "exclude_paths", true, {}); cleanExcludePaths(m_rSettings.includePaths, hiddenPaths, m_rSettings.excludePaths, sectReadEvents->sectionName()); bool insertMaxEventCount; uint32_t maxEventCount; if(parsedCfgVersion < QVersionNumber{2,4}){ // Backwards compatibility: older versions did not impose // an event limit updateNeeded = true; logDebug << "updating cfg-file to" << QVersionNumber{0,9}.toString(); insertMaxEventCount = true; maxEventCount = 0; } else { insertMaxEventCount = false; maxEventCount = 5000; } m_rSettings.maxEventCount = sectReadEvents->getValue( "max_event_count", maxEventCount, insertMaxEventCount); m_rSettings.maxEventCount = (m_rSettings.maxEventCount == 0) ? numeric_limits::max() : m_rSettings.maxEventCount; return updateNeeded; } void Settings::loadSectScriptFiles() { auto sectScriptFiles = m_cfg[SECT_SCRIPTS_NAME]; const QString scriptFiles_OnlyWritableKey = "only_writable"; sectScriptFiles->setComments( qtr("Configure what files (scripts), which were *read* " "by the observed command, shall be stored within " "%1's database.\n" "The maximal filesize may have units such as KiB, MiB, etc.. " "You can specify file-extensions or mime-types " "(only with the fanotify-backend) to match desired " "file-types, e.g. sh (without leading dot!) or application/x-shellscript. " "The following rules apply: if both are unset, " "accept all file-types (not recommended), if one of the " "two is unset then only the set one is considered, if both " "are set, at least one of the two has to match for the file to " "be stored. " "Note that finding out the mimetype is a lot more " "computationally expensive than the file-extension-method. %1 " "can list a mimetype for a given file, see also %1 --help." "\n" "Per default only the first N read files matching all given rules " "are saved for each command-sequence (max_count_of_files).\n" "%2: only store a read file, if you have write- (not only read-) " "permission for it.\n" "Storing read files is disabled by default.\n" ).arg(app::SHOURNAL, scriptFiles_OnlyWritableKey)); m_scriptSettings.enable = sectScriptFiles->getValue(SECT_SCRIPTS_ENABLE, false); m_scriptSettings.onlyWritable = sectScriptFiles->getValue(scriptFiles_OnlyWritableKey, true); m_scriptSettings.maxFileSize = sectScriptFiles->getFileSize("max_size", 500*1024) ; m_scriptSettings.maxCountOfFiles = static_cast(sectScriptFiles->getValue( "max_count_of_files", 3)); m_scriptSettings.excludeHidden = sectScriptFiles->getValue("exclude_hidden", true); PathTree* hiddenPaths = (m_scriptSettings.excludeHidden) ? m_scriptSettings.includePathsHidden.get() : nullptr; m_scriptSettings.includeExtensions = sectScriptFiles->getValues( SECT_SCRIPTS_INCLUDE_FILE_EXTENSIONS, {"sh"}, false, "\n"); m_scriptSettings.includeMimetypes = sectScriptFiles->getValues( "include_mime_types", {"application/x-shellscript"}, false, "\n"); m_scriptSettings.includePaths = loadPaths(sectScriptFiles, SECT_SCRIPTS_INCLUDE_PATHS, true, {"/"}, hiddenPaths); m_scriptSettings.excludePaths = loadPaths(sectScriptFiles, "exclude_paths", true, {}); cleanExcludePaths(m_scriptSettings.includePaths, hiddenPaths, m_scriptSettings.excludePaths, sectScriptFiles->sectionName()); // make user configurable? If so, make sure not bigger than sizeof(int)/2... m_scriptSettings.flushToDiskTotalSize = 1024 * 1024 * 10; } void Settings::loadSectIgnoreCmd() { auto sectIgnoreCmd = m_cfg["Ignore-commands"]; m_ignoreCmdsRegardlessOfArgs.clear(); m_ignoreCmds.clear(); const QString sect_ignore_cmds_commands = "commands"; sectIgnoreCmd->setComments(qtr( "Only applies to the shell-integration and the\n" "fanotify backend!\n" "Exclude specific commands from observation. " "The (optional) path to the commands must not contain whitepaces " "(create a symlink and import that PATH, if necessary). " "You can provide arguments so that a given " "command is only excluded, if it is followed " "by exactly the given arguments (order matters). " "Further wildcards (*) are supported, but ONLY " "for commands, so that a command can be excluded " "regardless of its arguments.\n\n" "%1 = '''bash\n" "bash -i\n" "screen\n" "mount*'''\n").arg(sect_ignore_cmds_commands) ); for(const auto & c : defaultIgnoreCmds()){ addIgnoreCmd(c, false, sectIgnoreCmd->sectionName()); } sectIgnoreCmd->setInsertDefaultToComments(false); for(const auto & c : sectIgnoreCmd->getValues(sect_ignore_cmds_commands, QStringList(), false, "\n")) { addIgnoreCmd(c, true, sectIgnoreCmd->sectionName()); } sectIgnoreCmd->setInsertDefaultToComments(true); } void Settings::loadSectMount() { auto sectMount = m_cfg["mounts"]; const QString sect_mount_ignore = "exclude_paths"; sectMount->setComments(qtr( "Only applies to the fanotify backend!" "Ignore sub-mount-paths from observation. " "This is typically only needed, if " "you don't have permissions on some " "mounts and want to supress warnings. " "Pseudo-filesytems like /proc are already excluded. " "Put each absolute path into a separate line.\n" "To ignore mounts for which you don't have access permissions, " "set the respective flag to true.\n" ) ); m_mountIgnoreNoPerm = sectMount->getValue("ignore_no_permission", false); m_mountIgnorePaths = sectMount->getValues(sect_mount_ignore, {}, false, "\n"); std::vector defaultMountIgnorePaths = {"/proc", "/sys", "/run", "/dev/hugepages", "/dev/mqueue", "/dev/pts"}; m_mountIgnorePaths.insert(defaultMountIgnorePaths.begin(), defaultMountIgnorePaths.end()); if(os::getuid() != 0){ m_mountIgnorePaths.insert("/root"); } } void Settings::loadSectHash() { auto sectHash = m_cfg["Hash"]; const QString sect_hash_enable = "enable"; const QString sect_hash_chunksize = "chunksize"; const QString sect_hash_maxCountReads = "max-count-reads"; sectHash->setComments(qtr( "Note: this section includes advanced settings and should not be " "changed at all in most cases and if so, only with a fresh database. " "%1 or %2 should *not* be changed during the lifetime of the database. " "Changing it is not a well tested feature and in any case causes overhead " "for hash-based database-queries."). arg(sect_hash_chunksize, sect_hash_maxCountReads)); m_hashSettings.hashEnable = sectHash->getValue(sect_hash_enable, true, true); // Exclude negative values by using uint m_hashSettings.hashMeta.chunkSize = static_cast( sectHash->getValue(sect_hash_chunksize, 256, true)); m_hashSettings.hashMeta.maxCountOfReads = static_cast( sectHash->getValue(sect_hash_maxCountReads, 3, true)); if(m_hashSettings.hashEnable){ // TODO: also limit maxCountOfReads -> see kernel module if(m_hashSettings.hashMeta.chunkSize < 8 || m_hashSettings.hashMeta.chunkSize > 1024 * 40 || m_hashSettings.hashMeta.maxCountOfReads < 1){ throw ExcCfg(qtr("Invalid hashsettings. Must be:" " 8 >= %1 <= 40KiB and %2 > 1") .arg(sect_hash_chunksize, sect_hash_maxCountReads)); } } } Settings::ReadVersionReturn Settings::readVersion(SafeFileUpdate& verUpd8) { ReadVersionReturn ret; verUpd8.read([&ret, &verUpd8]{ ret.ver = QVersionNumber::fromString(QTextStream(&verUpd8.file()).readLine()); }); ret.verFilePath = verUpd8.file().fileName(); if(ret.ver.isNull() ){ // check legacy version file (migrated...) ret.ver = readLegacyConfigFileVersion(); if(ret.ver.isNull()){ logInfo << qtr("No valid version-file found, although a config-file existed. This " "should only happen during the transition from shournal < 2.1 " "to a version >= 2.1."); ret.ver = app::initialVersion(); } else { ret.verFilePath = legacyCfgVersionFilePath(); } } return ret; } /// If cached cfg-version is newer than our app's version, throw, /// if it is older, migrate sections to new names. /// @return: true, if an update was necessary, else false bool Settings::updateCfgScheme (const QVersionNumber& configSchemeVer, Settings::ReadVersionReturn &readVerResult) { if(readVerResult.ver == configSchemeVer){ return false; } if(readVerResult.ver > configSchemeVer){ throw ExcCfg(qtr("The config-file version is greater than the " "scheme version. This most likely happens " "if running shournal's shell integration while " "shournal was updated. In that case " "simply exit the shell session and start it again. " "Otherwise you might have " "downgraded shournal and need to manually correct " "the version-file at %1. " "Cached version is %2, current scheme version is %3") .arg(readVerResult.verFilePath) .arg(readVerResult.ver.toString()) .arg(configSchemeVer.toString())); } if(readVerResult.ver < QVersionNumber{0,9}){ logDebug << "updating cfg-file to" << QVersionNumber{0,9}.toString(); m_cfg.renameParsedSection("Hash", "Hash for file write-events"); } if(readVerResult.ver < QVersionNumber{2,1}){ logDebug << "updating cfg-file to" << QVersionNumber{2,1}.toString(); // Because of new section [File read-events] for read-events for which // no (script-) files shall be stored, rename old [File read-events]. // Read files now also support hash, so rename the hash-section (again). m_cfg.renameParsedSection("File read-events", "File read-events storage settings"); m_cfg.renameParsedSection("Hash for file write-events", "Hash"); } return true; } /// Store the config to disk. Note that this is only done for new versions, /// that's why the version file is also updated alongside. void Settings::storeCfg (const QVersionNumber& configSchemeVer, SafeFileUpdate &cfgUpd8, SafeFileUpdate &verUpd8) { cfgUpd8.write([this, &cfgUpd8]{ m_cfg.store(cfgUpd8.file()); }); verUpd8.write([&verUpd8, &cfgUpd8, &configSchemeVer]{ QTextStream(&verUpd8.file()) << configSchemeVer.toString(); QFileInfo legacyVersionInfo(legacyCfgVersionFilePath()); if(legacyVersionInfo.exists() && ! legacyVersionInfo.isSymLink()){ // atomically create symlink to new cfg version file (using a temporary // symlink and rename/move that): logInfo << qtr("handle legacy config version file: creating symlink to " "new location..."); auto uuid = make_uuid(); QByteArray cfgPathBytes = cfgUpd8.file().fileName().toLocal8Bit(); QByteArray tmpSymlinkLocation = legacyVersionInfo.absoluteDir().filePath(uuid).toLocal8Bit(); os::symlink(cfgPathBytes.constData(), tmpSymlinkLocation.constData()); os::rename(tmpSymlinkLocation, legacyVersionInfo.absoluteFilePath().toLocal8Bit()); } }); } /// Parse or create the configuration file at the system's config path /// (please perform QCoreApplication::setApplicationName() before). /// Another file at config dir provides the version. If the config file version is greater /// than out scheme version, throw, if smaller, update the version and possibly /// the config-file-scheme as well. Scheme updates /// for sections work by directly renaming the sections in loadSections() and /// by moving the old to new sections in updateCfgScheme(). Note that /// this also works in case of "redundant" scheme updates, where over multiple /// scheme versions the same section is renamed multiple times. Intermediate /// sections are created as necessary and potentially dropped/renamed again /// by subsequent scheme updates. /// @throws ExcCfg void Settings::load() { const auto cfgPath = cfgFilepath(); const QString cfgDir(splitAbsPath(cfgPath).first); QDir dir; if(! dir.mkpath(cfgDir) ){ throw QExcIo(qtr("Failed to create configuration directory at %1") .arg(cfgDir) ); } SafeFileUpdate cfgUpd8(cfgPath); bool cfgFileExisted = cfgUpd8.read([this, &cfgUpd8]{ m_cfg.parse(cfgUpd8.file()); }); // Until shournal v3.2 the config version was always set to the application version. // This required a synchronized update of all machines sharing the same config dir. // Therefore, only update the config version if a scheme update is necessary. const auto configSchemeVer = QVersionNumber{3, 2}; auto parsedCfgVersion = configSchemeVer; SafeFileUpdate verUpd8(pathJoinFilename(cfgDir, QString(".config-version"))); bool cfgVersionNeedsUpdate = false; if(cfgFileExisted){ // do we need a version update? auto readVerRet = readVersion(verUpd8); cfgVersionNeedsUpdate = updateCfgScheme(configSchemeVer, readVerRet); parsedCfgVersion = readVerRet.ver; } try { cfgVersionNeedsUpdate |= loadSections(parsedCfgVersion); auto notReadKeys = m_cfg.generateNonReadSectionKeyPairs(); if(! notReadKeys.isEmpty()){ throw ExcCfg(qtr("Unexpected key in section [%1] - '%2'") .arg(notReadKeys.first().first, *notReadKeys.first().second.begin())); } // Only write configuration to disk, if there was no such file // or we are using a new scheme for the first time if(! cfgFileExisted || cfgVersionNeedsUpdate){ logDebug << "about to update config at" << cfgPath; storeCfg(configSchemeVer, cfgUpd8, verUpd8); } } catch(ExcCfg & ex) { ex.setDescrip(ex.descrip() + qtr(". The config file resides at %1").arg(cfgPath)); throw; } m_settingsLoaded = true; } /// Select which backend to choose based on local/global /// config files and availability (search $PATH). /// @return app::SHOURNAL_RUN, app::SHOURNAL_RUN_FANOTIFY or an /// empty string if not found. QString Settings::chooseShournalRunBackend() { auto appname = QCoreApplication::applicationName(); const QString localPath(cfgAppDir() + "/backend"); const QString globalPath = "/etc/shournal.d/backend"; QString selectedPath; if(QFile::exists(localPath)){ selectedPath = localPath; } else if(QFile::exists(globalPath)){ selectedPath = globalPath; } if(! selectedPath.isEmpty()){ // load backend from file QFileThrow backendCfgFile(selectedPath); backendCfgFile.open(QFile::OpenModeFlag::ReadOnly); QTextStream s(&backendCfgFile); auto backendStr = s.readLine(); if(backendStr == "ko") { return app::SHOURNAL_RUN; } if(backendStr == "fanotify") { return app::SHOURNAL_RUN_FANOTIFY; } logWarning << qtr("Invalid backend %1 at file %2 - " "supported options: [fanotify, ko]. " "Using defaults...").arg(backendStr, selectedPath); } if( ! QStandardPaths::findExecutable(app::SHOURNAL_RUN).isEmpty()) return app::SHOURNAL_RUN; if( ! QStandardPaths::findExecutable(app::SHOURNAL_RUN_FANOTIFY).isEmpty()) return app::SHOURNAL_RUN_FANOTIFY; return {}; } const Settings::StrLightSet &Settings::getMountIgnorePaths() { assert(m_settingsLoaded); return m_mountIgnorePaths; } bool Settings::getMountIgnoreNoPerm() const { return m_mountIgnoreNoPerm; } const Settings::StringSet &Settings::ignoreCmds() { return m_ignoreCmds; } const Settings::StringSet &Settings::ignoreCmdsRegardslessOfArgs() { return m_ignoreCmdsRegardlessOfArgs; } const Settings::WriteFileSettings &Settings::writeFileSettings() const { return m_wSettings; } const Settings::ReadFileSettings &Settings::readFileSettings() const { return m_rSettings; } const Settings::ScriptFileSettings &Settings::readEventScriptSettings() const { return m_scriptSettings; } const Settings::HashSettings &Settings::hashSettings() const { return m_hashSettings; } ================================================ FILE: src/common/settings.h ================================================ #pragma once #include #include #include #include #include "hashmeta.h" #include "pathtree.h" #include "cfg.h" #include "qfilethrow.h" using std::make_shared; class SafeFileUpdate; class Settings { public: typedef std::unordered_set StringSet; typedef std::unordered_set StrLightSet; typedef std::unordered_set MimeSet; static Settings & instance(); struct HashSettings { HashMeta hashMeta; bool hashEnable{}; }; struct WriteFileSettings { WriteFileSettings() : includePaths(make_shared()), includePathsHidden(make_shared()), excludePaths(make_shared()) {} std::shared_ptr includePaths; std::shared_ptr includePathsHidden; std::shared_ptr excludePaths; bool excludeHidden {true}; uint64_t maxEventCount{std::numeric_limits::max()}; Q_DISABLE_COPY(WriteFileSettings) DISABLE_MOVE(WriteFileSettings) }; struct ReadFileSettings { ReadFileSettings() : includePaths(make_shared()), includePathsHidden(make_shared()), excludePaths(make_shared()) {} bool enable {true}; std::shared_ptr includePaths; std::shared_ptr includePathsHidden; std::shared_ptr excludePaths; bool onlyWritable {true}; bool excludeHidden {true}; uint64_t maxEventCount{std::numeric_limits::max()}; Q_DISABLE_COPY(ReadFileSettings) DISABLE_MOVE(ReadFileSettings) }; /// Holds settings for read files which shall be stored to disk /// ( probably mostly scripts or similar files). struct ScriptFileSettings { ScriptFileSettings() : includePaths(make_shared()), includePathsHidden(make_shared()), excludePaths(make_shared()) {} // store read files to disk, if... bool enable {false}; // .. enabled bool onlyWritable {true}; // .. user has write permission bool excludeHidden {true}; // .. it is not hidden std::shared_ptr includePaths; // .. it is equal to or below an include path std::shared_ptr includePathsHidden; // .. see above std::shared_ptr excludePaths; // .. it is not equal to or below an exclude path StrLightSet includeExtensions; // .. file extension, mimetype matches ( it's MimeSet includeMimetypes; // more complicated than that) qint64 maxFileSize {500*1024}; // .. it's not bigger than this size uint maxCountOfFiles {3}; // .. we have not already collected that many read files int flushToDiskTotalSize {1024*1024*10}; // read files (scripts) are cached in memory. If their total size is // greater than that, flush to disk (database) Q_DISABLE_COPY(ScriptFileSettings) DISABLE_MOVE(ScriptFileSettings) }; public: void setUserCfgDir(const QString& p); void setUserDataDir(const QString& p); void load(); QString chooseShournalRunBackend(); const HashSettings& hashSettings() const; const WriteFileSettings& writeFileSettings() const; const ReadFileSettings& readFileSettings() const; const ScriptFileSettings& readEventScriptSettings() const; QString cfgAppDir(); QString cfgFilepath(); QString dataDir(); const QStringList& defaultIgnoreCmds(); const StringSet& ignoreCmds(); const StringSet& ignoreCmdsRegardslessOfArgs(); const StrLightSet &getMountIgnorePaths(); bool getMountIgnoreNoPerm() const; public: ~Settings() = default; Q_DISABLE_COPY(Settings) DISABLE_MOVE(Settings) public: static const char* SECT_READ_NAME; static const char* SECT_READ_KEY_ENABLE; static const char* SECT_READ_KEY_INCLUDE_PATHS; static const char* SECT_SCRIPTS_NAME; static const char* SECT_SCRIPTS_ENABLE; static const char* SECT_SCRIPTS_INCLUDE_PATHS; static const char* SECT_SCRIPTS_INCLUDE_FILE_EXTENSIONS; private: struct ReadVersionReturn { QVersionNumber ver; QString verFilePath; }; Settings() = default; void addIgnoreCmd(QString cmd, bool warnIfNotFound, const QString & ignoreCmdsSectName); bool loadSections(const QVersionNumber& parsedCfgVersion); bool loadSectWrite(const QVersionNumber& parsedCfgVersion); bool loadSectRead(const QVersionNumber& parsedCfgVersion); void loadSectScriptFiles(); void loadSectIgnoreCmd(); void loadSectMount(); void loadSectHash(); ReadVersionReturn readVersion(SafeFileUpdate &verUpd8); bool updateCfgScheme(const QVersionNumber&, ReadVersionReturn&); void storeCfg(const QVersionNumber &configSchemeVer, SafeFileUpdate& cfgUpd8, SafeFileUpdate& verUpd8); std::shared_ptr loadPaths(qsimplecfg::Cfg::Section_Ptr& section, const QString& keyName, bool eraseSubpaths, const std::unordered_set & defaultValues, PathTree* hiddenPaths=nullptr); qsimplecfg::Cfg m_cfg; HashSettings m_hashSettings; WriteFileSettings m_wSettings; ReadFileSettings m_rSettings; ScriptFileSettings m_scriptSettings; StrLightSet m_mountIgnorePaths; bool m_mountIgnoreNoPerm {false}; bool m_settingsLoaded {false}; StringSet m_ignoreCmds; StringSet m_ignoreCmdsRegardlessOfArgs; const QString m_userHome { QDir::homePath() }; const QString m_workingDir { QDir::currentPath() }; QString m_userCfgDir; QString m_userDataDir; private: // unit testing... friend class FileEventHandlerTest; friend class IntegrationTestShell; friend class GeneralTest; }; ================================================ FILE: src/common/shournal_run_common.cpp ================================================ #include "shournal_run_common.h" #include "app.h" #include "conversions.h" #include "qoutstream.h" #include "translation.h" void shournal_run_common::print_summary(uint64_t n_wEvents, uint64_t n_rEvents, uint64_t n_lostEvents, uint64_t n_storedEvents, uint64_t targetFileSize){ QErr() << qtr("=== %1 summary ===\n" "number of write-events: %2\n" "number of read-events: %3\n" "number of lost events: %4\n" "number of stored read files: %5\n" "size of tmp-file: %6\n") .arg(app::CURRENT_NAME) .arg(n_wEvents) .arg(n_rEvents) .arg(n_lostEvents) .arg(n_storedEvents) .arg(Conversions().bytesToHuman(targetFileSize)); } QOptArg shournal_run_common::mkarg_cfgdir() { return QOptArg("", "cfg-dir", qtr("Override the path to shournal's configuration directory.")); } QOptArg shournal_run_common::mkarg_datadir() { return QOptArg("", "data-dir", qtr("Override the path to shournal's data directory.")); } ================================================ FILE: src/common/shournal_run_common.h ================================================ #pragma once #include #include "util.h" #include "qoptarg.h" namespace shournal_run_common { void print_summary(uint64_t n_wEvents, uint64_t n_rEvents, uint64_t n_lostEvents, uint64_t n_storedEvents, uint64_t targetFileSize); QOptArg mkarg_cfgdir(); QOptArg mkarg_datadir(); } // namespace shournal_run_common ================================================ FILE: src/common/socket_message.cpp ================================================ #include "socket_message.h" const char* socket_message::socketMsgToStr(E_SocketMsg msg){ switch (msg) { case E_SocketMsg::SETUP_DONE: return "SETUP_DONE"; case E_SocketMsg::SETUP_FAIL: return "SETUP_FAIL"; case E_SocketMsg::CLEAR_EVENTS: return "CLEAR_EVENTS"; case E_SocketMsg::COMMAND: return "COMMAND"; case E_SocketMsg::RETURN_VALUE: return "RETURN_VALUE"; case E_SocketMsg::EMPTY: return "EMPTY"; case E_SocketMsg::LOG_MESSAGE: return "LOG_MESSAGE"; case E_SocketMsg::CMD_START_DATETIME: return "CMD_START_DATETIME"; case E_SocketMsg::ENUM_END: return "ENUM_END"; } return "UNHANDLED ENUM CASE"; } ================================================ FILE: src/common/socket_message.h ================================================ #pragma once namespace socket_message { /// Messages send from shell observation to shournal process or vice versa enum class E_SocketMsg { SETUP_DONE, SETUP_FAIL, CLEAR_EVENTS, COMMAND, RETURN_VALUE, EMPTY, LOG_MESSAGE, CMD_START_DATETIME, ENUM_END }; const char* socketMsgToStr(E_SocketMsg msg); } ================================================ FILE: src/common/stdiocpp.cpp ================================================ #include #include #include #include #include #include "stdiocpp.h" #include "util.h" #include "translation.h" #include "os.h" #include "osutil.h" stdiocpp::QExcStdio::QExcStdio (QString text, const FILE *file, bool collectErrno, bool collectStacktrace) : QExcCommon(text, false) { m_descrip += (file == nullptr) ? "" : " - flags: " + QString::number(file->_flags); if(collectErrno){ m_descrip += " (" + QString::number(errno) + "): " + translation::strerror_l(errno); } if(collectStacktrace){ appendStacktraceToDescrip(); } } /// Create an unnamed temp-file respecting /// env-variable TMPDIR (other than the canonical /// tmpfile(3)). FILE *stdiocpp::tmpfile(int o_flags __attribute__ ((unused))) { int fd = -1; try { fd = osutil::unnamed_tmp(); return stdiocpp::fdopen(fd, "w+"); } catch (const os::ExcOs& ex) { throw QExcStdio(ex.what(), nullptr, true); } catch (const QExcStdio&) { if(fd != -1){ close(fd); } throw ; } } FILE *stdiocpp::fopen(const char *pathname, const char *mode) { FILE* f = ::fopen(pathname, mode); if(f == nullptr ){ throw QExcStdio(QString("Cannot open %1 with mode %2: ") .arg(pathname, mode), nullptr, true); } return f; } FILE *stdiocpp::fdopen(int fd, const char *mode){ FILE* f = ::fdopen(fd, mode); if(f == nullptr ){ throw QExcStdio(QString("Cannot open fd with mode %2: ") .arg(mode), nullptr, true); } return f; } void stdiocpp::fclose(FILE *stream) { if(::fclose(stream) != 0){ throw QExcStdio("fclose failed: ", nullptr, true); } } int stdiocpp::fgetc_unlocked(FILE *stream) { return ::fgetc_unlocked(stream); } size_t stdiocpp::fwrite_unlocked(const void *ptr, size_t size, size_t n_items, FILE *stream) { size_t items_written = ::fwrite_unlocked(ptr , size, n_items, stream); if( items_written != n_items){ throw QExcStdio(QString("fwrite_unlocked failed (only %1 of %2 items written): ") .arg(items_written).arg(n_items), stream); } return items_written; } void stdiocpp::fflush(FILE *stream) { if(::fflush(stream) != 0){ throw QExcStdio("fflush failed: ", nullptr, true); } } size_t stdiocpp::fread_unlocked(void *ptr, size_t size, size_t n, FILE *stream) { return ::fread_unlocked(ptr, size, n, stream); } int stdiocpp::fseek(FILE *stream, long offset, int whence) { int new_offset = ::fseek(stream, offset, whence); if(new_offset == -1){ throw QExcStdio("fseek failed: ", stream, true); } return new_offset; } long int stdiocpp::ftell(FILE *stream){ long int offset = ::ftell(stream); if(offset == -1){ throw QExcStdio("ftell failed: ", stream, true); } return offset; } /// Warning: not threadsafe. /// stdio.h does not provide such a functionality, so we must take care /// that buffer is flushed before using the raw OS-ftruncate void stdiocpp::ftruncate_unlocked(FILE *stream) { stdiocpp::fflush(stream); stdiocpp::fseek(stream, 0, SEEK_SET); if(::ftruncate(fileno(stream), 0) == -1) { throw QExcStdio("POSIX ftruncate failed", stream, true); } } ================================================ FILE: src/common/stdiocpp.h ================================================ #pragma once #include "exccommon.h" /// Very thin wrappers around some stdio functions /// with exceptions, etc. namespace stdiocpp { class QExcStdio : public QExcCommon { public: explicit QExcStdio(QString text, const FILE* file, bool collectErrno=false, bool collectStacktrace=true); }; FILE* tmpfile(int o_flags=0); FILE *fopen(const char *pathname, const char *mode); FILE *fdopen(int fd, const char *mode); void fclose(FILE *stream); int fgetc_unlocked(FILE *stream); size_t fwrite_unlocked(const void *ptr, size_t size, size_t n_items, FILE *stream); void fflush(FILE *stream); size_t fread_unlocked(void *ptr, size_t size, size_t n, FILE *stream); int fseek(FILE *stream, long offset, int whence); long int ftell(FILE *stream); void ftruncate_unlocked(FILE* stream); } // namespace stdiocpp ================================================ FILE: src/common/stupidinject.cpp ================================================ #include "stupidinject.h" void StupidInject::addInjection(const StupidInject::Action& action) { m_actions.push_back(action); } void StupidInject::addInjection(const char *trigger, const char *replacement) { Action act; act.trigger = trigger; act.func = [replacement](QTextStream &out){ out << replacement; }; m_actions.push_back(act); } void StupidInject::addInjection(const char *trigger, const std::function& func) { Action act; act.trigger = trigger; act.func = func; m_actions.push_back(act); } void StupidInject::stream(const char *input, QTextStream &out) { const char* lastBegin = input; for(const auto& action : m_actions){ const char *triggerInInput = strstr(lastBegin, action.trigger.constData()); if(triggerInInput == nullptr){ throw EqcInjectTriggerNotFound("Trigger not found: " + action.trigger); } // length of the string that has yet to be written before injecting auto lastInputLenght = triggerInInput - lastBegin; const auto lastInput = QByteArray::fromRawData(lastBegin, int(lastInputLenght)); out << lastInput; action.func(out); lastBegin = triggerInInput + action.trigger.size(); } // the rest still needs to be written out << lastBegin; } ================================================ FILE: src/common/stupidinject.h ================================================ #pragma once #include #include #include #include "exccommon.h" class EqcInjectTriggerNotFound : public QExcCommon { public: using QExcCommon::QExcCommon; }; /// "Inject" arbitrary content into a text-stream. The actions /// have to be added in the order of their later occurence /// within the input-stream. class StupidInject { public: struct Action { std::function< void(QTextStream &out)> func; QByteArray trigger; }; void addInjection(const Action& action); void addInjection(const char* trigger, const char* replacement); void addInjection(const char* trigger,const std::function< void(QTextStream &out)>& func); void stream(const char* input, QTextStream& out); private: QVector m_actions; }; ================================================ FILE: src/common/subprocess.cpp ================================================ #include #include #include #include #include #include #include #include #include #include "subprocess.h" #include "os.h" #include "osutil.h" #include "excos.h" #include "util.h" #include "cleanupresource.h" #include "fdentries.h" using osutil::closeVerbose; enum class LaunchMsgType { PID, EXCEPTION, ENUM_END }; struct LaunchMsg{ LaunchMsgType msgType; int errorNumber; //errno pid_t pid; }; static_assert (sizeof (LaunchMsg) <= PIPE_BUF, "LaunchMsg is too big..." ); static_assert (std::is_pod(), ""); namespace { /// @throws ExcOs bool readMsg(int fd, LaunchMsg* msg){ ssize_t readN = os::read(fd, msg, sizeof (LaunchMsg)); if(readN == 0){ return false; } if(readN != sizeof (LaunchMsg)){ // should never happen throw os::ExcOs("Failed to launch external process, " "received invalid message from child.", EINVAL); } return true; } [[noreturn]] void throwFailedToLaunchEx(char *const argv[], const LaunchMsg & msg, const std::string& descrip=""){ std::string desc_; if(descrip.empty()){ desc_ = descrip; } else { desc_ = descrip + " - "; } throw os::ExcOs("Failed to launch external process <" + argvToStr(argv) + "> -" + desc_, msg.errorNumber); } /// convert to null-terminated vector which will be passed as argv. std::vector toPointerVect(const subprocess::Args_t& args){ std::vector pointerVec(args.size() + 1 ); // + 1 because of terminating NULL for(unsigned i = 0; i < args.size() ; ++i) { pointerVec[i] = const_cast(args[i].c_str()); } pointerVec.back() = nullptr; return pointerVec; } } // namespace subprocess::Subprocess::Subprocess() : m_lastPid(std::numeric_limits::max()), m_asRealUser(false), m_forwardAllFds(false), m_lastCallWasDetached(false), m_environ(nullptr), m_inNewSid(false), m_waitForSetup(true), m_lastCallWaitedForSetup(false) {} void subprocess::Subprocess::call(char *const argv[], bool forwardStdin, bool forwardStdout, bool forwardStderr) { this->call(argv[0], argv, forwardStdin, forwardStdout, forwardStderr); } void subprocess::Subprocess::call(const Args_t &args, bool forwardStdin, bool forwardStdout, bool forwardStderr) { this->call(toPointerVect(args).data(), forwardStdin, forwardStdout, forwardStderr); } /// Call provided program after fork. Note that per default all file descriptors /// except stdin, stdout and stderr are closed. /// @throw ExcOs void subprocess::Subprocess::call(const char *filename, char * const argv[], bool forwardStdin, bool forwardStdout, bool forwardStderr) { doCall(filename, argv, forwardStdin, forwardStdout, forwardStderr, false); } /// Call provided program after double-fork (daemonize). /// Waits, until grandchild-process. Note that per default all file descriptors /// except stdout and stderr are closed. /// @throw ExcOs void subprocess::Subprocess::callDetached(const char *filename, char * const argv[], bool forwardStdin, bool forwardStdout, bool forwardStderr) { doCall(filename, argv, forwardStdin, forwardStdout, forwardStderr, true); } void subprocess::Subprocess::callDetached(char * const argv[], bool forwardStdin, bool forwardStdout, bool forwardStderr) { callDetached(argv[0], argv, forwardStdin, forwardStdout, forwardStderr); } void subprocess::Subprocess::callDetached(const Args_t &args, bool forwardStdin, bool forwardStdout, bool forwardStderr) { this->callDetached(toPointerVect(args).data(), forwardStdin, forwardStdout, forwardStderr); } void subprocess::Subprocess:: doCall(const char *filename, char * const argv[], bool forwardStdin, bool forwardStdout, bool forwardStderr, bool detached) { m_lastCallWasDetached = detached; m_lastCallWaitedForSetup = m_waitForSetup; if(m_waitForSetup){ doCallWaitForSetup(filename, argv, forwardStdin, forwardStdout, forwardStderr, detached); } else { os::Pipes_t dummyPipe {-1, -1}; doFork(filename, argv, forwardStdin, forwardStdout, forwardStderr, detached, dummyPipe); } } /// Create a pipe and wait until the child-process has closed it's /// write-end, either by calling execve or on error. /// In case of a detached call, the pid of the grandchild-process /// is also send via the pipe and made available via m_lastPid. void subprocess::Subprocess:: doCallWaitForSetup(const char *filename, char * const argv[], bool forwardStdin, bool forwardStdout, bool forwardStderr, bool detached){ auto startPipe = os::pipe( O_CLOEXEC | O_DIRECT ); auto closeStartRead = finally([&startPipe] { closeVerbose(startPipe[0]); }); auto closeStartWrite = finally([&startPipe] { closeVerbose(startPipe[1]); }); doFork(filename, argv, forwardStdin, forwardStdout, forwardStderr, detached, startPipe); closeVerbose(startPipe[1]); closeStartWrite.setEnabled(false); LaunchMsg msg; if(! detached){ if(!readMsg(startPipe[0], &msg)){ // no error return; } assert(msg.msgType == LaunchMsgType::EXCEPTION); throwFailedToLaunchEx(argv, msg); } // if detached we need the grandchild-pid if(!readMsg(startPipe[0], &msg)){ // first message *must* be pid throwFailedToLaunchEx(argv, msg, "Missing pid reply from grandchild"); } switch (msg.msgType) { case LaunchMsgType::PID: // normal case: pid of grandchild m_lastPid = msg.pid; break; case LaunchMsgType::EXCEPTION: throwFailedToLaunchEx(argv, msg); default: assert(false); throwFailedToLaunchEx(argv, msg, " Bad response from grandchild: " + std::to_string(int(msg.msgType))); } // second reply (if any) must be an exception always if(!readMsg(startPipe[0], &msg)){ return; } assert(msg.msgType == LaunchMsgType::EXCEPTION); throwFailedToLaunchEx(argv, msg); } void subprocess::Subprocess:: doFork(const char *filename, char * const argv[], bool forwardStdin, bool forwardStdout, bool forwardStderr, bool detached, os::Pipes_t &startPipe){ m_lastPid = os::fork(); if(m_lastPid == 0){ if(m_inNewSid) os::setsid(); if(detached){ // child: fork again and exit try { pid_t pid2 = os::fork(); if(pid2 == 0){ handleChild(filename, argv, startPipe, forwardStdin, forwardStdout, forwardStderr); } } catch (const os::ExcOs& ex){ // should never happen std::cerr << __func__ << ": " << ex.what() << "\n"; exit(1); } exit(0); } else { handleChild(filename, argv, startPipe, forwardStdin, forwardStdout, forwardStderr); } } } /// Wait for the subprocess to finish. Does *not* work /// for detached process /// @return the exit value of the process /// @throws ExcOs, ExcProcessExitNotNormal int subprocess::Subprocess::waitFinish() { if(m_lastCallWasDetached){ throw os::ExcOs("Attempted to wait for child process, " "although last call was ", 0); } int child_status = 1; os::waitpid (m_lastPid, &child_status) ; return child_status; } void subprocess::Subprocess::setAsRealUser(bool val) { m_asRealUser = val; } /// Per default all file-descriptors are closed, except /// for "nomal" call: /// stdin, stdout and stderr. /// for detached call: /// stdout and stderr /// With this method you override the default. void subprocess::Subprocess::setForwardFdsOnExec(const std::unordered_set &forwardFds) { m_forwardFds = forwardFds; } void subprocess::Subprocess::setForwardAllFds(bool val) { m_forwardAllFds = val; } void subprocess::Subprocess::setInNewSid(bool val) { m_inNewSid = val; } /// Wait for child-process-setup on call or callDetached. void subprocess::Subprocess::setWaitForSetup(bool waitForSetup) { m_waitForSetup = waitForSetup; } void subprocess::Subprocess::closeAllButForwardFds(os::Pipes_t &startPipe) { // startpipe fds have O_CLOEXEC set, if exec fails, the respond is sent via // them, so do not close here. for(const int fd : osutil::FdEntries()){ if(fd <= 2){ // stdin, -out and -err are handeled separately continue; } if(m_forwardFds.find(fd) == m_forwardFds.end() && fd != startPipe[0] && fd != startPipe[1]) // not in white-list, close closeVerbose(fd); } } void subprocess::Subprocess::handleChild(const char *filename, char *const argv[], os::Pipes_t &startPipe, bool forwardStdin, bool forwardStdout, bool forwardStderr) { try { if(m_callbackAsChild) m_callbackAsChild(); if(m_asRealUser){ os::setgid(os::getgid()); os::setuid(os::getuid()); } if(startPipe[0] != -1 && m_lastCallWasDetached){ LaunchMsg msg{}; msg.msgType = LaunchMsgType::PID; msg.pid = getpid(); os::write(startPipe[1], &msg, sizeof (LaunchMsg)); } if(! m_forwardAllFds){ if(! forwardStdin) closeVerbose(STDIN_FILENO); if(! forwardStdout) closeVerbose(STDOUT_FILENO); if(! forwardStderr) closeVerbose(STDERR_FILENO); closeAllButForwardFds(startPipe); } os::exec(filename, argv, m_environ); } catch (const os::ExcOs& ex) { if(startPipe[0] == -1){ std::cerr << "Failed to launch subprocess: " << ex.what() << "\n"; exit(ex.errorNumber()); } LaunchMsg msg{}; msg.msgType = LaunchMsgType::EXCEPTION; msg.errorNumber = ex.errorNumber(); try { os::write(startPipe[1], &msg, sizeof (LaunchMsg)); } catch (const os::ExcOs& ex) { // should never happen std::cerr << __func__ << ": " << ex.what() << "\n"; } exit(ex.errorNumber()); } } void subprocess::Subprocess::setEnviron(char **env) { m_environ = env; } void subprocess::Subprocess::setCallbackAsChild (const std::function &callbackAsChild) { m_callbackAsChild = callbackAsChild; } /// In case of callDetached the grandchild-PID is returned ( /// but only if we waited for setup, else the pid is invalid!). pid_t subprocess::Subprocess::lastPid() const { if(m_lastCallWasDetached && ! m_lastCallWaitedForSetup){ throw QExcProgramming("m_lastCallWasDetached && ! m_lastCallWaitedForSetup"); } return m_lastPid; } ================================================ FILE: src/common/subprocess.h ================================================ #pragma once #include #include "os.h" namespace subprocess { typedef std::vector Args_t; /// Call external programs via fork and exec /// and wait for it to finish later class Subprocess { public: Subprocess(); void call(char *const argv[], bool forwardStdin=true, bool forwardStdout=true, bool forwardStderr=true); void call(const Args_t &args, bool forwardStdin=true, bool forwardStdout=true, bool forwardStderr=true); void call(const char *filename, char * const argv[], bool forwardStdin=true, bool forwardStdout=true, bool forwardStderr=true); void callDetached(char *const argv[], bool forwardStdin=false, bool forwardStdout=true, bool forwardStderr=true); void callDetached(const char *filename, char *const argv[], bool forwardStdin=false, bool forwardStdout=true, bool forwardStderr=true); void callDetached(const Args_t &args, bool forwardStdin=false, bool forwardStdout=true, bool forwardStderr=true); int waitFinish(); void setAsRealUser(bool val); void setForwardFdsOnExec(const std::unordered_set& forwardFds); void setForwardAllFds(bool val); void setInNewSid(bool val); void setWaitForSetup(bool waitForSetup); pid_t lastPid() const; void setEnviron(char **env); void setCallbackAsChild(const std::function &callbackAsChild); private: void closeAllButForwardFds(os::Pipes_t &startPipe); [[noreturn]] void handleChild(const char *filename, char * const argv[], os::Pipes_t & startPipe, bool forwardStdin, bool forwardStdout, bool forwardStderr); void doCall(const char *filename, char * const argv[], bool forwardStdin, bool forwardStdout, bool forwardStderr, bool detached); void doCallWaitForSetup(const char *filename, char * const argv[], bool forwardStdin, bool forwardStdout, bool forwardStderr, bool detached); void doFork(const char *filename, char * const argv[], bool forwardStdin, bool forwardStdout, bool forwardStderr, bool detached, os::Pipes_t &startPipe); pid_t m_lastPid; bool m_asRealUser; std::unordered_set m_forwardFds; bool m_forwardAllFds; bool m_lastCallWasDetached; char** m_environ; bool m_inNewSid; bool m_waitForSetup; bool m_lastCallWaitedForSetup; std::function< void()> m_callbackAsChild; }; } // namespace subprocess ================================================ FILE: src/common/user_kernerl.h ================================================ /* Common helpers which can be used * from within user- and linux-kernel- * space */ #pragma once #ifdef __KERNEL__ #ifdef DEBUG #include // #define kuassert WARN_ON #define kuassert(condition) WARN_ON(!(condition)) #else #define kuassert(condition) #endif #else #include #define kuassert assert #ifndef likely #ifdef __GNUC__ #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) #else #define likely(x) (x) #define unlikely(x) (x) #endif #endif #endif ================================================ FILE: src/common/util/CMakeLists.txt ================================================ add_library(lib_util cleanupresource.h compat.h compareoperator.cpp conversions.cpp cpp_exit.cpp exccommon.cpp qoutstream.cpp qformattedstream.cpp strlight.cpp strlight_util.cpp staticinitializer.h sys_ioprio.h translation.cpp util.cpp util_performance.cpp ) target_link_libraries(lib_util PUBLIC Qt5::Core uuid ) ================================================ FILE: src/common/util/cleanupresource.h ================================================ #pragma once #include #include "util.h" namespace private_namesapce { template struct CleanupResource { CleanupResource(F f, bool enable) : m_cleanF{f}, m_enabled(enable) {} ~CleanupResource() { // Do not throw from destructor try { if(m_enabled){ m_cleanF(); } } catch (const std::exception& ex ) { std::cerr << ex.what() << "\n"; } } void setEnabled(bool val){ m_enabled = val; } public: Q_DISABLE_COPY(CleanupResource) DEFAULT_MOVE(CleanupResource) private: F m_cleanF; bool m_enabled; }; } // namespace private_namesapce template private_namesapce::CleanupResource finally(F f, bool enable=true) __attribute__ ((warn_unused_result)); /// Perform a final action before leaving the block: /// Usage: /// char* buf = new char; /// auto deleter = finally([buf] {delete buf; }); template private_namesapce::CleanupResource finally(F f, bool enable){ return private_namesapce::CleanupResource(f, enable); } ================================================ FILE: src/common/util/compareoperator.cpp ================================================ #include #include #include "compareoperator.h" #include "exccommon.h" namespace { const QHash& termEnumHash(){ static const QHash termEnumHash = { {"-gt", E_CompareOperator::GT}, {"-ge", E_CompareOperator::GE}, {"-lt", E_CompareOperator::LT}, {"-le", E_CompareOperator::LE}, {"-eq", E_CompareOperator::EQ}, {"-ne", E_CompareOperator::NE}, {"-like", E_CompareOperator::LIKE}, {"-between", E_CompareOperator::BETWEEN} }; return termEnumHash; } } // namespace CompareOperator::CompareOperator(E_CompareOperator op) : m_operator(op) {} /// Transform one of the commandline-passed operators into the enum and store it. bool CompareOperator::fromTerminal(const QString &val){ if(val.isEmpty()){ return false; } assert(val.at(0) == '-'); const auto enumIt = termEnumHash().find(val); if(enumIt == termEnumHash().constEnd()){ return false; } m_operator = enumIt.value(); return true; } QString CompareOperator::asSql() const { QString sqlOperator; switch (m_operator) { case E_CompareOperator::GT: sqlOperator = ">"; break; case E_CompareOperator::GE: sqlOperator = ">="; break; case E_CompareOperator::LT: sqlOperator = "<"; break; case E_CompareOperator::LE: sqlOperator = "<="; break; case E_CompareOperator::EQ: sqlOperator = "="; break; case E_CompareOperator::NE: sqlOperator = "!="; break; case E_CompareOperator::LIKE: sqlOperator = " LIKE "; break; case E_CompareOperator::BETWEEN: sqlOperator = " BETWEEN "; break; case E_CompareOperator::ENUM_END: throw QExcProgramming("E_CompareOperator::ENUM_END"); } return sqlOperator; } QString CompareOperator::asTerminal() const { QString sqlOperator; switch (m_operator) { case E_CompareOperator::GT: sqlOperator = "-gt"; break; case E_CompareOperator::GE: sqlOperator = "-ge"; break; case E_CompareOperator::LT: sqlOperator = "-lt"; break; case E_CompareOperator::LE: sqlOperator = "-le"; break; case E_CompareOperator::EQ: sqlOperator = "-eq"; break; case E_CompareOperator::NE: sqlOperator = "-ne"; break; case E_CompareOperator::LIKE: sqlOperator = "-like"; break; case E_CompareOperator::BETWEEN: sqlOperator = "-between"; break; case E_CompareOperator::ENUM_END: throw QExcProgramming("E_CompareOperator::ENUM_END"); } return sqlOperator; } E_CompareOperator CompareOperator::asEnum() const { return m_operator; } ================================================ FILE: src/common/util/compareoperator.h ================================================ #pragma once #include #include #include enum class E_CompareOperator { GT,GE,LT,LE,EQ,NE,LIKE,BETWEEN,ENUM_END }; /// The most important sql-operators which are used /// in this program as user-input (LE for less or equal), /// enum and sql operator (<=) class CompareOperator { public: CompareOperator() = default; CompareOperator(E_CompareOperator op); bool fromTerminal(const QString& val); QString asSql() const; QString asTerminal() const; E_CompareOperator asEnum() const; private: E_CompareOperator m_operator{E_CompareOperator::EQ}; }; ================================================ FILE: src/common/util/compat.h ================================================ #pragma once #include #include #include #include #include #if QT_VERSION < QT_VERSION_CHECK(5, 14, 0) namespace Qt { using SplitBehavior = QString::SplitBehavior; const SplitBehavior SkipEmptyParts = SplitBehavior::SkipEmptyParts; const auto endl = ::endl; inline QDateTime datetimeFromDate(const QDate& date){ return QDateTime(date); } } #else namespace Qt { inline QDateTime datetimeFromDate(const QDate& date){ return date.startOfDay(); } } #endif #if QT_VERSION < QT_VERSION_CHECK(5, 13, 0) namespace Qt { inline bool resourceIsCompressed(QResource &r){ return r.isCompressed(); } } #else namespace Qt { inline bool resourceIsCompressed(QResource &r){ return r.compressionAlgorithm() != QResource::NoCompression; } } #endif ================================================ FILE: src/common/util/conversions.cpp ================================================ #include #include #include #include #include "conversions.h" #include "util.h" static QHash validTimeUnitHash(){ static const QHash units { {qtr("y"), 'y'}, // Year {qtr("m"), 'm'}, // month {qtr("d"), 'd'}, // day {qtr("h"), 'h'}, // hour {qtr("min"), 'M'}, // minute {qtr("s"), 's'}, // second }; return units; } ExcConversion::ExcConversion(const QString & text) : QExcCommon(text, false) {} /// @returns by comma separated list of valid relative time units with description /// (y: year, m: month, ...). const QString &Conversions::relativeDateTimeUnitDescriptions() { static const auto s = qtr("y: year, m: month, d: day, h: hour, min: minute, s: second"); return s; } /// Transform user supplied byte-sizes ("3KiB", "2 MiB ", etc.) to int. /// @throws ExcConversion qint64 Conversions::bytesFromHuman(QString str) { str = str.simplified(); str.replace( " ", "" ); const QString errPreamble(qtr("Failed to convert bytesize '%1' - ").arg(str)); if(str.isEmpty()){ throw ExcConversion(errPreamble + qtr("it is empty.")); } if(str[str.size() - 1].isDigit()){ // assuming bytes size qint64 bytes; if(! qVariantTo(str, &bytes)){ throw ExcConversion(errPreamble + qtr("it appears to be not an integer " "although no unit was given.")); } return bytes; } int unitIdx = str.size() - 2; for(; unitIdx >= 0; unitIdx--){ if(str[unitIdx].isDigit()){ unitIdx++; break; } } if(unitIdx == -1){ throw ExcConversion(errPreamble + qtr("no digit was given")); } const QString unit = str.mid(unitIdx); const QString val = str.left(unitIdx); double bytesFloat; if(! qVariantTo(val, &bytesFloat)){ throw ExcConversion(errPreamble + qtr("conversion from string '%1' to float failed").arg(val)); } if(bytesFloat < 0){ bytesFloat += -1; } static const std::unordered_set validUnitSet { "k", "kb", "kib", "m", "mb", "mib", "g", "gb", "gib", "t", "tb", "tib" }; if(validUnitSet.find(unit.toLower()) == validUnitSet.end()){ const QString validUnits(qtr("valid units include 'no unit', " "K (Kib), M (MiB), G (GiB) and T (TiB) " "but '%1' was given").arg(unit)); throw ExcConversion(errPreamble + validUnits); } switch (unit[0].toLower().toLatin1()) { case 'k': bytesFloat *= 1024.0; break; case 'm': bytesFloat *= 1024.0*1024; break; case 'g': bytesFloat *= 1024.0*1024*1024; break; case 't': bytesFloat *= 1024.0*1024*1024*1024; break; default: assert(false); } return static_cast(bytesFloat); } /// size to human readbale string (Kib, Mib, etc....) QString Conversions::bytesToHuman(const qint64 bytes) { float s = bytes; static const QStringList list({"KiB", "MiB", "GiB", "TiB"}); QStringListIterator i(list); QString unit("bytes"); if(s <= 1024.0f){ return QString().setNum(s,'f',0)+" "+unit; } do { unit = i.next(); s /= 1024.0f; } while(s >= 1024.0f && i.hasNext()); return QString().setNum(s,'f',2)+" "+unit; } /// @param subtractIt: if true, the parsed date is subtracted from current one, /// else it is added. /// @throws ExcConversion QDateTime Conversions::relativeDateTimeFromHuman(const QString &str, bool subtractIt) { static const QRegularExpression re(R"((\d+)(.+))"); QRegularExpressionMatch match = re.match(str); const QString errPreamble(qtr("Failed to convert relative date(time) '%1' - ").arg(str)); if (! match.hasMatch()) { throw ExcConversion(errPreamble + qtr("It must be a digit followed by a timespec.")); } // must always succeed, otherwise regex would be broken int number = match.captured(1).toInt(); const QString parsedTimeSpec = match.captured(2).trimmed(); const auto & validUnits = validTimeUnitHash(); auto matchedUnitIt = validUnits.find(parsedTimeSpec); if(matchedUnitIt == validUnits.end()){ // don't use auto here: older version of qt do not support QList::join... QStringList units = validUnits.keys(); throw ExcConversion(errPreamble + qtr("%1 is not a valid timespec. Those are %2") .arg(units.join(","))); } if(subtractIt){ // go back in time: number = -number; } auto now = QDateTime::currentDateTime(); switch (matchedUnitIt.value()) { case 'y': return now.addYears(number); case 'm': return now.addMonths(number); case 'd': return now.addDays(number); case 'h': return now.addSecs(number*3600); case 'M': return now.addSecs(number*60); case 's': return now.addSecs(number); default: assert(false); } return {}; } const QString &Conversions::dateIsoFormatWithMilliseconds() { static const QString f{"yyyy-MM-ddTHH:mm:ss.zzz"}; return f; } ================================================ FILE: src/common/util/conversions.h ================================================ #pragma once #include #include #include "exccommon.h" class ExcConversion : public QExcCommon { public: ExcConversion(const QString &text); }; /// Parse datatypes from human input, display them human readable class Conversions { public: static const QString& relativeDateTimeUnitDescriptions(); qint64 bytesFromHuman(QString str); QString bytesToHuman(qint64 bytes); QDateTime relativeDateTimeFromHuman(const QString& str, bool subtractIt); static const QString& dateIsoFormatWithMilliseconds(); }; ================================================ FILE: src/common/util/cpp_exit.cpp ================================================ #include "cpp_exit.h" /// To allow for destructor calling of local objects, throw instead of /// calling exit void cpp_exit(int ret) { throw ExcCppExit(ret); } ================================================ FILE: src/common/util/cpp_exit.h ================================================ #pragma once class ExcCppExit { public: ExcCppExit(int ret) : m_ret(ret){} int ret() const { return m_ret; } private: int m_ret; }; [[noreturn]] void cpp_exit(int ret); ================================================ FILE: src/common/util/exccommon.cpp ================================================ #include #include #include "exccommon.h" #include "util.h" #include "translation.h" ExcCommon::ExcCommon(std::string text) : m_descrip(std::move(text)) {} const char *ExcCommon::what() const noexcept { return m_descrip.c_str(); } std::string &ExcCommon::descrip() { return m_descrip; } QExcCommon::QExcCommon(QString text, bool collectStacktrace) : m_descrip(std::move(text)) { if(collectStacktrace){ appendStacktraceToDescrip(); } } const char *QExcCommon::what() const noexcept { m_local8Bit = m_descrip.toLocal8Bit(); return m_local8Bit.constData(); } QString QExcCommon::descrip() const { return m_descrip; } void QExcCommon::setDescrip(const QString &descrip) { m_descrip = descrip; m_local8Bit = descrip.toLocal8Bit(); } void QExcCommon::appendStacktraceToDescrip() { const auto st = generate_trace_string(); m_descrip += "\n" + QString::fromStdString(st); } QExcIllegalArgument::QExcIllegalArgument(const QString &text) : QExcCommon (text) { } QExcProgramming::QExcProgramming(const QString &text) : QExcCommon (text) { } QExcIo::QExcIo(QString text, bool collectStacktrace) : QExcCommon("", false) { m_errorNumber = errno; if(errno != 0){ text += " (" + QString::number(errno) + "): " + translation::strerror_l(errno); } this->setDescrip(text); if(collectStacktrace){ appendStacktraceToDescrip(); } } int QExcIo::errorNumber() const { return m_errorNumber; } ================================================ FILE: src/common/util/exccommon.h ================================================ #pragma once #include #include #include class ExcCommon : public std::exception { public: explicit ExcCommon(std::string text); const char *what () const noexcept override; std::string & descrip(); protected: std::string m_descrip; }; class QExcCommon : public std::exception { public: explicit QExcCommon(QString text, bool collectStacktrace=true); const char *what () const noexcept override; QString descrip() const; void setDescrip(const QString &descrip); protected: void appendStacktraceToDescrip(); QString m_descrip; private: mutable QByteArray m_local8Bit; }; class QExcIllegalArgument : public QExcCommon { public: QExcIllegalArgument(const QString & text); }; /// Thrown in case of a detected bug^^ class QExcProgramming : public QExcCommon { public: QExcProgramming(const QString & text); }; class QExcIo : public QExcCommon { public: explicit QExcIo(QString text, bool collectStacktrace=true); int errorNumber() const; private: int m_errorNumber; }; ================================================ FILE: src/common/util/nullable_value.h ================================================ #pragma once #include #include "exccommon.h" class QExcNullDeref : public QExcCommon { public: using QExcCommon::QExcCommon; }; template class NullableValue { public: typedef T value_type; NullableValue() : m_isNull(true){} NullableValue(const T& t) : m_isNull(false), m_value(t){} const T& value() const { if(m_isNull){ throw QExcNullDeref("Tried to obtain value while it is set to null"); } return m_value; } void setValue(const T& val){ m_value = val; m_isNull = false; } bool isNull() const { return m_isNull; } void setNull(){ m_isNull = true; } bool operator==(const NullableValue& rhs) const { if(isNull()){ return rhs.isNull(); } // we are not null if( rhs.isNull()){ return false; } // other is also not null. // compare vals return value() == rhs.value(); } bool operator!=(const NullableValue& rhs) const { return!(operator==(rhs)); } NullableValue& operator=(const T& val) { setValue(val); return *this; } protected: bool m_isNull; T m_value; }; template bool operator==(const T& lhs, const NullableValue& rhs) { // comparing to value, so it cannot be null if(rhs.isNull()){ return false; } return lhs == rhs.value(); } template bool operator==(const NullableValue& lhs, const T& rhs) { return rhs == lhs.value(); } typedef NullableValue HashValue; Q_DECLARE_METATYPE(HashValue) ================================================ FILE: src/common/util/qformattedstream.cpp ================================================ #include "qformattedstream.h" #include "util.h" QFormattedStream::QFormattedStream(QString *string, QIODevice::OpenMode openMode) : m_textStream(string, openMode) { this->initCommon(); } QFormattedStream::QFormattedStream(FILE *fileHandle, QIODevice::OpenMode openMode) : m_textStream(fileHandle, openMode) { this->initCommon(); } QFormattedStream::QFormattedStream(QIODevice *device) : m_textStream(device) { this->initCommon(); } QFormattedStream::QFormattedStream(QByteArray *array, QIODevice::OpenMode openMode) : m_textStream(array, openMode) { this->initCommon(); } QFormattedStream::QFormattedStream(const QByteArray &array, QIODevice::OpenMode openMode) : m_textStream(array, openMode) { this->initCommon(); } void QFormattedStream::initCommon() { m_colNChars = 0; m_maxLineWidth = std::numeric_limits::max(); m_autoSepStreamChunks = true; m_streamChunkSep = ' '; } QFormattedStream &QFormattedStream::operator<<(const QString &str) { return (*this)<<(QStringRef(&str)); } QFormattedStream &QFormattedStream::operator<<(const QStringRef &str) { int wordStartIdx = -1; for(int i=0; i < str.size(); i++){ const QChar& c = str.at(i); if(m_colNChars == 0){ writeLineStart(); } if(c.isSpace()){ if(wordStartIdx != -1){ QStringRef word = str.mid(wordStartIdx, i - wordStartIdx); handleWordEnd(word); wordStartIdx = -1; } writeSpace(c); } else { if(wordStartIdx == -1){ wordStartIdx = i; } } } // Final word might not be written yet. // Note that word-breaks spreading over multiple strings (multiples calls of operator<<) // are not correctly handled, if autoSepWords is false. if(wordStartIdx != -1){ QStringRef word = str.mid(wordStartIdx); handleWordEnd(word); } // if at beginning of line, words are already separated, // so don't write space in that case if(m_autoSepStreamChunks && m_colNChars != 0){ writeSpace(m_streamChunkSep); } return *this; } /// Each line in the stream will start with the given string (also applies /// to the first line) void QFormattedStream::setLineStart(const QString &lineStart) { m_lineStart = lineStart; } /// Latest after that many characters a word-conscious line-break is /// performed. If a word is longer than maxLineWidth, it will be splittet, /// so it fits into the minimum possible number of lines. void QFormattedStream::setMaxLineWidth(int maxLineWidth) { m_maxLineWidth = maxLineWidth; } void QFormattedStream::writeLineStart() { m_textStream << m_lineStart; m_colNChars = m_lineStart.size(); } void QFormattedStream::handleWordEnd(const QStringRef &word) { // Check if it fits in current line. if(m_colNChars + word.size() > m_maxLineWidth){ if(word.size() + m_lineStart.size() <= m_maxLineWidth){ // write it to next line m_textStream << "\n"; writeLineStart(); m_textStream << word; m_colNChars += word.size(); } else { writeLongWord(word); } } else { m_textStream << word; m_colNChars += word.size(); } } /// If a word is too large to fit into one line, /// print as much into each line as possible. void QFormattedStream::writeLongWord(const QStringRef &word) { // dont use stl-style iterator for compatability with qt-version < 5.4 for(int i=0; i < word.size(); i++){ const QChar c = word.at(i); if(m_colNChars >= m_maxLineWidth){ m_textStream << "\n"; writeLineStart(); } m_textStream << c; m_colNChars++; } } /// If we are at end of desired width, /// always write line feed. void QFormattedStream::writeSpace(const QChar &c) { if(c == QChar::LineFeed || m_colNChars >= m_maxLineWidth){ m_textStream << "\n"; m_colNChars = 0; } else { m_textStream << c; m_colNChars++; } } const QString &QFormattedStream::lineStart() const { return m_lineStart; } int QFormattedStream::maxLineWidth() const { return m_maxLineWidth; } QChar QFormattedStream::streamChunkSep() const { return m_streamChunkSep; } void QFormattedStream::setStreamChunkSep(const QChar &streamChunkSep) { m_streamChunkSep = streamChunkSep; } ================================================ FILE: src/common/util/qformattedstream.h ================================================ #pragma once #include /// Write strings to text-streams with a custom formatting. /// Each line can be set to start with an arbitrary string. /// If maxLineWidth is set, split a string word-aware once /// a line becomes too long. /// Strings received during multiple <<-operator-calls are /// automatically separated by whitespace (or the desired char), if not already separated /// by a character for which QChar::isSpace() returns true. /// Note: Avoid using the tab-character as its width is controlled by the terminal. class QFormattedStream { public: QFormattedStream(QString *string, QIODevice::OpenMode openMode = QIODevice::ReadWrite); QFormattedStream(FILE *fileHandle, QIODevice::OpenMode openMode = QIODevice::ReadWrite); QFormattedStream(QIODevice *device); QFormattedStream(QByteArray *array, QIODevice::OpenMode openMode = QIODevice::ReadWrite); QFormattedStream(const QByteArray &array, QIODevice::OpenMode openMode = QIODevice::ReadOnly); QFormattedStream& operator<<(const QString& str); QFormattedStream& operator<<(const QStringRef& str); void setLineStart(const QString &lineStart); void setMaxLineWidth(int maxLineWidth); void setStreamChunkSep(const QChar &streamChunkSep); const QString& lineStart() const; int maxLineWidth() const; QChar streamChunkSep() const; private: void initCommon(); void writeLineStart(); void handleWordEnd(const QStringRef &word); void writeLongWord(const QStringRef &word); void writeSpace(const QChar& c); QTextStream m_textStream; QString m_lineStart; int m_colNChars; // number of written characters in current line int m_maxLineWidth; bool m_autoSepStreamChunks; QChar m_streamChunkSep; }; ================================================ FILE: src/common/util/qoutstream.cpp ================================================ #include #include "qoutstream.h" #include "compat.h" QOut::QOut() : m_textStream(stdout) { } QOut::~QOut() { m_textStream.flush(); } QErr::QErr() : m_textStream(stderr) { } QErr::~QErr() { m_textStream.flush(); } std::function QIErr::s_preambleCallback = []() { return ""; }; QIErr::QIErr() : m_ts(stderr) { if(s_preambleCallback){ m_ts << s_preambleCallback(); } } QIErr::~QIErr() { m_ts << Qt::endl; } void QIErr::setPreambleCallback(const std::function &f){ s_preambleCallback = f; } ================================================ FILE: src/common/util/qoutstream.h ================================================ #pragma once #include #include /// Print QString's and other compatible types /// to stdout (flush on destructor). class QOut { public: QOut(); ~QOut(); /// See QTextStream::operator<<(const char*) - ISO-8859-1 encoding is assumed. /// We use UTF-8 everywhere. QOut& operator<<(const char* str){ return *this << QString::fromUtf8(str); } template QOut& operator<<(const T& t) { m_textStream << t; return *this; } private: QTextStream m_textStream; }; /// Print QString's and other compatible types /// to stderr (flush on destructor). class QErr { public: QErr(); ~QErr(); QErr& operator<<(const char* str){ return *this << QString::fromUtf8(str); } template QErr& operator<<(const T& t) { m_textStream << t; return *this; } private: QTextStream m_textStream; }; /// Informative QErr. /// Wrap stderr and add a custom preamble before every stream start. /// Print a custom message in the constructor. /// Auto-separate words by whitespace. /// In destructor, add 'newline '\n' and flush (endl) /// Use it like: /// QICerr::setPreambleCallback([]() { return QCoreApplication::applicationName() + ": "; }); /// QICerr() << "Foo" << "bar"; class QIErr { public: QIErr(); ~QIErr(); QIErr& operator<<(const char* str){ return *this << QString::fromUtf8(str); } template QIErr& operator<<(const T& t) { if(m_WrittenTo){ // auto whitespace m_ts << ' '; } else { m_WrittenTo = true; } m_ts << t; return *this; } static void setPreambleCallback(const std::function& f); private: bool m_WrittenTo {false}; QTextStream m_ts; static std::function s_preambleCallback; }; ================================================ FILE: src/common/util/staticinitializer.h ================================================ #pragma once #include "util.h" /// Call an arbitrary function in the constructor, which can /// be used for static initialization class StaticInitializer { public: template StaticInitializer(Lambda f){ f(); } public: ~StaticInitializer() = default; public: Q_DISABLE_COPY(StaticInitializer) DEFAULT_MOVE(StaticInitializer) }; ================================================ FILE: src/common/util/strlight.cpp ================================================ #include #include "strlight.h" /// Set interal buffer and size. Only allowed /// if *this was constructed via the default constructor. void StrLight::setRawData(const char *buf, StrLight::size_type n){ assert( ! m_weOwnBuf || m_buf == nullptr ); // we won't change buf, promised (; char* b = const_cast(buf); m_buf = b; m_size = n; } void StrLight::setRawSize(StrLight::size_type n){ assert( ! m_weOwnBuf || m_buf == nullptr ); m_size = n; } /// Allocate count chars and initialize with ch StrLight::StrLight(StrLight::size_type count, char ch) { allocatePlusX(count); memset(m_buf, ch, count); setSizeInternal(count); } /// Allocates memory, creates copy of nullterminated cstring. /// @param cstring *must not be null*. StrLight::StrLight(const char *cstring) : StrLight(cstring, (cstring == nullptr) ? 0 : strlen(cstring)) {} /// Allocates memory, creates copy of cstring StrLight::StrLight(const char *cstring, StrLight::size_type size) { if(cstring == nullptr){ throw QExcProgramming("cstring == nullptr"); } allocatePlusX(size); memcpy(m_buf, cstring, size); setSizeInternal(size); } StrLight::~StrLight(){ if(m_weOwnBuf && m_buf != nullptr){ delete[] m_buf; } } /// Warning: if buffer is non-null, copying is only allowed if /// buf is managed internally (owned by us). The rationale /// is e.g. to no accidantially store a StrLight with /// external buffer in a container. On the other hand /// it could be supported to copy the external buffer instead, /// however, a raw buffer is probably set for performance reasons /// so avoid implicit copying (see also deepCopy) StrLight::StrLight(const StrLight &other) { if(other.m_buf == nullptr){ // Nothing to copy return; } if(! other.m_weOwnBuf){ throw QExcProgramming("Copy constructor called for " "externally managed buffer"); } // copy allocatePlusX(other.m_size); memcpy(m_buf, other.constData(), other.m_size); setSizeInternal(other.m_size); } /// Explicit function to also allow for /// easy copying of StrLight's with externally /// managed buffer. StrLight StrLight::deepCopy() const { if(m_buf == nullptr){ return StrLight(); } // copy the data: works for internal // and external managed buffer. // Do not call copy-constructor here! StrLight str(m_buf, m_size); return str; } StrLight &StrLight::operator=(StrLight other) { swap(*this, other); return *this; } StrLight &StrLight::operator=(char c) { this->resize(1); *m_buf = c; return *this; } StrLight &StrLight::operator+=(const StrLight &rhs){ append(rhs.constData(), rhs.size()); return *this; } StrLight &StrLight::operator+=(const char rhs){ char buf[1]; buf[0] = rhs; append(buf, 1); return *this; } void StrLight::append(const char *str, StrLight::size_type n){ auto oldsize = m_size; this->resize( m_size + n); memcpy(&m_buf[oldsize], str, n); assert(m_buf[m_size] == '\0'); // should be done during resize } const char& StrLight::operator[](StrLight::size_type idx) const{ assert(idx < m_size); return m_buf[idx]; } /// move constructor StrLight::StrLight(StrLight &&other) : StrLight() { swap(*this, other); } char StrLight::back() const { assert(m_size > 0); return m_buf[m_size - 1]; } /// @return true, if *this object owns /// the buffer. Else, the buffer was passed /// from outside. bool StrLight::bufIsManagedByThis() const { return m_weOwnBuf; } StrLight::size_type StrLight::find(const char *s, StrLight::size_type pos) const { assert(m_buf != nullptr); assert(pos < m_size); const char* haystackStart = &m_buf[pos]; const char* match = strstr(haystackStart, s); if(match == nullptr) { return StrLight::npos; } return match - m_buf; } StrLight::size_type StrLight::find(const StrLight &s, StrLight::size_type pos) const { return StrLight::find(s.constData(), pos); } /// Find the last occurence of ch in str. /// @return the found index int StrLight::lastIndexOf(char ch) const { for(ssize_type i=m_size - 1; i >= 0; i--){ assert(m_buf != nullptr); if(m_buf[i] == ch){ return int(i); } } return -1; } /// See QbyteArray::left StrLight StrLight::left(int len) const { StrLight s(m_buf, std::min(len, m_size)); return s; } /// See QbyteArray::mid StrLight StrLight::mid(int pos) const { StrLight s(&m_buf[pos], m_size - pos); return s; } bool StrLight::empty() const { return m_size == 0; } StrLight::size_type StrLight::capacity() const { return m_capacity; } /// Warning: only allowed, if we manage the non-null buffer ourselves. /// If it is null, create a new buffer. void StrLight::resize(StrLight::size_type n){ reserve(n); setSizeInternal(n); } void StrLight::reserve(StrLight::size_type n) { if(n < m_capacity){ // capacity nonzero -> buf cannot be null assert(m_buf != nullptr); assert(m_weOwnBuf); return; } if(m_buf == nullptr){ // no external buffer is set: allocate our own allocatePlusX(n); } else { assert(m_weOwnBuf); realloc(n + 64); } } const char *StrLight::constData() const { return m_buf; } char *StrLight::data() { return m_buf; } const char *StrLight::c_str() const { return constData(); } /// Warning: only allowed, if not null and not empty. /// Returns pointer to the final char const char *StrLight::constDataEnd() const { assert(m_buf != nullptr); assert(! this->empty()); return m_buf + m_size - 1; } StrLight::size_type StrLight::size() const { return m_size; } void StrLight::pop_back(){ assert(m_size > 0); this->resize(m_size - 1); } /////////// Private /////////// void StrLight::realloc(StrLight::size_type newCapacity) { assert(m_buf != nullptr); assert(m_weOwnBuf); char* newArr = new char[newCapacity]; memcpy(newArr, m_buf, m_size); delete[] m_buf; m_buf = newArr; m_capacity = newCapacity; m_weOwnBuf = true; } /// allocate a little more than needed void StrLight::allocatePlusX(StrLight::size_type approxNewCapacity) { assert(m_buf == nullptr); // Do not change - other functions rely on this approxNewCapacity += 256; m_buf = new char[approxNewCapacity]; m_capacity = approxNewCapacity; m_weOwnBuf = true; } /// setting buffersize only allowed if we manage it void StrLight::setSizeInternal(StrLight::size_type n) { assert(m_buf != nullptr); assert(m_weOwnBuf); m_size = n; // otherwise the following would be illegal: assert(m_capacity > m_size); m_buf[n] = '\0'; } void swap(StrLight &first, StrLight &second) { using std::swap; swap(first.m_weOwnBuf, second.m_weOwnBuf); swap(first.m_buf, second.m_buf); swap(first.m_size, second.m_size); swap(first.m_capacity, second.m_capacity); } /////////// General /////////// uint qHash(const StrLight &key, uint seed) { if (key.size() == 0) return seed; else return qHashBits(key.constData(), key.size(), seed); } bool operator==(const StrLight &lhs, const StrLight &rhs) { if(lhs.size() != rhs.size()){ return false; } if(lhs.size() == 0){ // both empty return true; } return memcmp(lhs.constData(), rhs.constData(), lhs.size()) == 0; } bool operator==(const StrLight &lhs, const char &c) { return lhs.size() == 1 && *lhs.constData() == c; } bool operator!=(const StrLight &lhs, const char &c) { return !(lhs == c); } QDebug operator<<(QDebug debug, const StrLight &c) { QDebugStateSaver saver(debug); debug.nospace() << QByteArray::fromRawData(c.constData(), int(c.size())); return debug; } const StrLight operator+(const StrLight &s1, const StrLight &s2) { StrLight res; res.reserve(s1.size() + s2.size()); res.append(s1.constData(), s1.size()); res.append(s2.constData(), s2.size()); return res; } const StrLight operator+(const StrLight &s1, const char &c) { StrLight res; res.reserve(s1.size() + 1); res.append(s1.constData(), s1.size()); res += c; return res; } ================================================ FILE: src/common/util/strlight.h ================================================ #pragma once #include #include #include #include "exccommon.h" /// Yet another String class which aims to perform very fast /// in critical sections. For example resizing does not initialize memory /// (other than std::string). Further a "raw"-mode is supported /// (setRawData, setRawSize), where the caller is responsible /// for the buffer. Note that the raw mode is only allowed, /// if StrLight was default-constructed. If Strlight manages /// the memory (bufIsManagedByThis), setRaw* is prohibited. /// By using a raw buffer the user clearly shows the intention /// to care for performance, so in this mode the copy-constructor /// throws! Use the explicit deepCopy in those cases. class StrLight { public: typedef size_t size_type; typedef ssize_t ssize_type; static const size_type npos = static_cast(-1); StrLight() = default; void setRawData(const char* buf, size_type n); void setRawSize(size_type n); StrLight(size_type count, char ch); StrLight(const char* cstring); StrLight(const char* cstring, size_type size); ~StrLight(); StrLight(const StrLight& other); StrLight deepCopy() const; StrLight(StrLight&& other); StrLight& operator=(StrLight other); StrLight& operator=(char c); StrLight& operator+=(const StrLight& rhs); StrLight& operator+=(const char rhs); void append(const char* str, size_type n); const char& operator[](size_type idx) const; char back() const; bool bufIsManagedByThis() const; size_type find(const char* s, size_type pos = 0) const; size_type find(const StrLight& s, size_type pos = 0) const; int lastIndexOf(char ch) const; StrLight left(int len) const; StrLight mid(int pos) const; bool empty() const; size_type capacity() const; size_type size() const; void pop_back(); void resize(size_type n); void reserve(size_type n); const char *constData() const; char *data(); const char *c_str() const; const char* constDataEnd() const; private: void realloc(size_type newCapacity); void allocatePlusX(size_type approxNewCapacity); void setSizeInternal(size_type n); bool m_weOwnBuf {false}; char* m_buf {nullptr}; size_type m_size {0}; size_type m_capacity {0}; public: friend void swap(StrLight& first, StrLight& second); }; uint qHash(const StrLight &key, uint seed = 0); namespace std { template<> struct hash { std::size_t operator()(const StrLight& s) const { return qHash(s); } }; } const StrLight operator+(const StrLight &s1, const StrLight &s2); const StrLight operator+(const StrLight &s1, const char &c); bool operator==(const StrLight &lhs, const StrLight &rhs); bool operator==(const StrLight &lhs, const char &c); bool operator!=(const StrLight &lhs, const char &c); QDebug operator<<(QDebug debug, const StrLight &c); ================================================ FILE: src/common/util/strlight_util.cpp ================================================ #include #include "strlight_util.h" /// Get a str-reference to the file-extension in the canonical /// filename src (no trailing slashes, etc). dest must be /// a raw Buffer! void strlight_util::findFileExtension_raw(const StrLight &src, StrLight &dest) { if(src.size() < 3 || src.back() == '.'){ // smallest possible filname with suffx is x.y -> 3 chars // Last char==dot means no extension. dest.setRawSize(0); return; } assert(src.back() != '/'); const char* srcEnd = src.constDataEnd(); // size - 2 or pEnd - 1, because a final dot is already excluded above for(const char* str = srcEnd - 1; str >= src.constData(); str-- ){ if(*str == '/'){ // nothing found break; } if(*str == '.'){ const char* extStart = str + 1; dest.setRawData(extStart, srcEnd - extStart + 1); return; } } // No file extension found dest.setRawSize(0); return; } ================================================ FILE: src/common/util/strlight_util.h ================================================ #pragma once #include "strlight.h" namespace strlight_util { void findFileExtension_raw(const StrLight& src, StrLight& dest); } ================================================ FILE: src/common/util/sys_ioprio.h ================================================ /* As of glib 2.28 no wrapper exists for this syscall. * The definitions were copied from linux-4.19 include/linux/ioprio.h. * The same is also done in * ionice from util-linux 4.19 (github.com/karelzak/util-linux) * */ #pragma once #define IOPRIO_CLASS_SHIFT (13) #define IOPRIO_PRIO_MASK ((1UL << IOPRIO_CLASS_SHIFT) - 1) #define IOPRIO_PRIO_CLASS(mask) ((mask) >> IOPRIO_CLASS_SHIFT) #define IOPRIO_PRIO_DATA(mask) ((mask) & IOPRIO_PRIO_MASK) #define IOPRIO_PRIO_VALUE(class, data) (((class) << IOPRIO_CLASS_SHIFT) | data) enum { IOPRIO_CLASS_NONE, IOPRIO_CLASS_RT, IOPRIO_CLASS_BE, IOPRIO_CLASS_IDLE, }; enum { IOPRIO_WHO_PROCESS = 1, IOPRIO_WHO_PGRP, IOPRIO_WHO_USER, }; ================================================ FILE: src/common/util/translation.cpp ================================================ #include #include #include "translation.h" #include "util.h" static locale_t g_locale = nullptr; bool translation::init() { if(g_locale == nullptr){ g_locale = newlocale(LC_CTYPE_MASK|LC_NUMERIC_MASK|LC_TIME_MASK| LC_COLLATE_MASK|LC_MONETARY_MASK|LC_MESSAGES_MASK, "",locale_t(nullptr)); } return g_locale != locale_t(nullptr); } char *translation::strerror_l(int errorNumber) { if (g_locale == locale_t(nullptr)) { return ::strerror(errorNumber); } return ::strerror_l(errorNumber, g_locale); } translation::TrSnippets& translation::TrSnippets::instance() { static TrSnippets s_instance; return s_instance; } ================================================ FILE: src/common/util/translation.h ================================================ #pragma once #include #include #include "util.h" namespace translation { bool init(); char* strerror_l(int errorNumber=errno); class TrSnippets { public: const QString enable {qtr("enable")}; const QString shournalShellIntegration {qtr("shournal shell-integration")}; const QString shournalRestore {qtr("shournal-restore")}; static TrSnippets &instance(); public: Q_DISABLE_COPY(TrSnippets) ~TrSnippets() = default; private: TrSnippets() = default; }; } // namespace translation ================================================ FILE: src/common/util/util.cpp ================================================ #include #include #include #include #include #include "util.h" #include "nullable_value.h" #include "os.h" /// stream.readLineInto added in qt 5.5, be backwards compatible... bool readLineInto(QTextStream& stream, QString *line, qint64 maxlen){ #if QT_VERSION < QT_VERSION_CHECK(5, 5, 0) QString str = stream.readLine(maxlen); if(line != nullptr){ *line = str; } return ! str.isNull(); #else return stream.readLineInto(line, maxlen); #endif } #if QT_VERSION < QT_VERSION_CHECK(5, 6, 0) QTextStream& operator<<(QTextStream& stream, const QStringRef &string){ for(int i=0; i < string.size(); i++){ (stream) << string.at(i); } return stream; } #endif /// Initialize essential components which are used all over shournal, including /// unit-tests. bool shournal_common_init() { return QMetaType::registerConverter( [](const QString& str){ return str.toStdString(); }) && QMetaType::registerConverter( [](const std::string& str){ return QString::fromStdString(str); }) && QMetaType::registerConverter( [](const QString& str){ QByteArray b = str.toUtf8(); return StrLight(b.constData(), b.size()); }) && QMetaType::registerConverter( [](const StrLight& str){ return QString::fromUtf8(str.constData(), int(str.size())); }) && QMetaType::registerConverter( [](const HashValue& val){ return ((val.isNull()) ? QString() : QString::number(val.value())); }) && QMetaType::registerConverter( [](const QString& val){ return ((val.isEmpty()) ? HashValue() : HashValue(qVariantTo_throw(val))); }) ; } QDebug &operator<<(QDebug &out, const std::string &str) { out << str.c_str(); return out; } StrLight toStrLight(const QString &str){ return StrLight(str.toUtf8()) ; } void bytesCombine(std::string &){} /// Find out, if fullstring ends with ending bool hasEnding(const std::string &fullString, const std::string &ending) { if (fullString.length() >= ending.length()) { return (0 == fullString.compare (fullString.length() - ending.length(), ending.length(), ending)); } return false; } /// Same as QFileInfo::absoluteFilePath but additionaly /// strips a trailing slash, if any, except the path is only /// root / QString absPath(const QString &path) { if(path == "/"){ return path; } QFileInfo inf(path); QString abs = inf.absoluteFilePath(); if(abs.endsWith("/")){ abs = abs.left(abs.length() - 1); } return abs; } /// Equivalent of python's uuid1: 'import uuid; print(uuid.uuid1())' /// @param madeSafe: pass a bool to know afterwards, whether /// the uuid was created in a safe way. QByteArray make_uuid(bool *madeSafe){ QByteArray uuid; uuid.resize(sizeof (uuid_t)); int ret = uuid_generate_time_safe( static_cast(static_cast(uuid.data()))) ; if(madeSafe != nullptr){ *madeSafe = ret == 0; } return uuid; } const char* strDataAccess(const char* str){ return str; } char* strDataAccess(char* str){ return str; } char *strDataAccess(std::string &str){ return &str[0]; } const char *strDataAccess(const std::string &str){ return str.c_str(); } char *strDataAccess(QByteArray &str){ return str.data(); } const char* strDataAccess(const QByteArray& str){ return str.constData(); } char *strDataAccess(StrLight &str) { return str.data(); } const char *strDataAccess(const StrLight &str) { return str.constData(); } /// Constructs a string containing "null", if cstr is null, /// else the respective value std::string strFromCString(const char *cstr) { if(cstr == nullptr) return "null"; return cstr; } QPair splitAbsPath(const std::string &fullPath) { char sep = '/'; #ifdef _WIN32 sep = '\\'; #endif size_t i = fullPath.rfind(sep, fullPath.length()); QPair pair; if(i == std::string::npos){ pair.first = fullPath; return pair; } if(i == 0){ pair.first = "/"; pair.second = fullPath.substr(1, fullPath.length() - 1); return pair; } pair.first = fullPath.substr(0, i); pair.second = fullPath.substr(i+1, fullPath.length() - i); return pair; } /// Hidden files start with a dot, the the first dot is ignored /// ( the part before the last dot must not be empty or no file extension /// returned) /// @param fname: must not contain any os-separator (e.g. /) std::string getFileExtension(const std::string &fname) { const auto dotIdx = fname.find_last_of('.'); if(dotIdx != std::string::npos && dotIdx != 0){ return fname.substr(dotIdx +1); } return ""; } // maybe_todo: add #IF GCC here to allow for other compiler... /// @param startIdx: generally you would want to choose a value /// greater than zero, otherwise this function will be added as well. std::string generate_trace_string(int startIdx) { const int MAX_STACKTRACE_SIZE = 10; void *array[MAX_STACKTRACE_SIZE]; char **strings; auto size = backtrace (array, MAX_STACKTRACE_SIZE); strings = backtrace_symbols (array, size); std::string bt; for (int i = startIdx; i < size; i++){ bt += std::string(" at ") + strings[i] + "\n"; } free (strings); return bt; } QString argvToQStr(int argc, char * const argv[]){ QStringList l; for(int i=0; i < argc; i++){ l.push_back(argv[i]); } return l.join(" "); } std::string argvToStr(int argc, char *const argv[]) { std::string argStr; for(int i=0; i < argc; i++){ argStr += std::string(argv[i]) + ' '; } if(! argStr.empty()){ // strip final whitespace argStr.resize(argStr.size() - 1); } return argStr; } /// Argv to space-separated string /// As usual, argv must be terminated by a final nullptr. std::string argvToStr(char *const argv[]) { std::string argStr; while(true){ if(*argv == nullptr){ break; } argStr += std::string(*argv) + ' '; ++argv; } if(! argStr.empty()){ // strip final whitespace argStr.resize(argStr.size() - 1); } return argStr; } /// see also: QChar::isSpace int indexOfNonWhiteSpace(const QString &str) { for(int i=0; i < str.size(); i++){ if(! str[i].isSpace()){ return i; } } return -1; } bool qVariantTo(const std::string &str, QString *result) { *result = QString::fromStdString(str); return true; } bool qVariantTo(const StrLight& str, QString* result){ *result = QString::fromUtf8(str.constData(), int(str.size())); return true; } ================================================ FILE: src/common/util/util.h ================================================ #pragma once #ifndef qtr #define qtr QObject::tr #endif #define GET_VARIABLE_NAME(Variable) (#Variable) #include #include #include #include #include #include #include #include #include #include #ifndef likely #ifdef __GNUC__ #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) #else #define likely(x) (x) #define unlikely(x) (x) #endif #endif #include "exccommon.h" #include "UninitializedMemoryHacks.h" #include "strlight.h" FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT(signed char) FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT(unsigned char) FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT(char16_t) FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT(char32_t) FOLLY_DECLARE_STRING_RESIZE_WITHOUT_INIT(unsigned short) #define DISABLE_MOVE(Class) \ Class(const Class &&) Q_DECL_EQ_DELETE;\ Class &operator=(Class &&) Q_DECL_EQ_DELETE; #define DEFAULT_MOVE(Class) \ Class(Class &&) noexcept Q_DECL_EQ_DEFAULT;\ Class &operator=(Class &&) noexcept Q_DECL_EQ_DEFAULT ; bool readLineInto(QTextStream& stream, QString *line, qint64 maxlen = 0); #if QT_VERSION < QT_VERSION_CHECK(5, 6, 0) #include QTextStream& operator<<(QTextStream& stream, const QStringRef &string); #endif /// @return true, if the given weak pointer is default constructed template bool is_uninitialized(std::weak_ptr const& weak) { using wt = std::weak_ptr; return !weak.owner_before(wt{}) && !wt{}.owner_before(weak); } Q_DECLARE_METATYPE(std::string) Q_DECLARE_METATYPE(StrLight) bool shournal_common_init(); #if QT_VERSION < QT_VERSION_CHECK (5, 14, 0) namespace std { /// Make QString hashable in stl-containers template<> struct hash { std::size_t operator()(const QString& s) const { return qHash(s); } }; } #endif StrLight toStrLight(const QString& str); /// allow std::string to be printed via qDebug() QDebug& operator<<(QDebug& out, const std::string& str); /// Common functions to get raw data access for std::string and QByteArray const char* strDataAccess(const char* str); char* strDataAccess(char* str); char* strDataAccess(std::string& str); const char* strDataAccess(const std::string& str); char* strDataAccess(QByteArray& str); const char* strDataAccess(const QByteArray& str); char* strDataAccess(StrLight& str); const char* strDataAccess(const StrLight& str); int indexOfNonWhiteSpace(const QString& str); /// return argv as space-separated string QString argvToQStr(int argc, char *const argv[]); std::string argvToStr(int argc, char * const argv[]); std::string argvToStr(char * const argv[]); template std::string bytesFromVar(const T& t) { static_assert (std::is_pod(), ""); const char* raw_ = static_cast(static_cast(&t)); std::string str(raw_, sizeof (T)); return str; } /// @return defaultVal if size does not match template T varFromBytes(const std::string &str, const T& defaultVal) { static_assert (std::is_pod(), ""); if(str.size() != sizeof (T)){ return defaultVal; } const T* pT = static_cast( static_cast(&str[0]) ); T copy_ = *pT; return copy_; } template QByteArray qBytesFromVar(const T& t) { static_assert (std::is_pod(), ""); const char* raw_ = static_cast(static_cast(&t)); QByteArray str(raw_, sizeof (T)); return str; } /// @return defaultVal if size does not match template T varFromQBytes(const QByteArray &str, const T& defaultVal) { static_assert (std::is_pod(), ""); if(str.size() != sizeof (T)){ return defaultVal; } const T* pT = static_cast( static_cast(str.constData()) ); T copy_ = *pT; return copy_; } template T varFromQBytes(const QByteArray &str) { static_assert (std::is_pod(), ""); if(str.size() != sizeof (T)){ return T(); } const T* pT = static_cast( static_cast(str.constData()) ); T copy_ = *pT; return copy_; } template T BIT(const T & x) { return T(1) << x; } template bool IsBitSet(const T & x, const T & y) { return (x & y) != 0; } /// unset mask in flags; template void clearBitIn(T& flags, const T& mask){ flags &= ~mask; } /// set mask in flags; template void setBitIn(T& flags, const T& mask){ flags |= mask; } bool hasEnding(const std::string &fullString, const std::string &ending); /// recursion end of bytesCombine ... void bytesCombine(std::string &); /// Obtain all bytes of all arguments and append them to str template void bytesCombine(std::string & result, First arg, const Values&... rest ){ for (size_t idx = 0; idx < sizeof(First); idx++){ char byte = *((char *)&arg + idx); result.push_back(byte); } bytesCombine(result, rest...); } template bool contains(const Container& container, const typename Container::value_type& element) { return std::find(container.begin(), container.end(), element) != container.end(); } /// Drop certain fields of the time, e.g. milliseconds. Also /// sets lower fields, e.g. dropping minutes sets seconds /// and ms to 0 as well /// @param c one of M(inutes), s(econds), m(illiseconds) static inline void dropFromTime(QTime& t, char c){ int m=t.minute(), s=t.second(), ms=t.msec(); switch (c) { case 'M': m = 0; break; case 's': s = 0; break; case 'm': ms = 0; break; default: throw QExcIllegalArgument(QString("Bad format c %1").arg(c)); } t.setHMS(t.hour(), m, s, ms); } /// @overload static inline void dropFromTime(QDateTime& d, char c){ auto t = d.time(); dropFromTime(t, c); d.setTime(t); } QString absPath(const QString& path); /// Convert the passed value to QVariant, convert /// to target-type and return true /// on success template bool qVariantTo(QVariant var, T* result) { if(! var.convert(qMetaTypeId())){ return false; } *result = var.value(); return true; } // Don't forget to register converter functions when adding more types... bool qVariantTo(const std::string& str, QString* result); bool qVariantTo(const StrLight& str, QString* result); class ExcQVariantConvert : public QExcCommon { public: using QExcCommon::QExcCommon; }; /// Convert to target-type, throw on error /// @throws ExcQVariantConvert template void qVariantTo_throw(const QVariant& src, T* dst, bool collectStacktrace=true) { if(! qVariantTo(src, dst)){ const char* targetTypeName = QVariant::fromValue(*dst).typeName(); QString actualTypeName; if(targetTypeName == nullptr){ actualTypeName = "invalid/unknown"; } else { actualTypeName = targetTypeName; } QString mesg = qtr( "Failed to convert '%1' to type '%2'" ).arg(src.toString()).arg(actualTypeName); throw ExcQVariantConvert(mesg, collectStacktrace); } } /// @overload template T qVariantTo_throw(const QVariant& src, bool collectStacktrace=true){ T dst; qVariantTo_throw(src, &dst, collectStacktrace); return dst; } /// @overload template T qVariantTo_throw(const std::string& src, bool collectStacktrace=true ){ T dst; if(! qVariantTo(src, &dst)){ // this should never happen, because of having // specialized for std::string... QString mesg = qtr( "Failed to convert std::string to target type."); throw ExcQVariantConvert(mesg, collectStacktrace); } return dst; } QByteArray make_uuid(bool *madeSafe=nullptr); /// Split an absolute path into directory-path and filename: /// /home/user/foo -> "/home/user", "foo" /// If no separator is contained, the full path is returned /// Works with QByteArray, QString and std::string (see overload) template QPair splitAbsPath(const T& path){ QPair pair; const int lastSlash = path.lastIndexOf('/'); if(lastSlash == -1 || lastSlash == int(path.size()) - 1){ pair.first = path; return pair; } if(lastSlash == 0){ pair.first = "/"; pair.second = (path.mid(1)); return pair; } pair.first = path.left(lastSlash); pair.second = path.mid(lastSlash + 1); return pair; } template T pathJoinFilename(const T& path, const T& filename){ assert(path.size() != 0); assert(filename.size() != 0); // special case root if(path == "/"){ return path + filename; } return path + "/" + filename; } /// @overload QPair splitAbsPath(const std::string& fullPath); std::string getFileExtension(const std::string& fname); std::string strFromCString(const char* cstr); std::string generate_trace_string(int startIdx=2); ================================================ FILE: src/common/util/util_performance.cpp ================================================ #include "util_performance.h" /// reverse: reverse string s in place void util_performance::reverse(char *s, int size) { int i, j; char c; for (i = 0, j = size-1; i 0); if (sign < 0) s[i++] = '-'; s[i] = '\0'; reverse(s, i); } /// like itoa but avoid sign checks void util_performance::uitoa(unsigned n, char *s) { int i; i = 0; do { s[i++] = n % 10 + '0'; } while ((n /= 10) > 0); s[i] = '\0'; reverse(s, i); } ================================================ FILE: src/common/util/util_performance.h ================================================ #pragma once /// Warning. Some of the functions used /// here are dangerous and should not /// be used in general namespace util_performance { void reverse(char* s, int size); void itoa(int n, char* s); void uitoa(unsigned n, char* s); } // namespace util_performance ================================================ FILE: src/common/xxhash_common.c ================================================ #include "xxhash_common.h" #ifdef __KERNEL__ #include #include #include #include #include "kutil.h" #include "xxhash_shournalk.h" #else #include #include #include #include #include #define xxh64_update XXH64_update #define xxh64_reset XXH64_reset #define xxh64_digest XXH64_digest #define xxh64 XXH64 #endif #ifndef min #define min MIN #endif #include "user_kernerl.h" static ssize_t __do_read(xxh_common_file_t file, void *buf, size_t nbytes){ #ifdef __KERNEL__ return kutil_kernel_read_cachefriendly(file, buf, nbytes, &file->f_pos); #else ssize_t ret = read(file, buf, nbytes); if(unlikely(ret < 0)){ return -errno; } return ret; #endif } static loff_t __do_seek(xxh_common_file_t file, loff_t offset){ #ifdef __KERNEL__ file->f_pos += offset; return 0; // return vfs_llseek(file, offset, whence); #else loff_t ret = lseek(file, offset, SEEK_CUR); if(unlikely(ret < 0)){ return -errno; } return ret; #endif } /// read bufsize bytes from file and directly hash them static ssize_t __read_and_hash(xxh_common_file_t file, void* buf, size_t bufsize, XXH_COMMON_STATE* xxh_state){ ssize_t readBytes = __do_read(file, buf, bufsize); if(likely(readBytes > 0)){ long xxh_ret = xxh64_update(xxh_state, buf, readBytes); // we always provide a valid buffer, so no // need to check for errors in production. kuassert(xxh_ret == 0); (void)xxh_ret; // avoid unused warning for release builds } return readBytes; } /// read a chunk of bytes from the file and hash it. /// Do that multiple times, if the buffer-size is smaller /// than the chunksize. /// @return The number of read/hashed bytes or a neg. error. static ssize_t __read_chunk(xxh_common_file_t file, struct partial_xxhash* part_hash){ ssize_t readBytes; if(part_hash->chunksize <= part_hash->bufsize){ readBytes = __read_and_hash(file, part_hash->buf, part_hash->chunksize, part_hash->xxh_state); } else { // read and digest immediatly, as our buffer is small size_t missing_bytes = part_hash->chunksize; size_t readsize = part_hash->bufsize; while(true){ ssize_t bytes = __read_and_hash(file, part_hash->buf, readsize, part_hash->xxh_state); if(unlikely(bytes < 0)) return bytes; missing_bytes -= bytes; kuassert(missing_bytes >= 0); if((size_t)bytes < readsize || missing_bytes == 0){ break; } if(missing_bytes < part_hash->bufsize){ // almost done. Loop one last time with // a smaller read size readsize = missing_bytes; } } readBytes = part_hash->chunksize - missing_bytes; } return readBytes; } /// XXHASH-digest a whole file or parts of it at regular intervals. /// @param file the fildescriptor of the file. Note that in general you would want /// to make sure, that the offset is at 0. Note that the offset /// may be changed during the call. /// @param chunksize size of the chunks to read at once. /// @param seekstep Read chunks from the file every seekstep bytes. The read chunk /// does not count into this, so if you actually want to skip bytes, /// seekstep must be greater than chunksize. Otherwise NO SEEK is /// performed at all. /// @param maxCountOfReads stop reading and digest after that count of 'read'- /// operations. /// @param result write hash and count of read bytes in here /// @return 0 on success, else a positive error long partial_xxh_digest_file(xxh_common_file_t file, struct partial_xxhash* part_hash, struct partial_xxhash_result* result ) { long err; int countOfReads; loff_t net_seek; result->count_of_bytes = 0; kuassert(part_hash->max_count_of_reads > 0); kuassert(part_hash->chunksize > 0); kuassert(part_hash->bufsize > 0); xxh64_reset(part_hash->xxh_state, 0); net_seek = part_hash->seekstep - part_hash->chunksize; for(countOfReads=0; countOfReads < part_hash->max_count_of_reads ; ++countOfReads) { // maybe_todo: preload next chunk with filemap.c:page_cache_read? ssize_t readBytes = __read_chunk(file, part_hash); if(unlikely(readBytes < 0)) return -readBytes; result->count_of_bytes += readBytes; if(readBytes < part_hash->chunksize) { break; // EOF } if( net_seek > 0 && unlikely((err = __do_seek(file, net_seek)) < 0) ) { return -err; } } if(result->count_of_bytes == 0){ result->hash = 0; } else { result->hash = xxh64_digest(part_hash->xxh_state); } return 0; } ================================================ FILE: src/common/xxhash_common.h ================================================ /* Common code for usage both in user and (linux-)kernelspace * */ #pragma once #ifdef __KERNEL__ #include "shournalk_global.h" #include struct file; #define XXH_COMMON_STATE struct xxh64_state typedef struct file* xxh_common_file_t; #else #include #include #include "xxhash.h" #define XXH_COMMON_STATE XXH64_state_t typedef int xxh_common_file_t; #endif // __KERNEL__ #ifdef __cplusplus extern "C" { #endif struct partial_xxhash { unsigned chunksize; /* read and hash that many bytes per chunk */ int max_count_of_reads; /* do not read more than that many chunks */ loff_t seekstep; /* determined by file size and max_count_of_reads */ XXH_COMMON_STATE* xxh_state; char* buf; size_t bufsize; }; struct partial_xxhash_result { uint64_t hash; unsigned long long count_of_bytes; // number of read bytes }; long partial_xxh_digest_file(xxh_common_file_t file, struct partial_xxhash* part_hash, struct partial_xxhash_result* result); #ifdef __cplusplus } #endif ================================================ FILE: src/shell-integration-fanotify/CMakeLists.txt ================================================ include(GenerateExportHeader) include_directories( ../common ../common/oscpp ../common/qsimplecfg ../common/qsqlthrow ../common/database ) add_library(libshournal-shellwatch SHARED libshournal-shellwatch.cpp attached_bash.cpp attached_shell.cpp event_open.cpp event_process.cpp shell_globals.cpp shell_logger.cpp shell_request_handler.cpp ) # to list exported symbols of the compiled .so: # nm -D libshournal-shellwatch.so | grep ' T ' GENERATE_EXPORT_HEADER(libshournal-shellwatch) hide_static_lib_symbols(libshournal-shellwatch) # manually set the name of the .so -> we need it later # Do not change the name libshournal-shellwatch.so, it is hardcoded into the shell-integration-scripts. set_target_properties(libshournal-shellwatch PROPERTIES OUTPUT_NAME "${libshournal_fullname}") set_target_properties(libshournal-shellwatch PROPERTIES PREFIX "") set_target_properties(libshournal-shellwatch PROPERTIES SUFFIX "") target_link_libraries(libshournal-shellwatch ${CMAKE_DL_LIBS} # dlsym # Using the static lib_shournal in our shared library exposes # all symbols (not a good idea in a LD_PRELOAD-lib). # Hiding them requires either a --version-script # or all libraries being compiled with -fvisibility=hidden. The former # approach seems more elegant. # See also https://stackoverflow.com/a/22110050/7015849 "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/libshellwatch.version" lib_shournal_common uuid ) ########################## Installation ########################## install(TARGETS libshournal-shellwatch DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/${PROJECT_NAME} ) ================================================ FILE: src/shell-integration-fanotify/attached_bash.cpp ================================================ #include #include #include "logger.h" #include "attached_bash.h" #include "os.h" static int read_seq(){ const char* _LIBSHOURNAL_SEQ_COUNTER = "_LIBSHOURNAL_SEQ_COUNTER"; const char* seq_val = getenv(_LIBSHOURNAL_SEQ_COUNTER); if(seq_val == nullptr){ logWarning << qtr("Required environment variable '%1' " "is unset.").arg(_LIBSHOURNAL_SEQ_COUNTER); return -1; } int seq; try { qVariantTo_throw(seq_val, &seq); } catch (const ExcQVariantConvert& ex) { logWarning << "Failed to convert sequnce:" << ex.descrip(); return -1; } return seq; } /// @throws ExcOs AttachedBash::AttachedBash() : m_lastSeq(1) {} void AttachedBash::handleEnable() { m_lastSeq = read_seq(); } /// The command is considered valid, if the command-counter /// has changed since the last call of this function or handleEnable(). /// This function is meant to be called only *once* /// per command sequence. bool AttachedBash::cmdCounterJustIncremented() { int current_seq = read_seq(); if(current_seq == -1){ return false; // error } if(current_seq == m_lastSeq){ return false; } m_lastSeq = current_seq; return true; } ================================================ FILE: src/shell-integration-fanotify/attached_bash.h ================================================ #pragma once #include "attached_shell.h" /// We read the env-variable set in bash's PS0 /// in order to prepare the observation of the next command /// sequence. We cannot do this easily from within a function /// called in PS0, because that is run in a subshell. /// Another possibility is to run a signal handler but there we must /// again be careful not to interfere with custom handlers of the /// user. /// counter=0 /// trap_handler(){ /// counter=$((counter+1)) /// echo "hi from trap_handler: $counter: $(history 1)" >&2 /// } /// trap trap_handler SIGRTMIN /// PS0='$(echo "sending signal... "; kill -SIGRTMIN $$; )' class AttachedBash : public AttachedShell { public: AttachedBash(); void handleEnable() override; bool cmdCounterJustIncremented() override; private: int m_lastSeq; }; ================================================ FILE: src/shell-integration-fanotify/attached_shell.cpp ================================================ #include "attached_shell.h" void AttachedShell::handleEnable() {} /// This function is meant to be called only *once* /// per command sequence. bool AttachedShell::cmdCounterJustIncremented() { return false; } ================================================ FILE: src/shell-integration-fanotify/attached_shell.h ================================================ #pragma once #include "util.h" /// Abstract base class for shells class AttachedShell { public: AttachedShell() = default; virtual ~AttachedShell() = default; virtual void handleEnable(); virtual bool cmdCounterJustIncremented(); public: Q_DISABLE_COPY(AttachedShell) DEFAULT_MOVE(AttachedShell) }; ================================================ FILE: src/shell-integration-fanotify/event_open.cpp ================================================ #define _LARGEFILE64_SOURCE #include #include #include #include #include #include #include "event_open.h" #include "logger.h" #include "shell_globals.h" #include "cleanupresource.h" #include "qoutstream.h" #include "osutil.h" #include "shell_request_handler.h" #include "shell_logger.h" #include "translation.h" using shell_request_handler::ShellRequest; using shell_request_handler::checkForTriggerAndHandle; /// @return absolute version of the passed path or an empty string in case /// of an error. static std::string mkAbsPath(const char* path){ if(path[0] == '/'){ return path; } std::string buf(PATH_MAX, '\0'); char* rawBuf = strDataAccess(buf); if(getcwd(rawBuf, buf.size()) == nullptr){ logWarning << qtr("Failed to resolve relative path %1. " "The working-directory could not be determined (%2). " "File events will not be registered.") .arg(path, translation::strerror_l()); return {}; } if(rawBuf[0] != '/'){ // see also man 3 getcwd logWarning << qtr("Failed to resolve relative path %1. " "The working-directory does not begin with '/' but %2. " "File events will not be registered.") .arg(path, rawBuf); return {}; } // resize to actual length buf.resize(strlen(rawBuf)); if(buf.size() != 1){ buf += '/'; } buf += path; return buf; } /// Write to a new unnamed tmp-file, if the shell-request /// was successful. Note that the shell will close the fd /// for us later. /// @return fd to deleted tmp-file. static int writerTriggerResponse(bool success){ int fd = osutil::unnamed_tmp(); std::string mesg = (success) ? "ok" : "fail"; // write string null-terminated (size+1) so in the shell // we can read -d '' trigger_response < '_///shournal_trigger_response///_' os::write(fd, mesg.c_str(), mesg.size() + 1); os::lseek(fd, 0, SEEK_SET); return fd; } int event_open::handleOpen(const char *pathname, int flags, mode_t mode, bool largeFile) { if(largeFile){ setBitIn(flags, O_LARGEFILE); } auto& g_shell = ShellGlobals::instance(); if(g_shell.ignoreEvents.test_and_set()){ return g_shell.orig_open(pathname, flags, mode); } auto clearIgnEvents = finally([&g_shell] { g_shell.ignoreEvents.clear(); }); // Note: we only process shell-request if the trigger env-variable is set AND the current // pathname is _///shournal_trigger_response///_ // So check for the pathname before handling the request in // checkForTriggerAndHandle (this is for cases where the trigger variable is set // and other redirections occurr in between). if(strcmp(pathname, "_///shournal_trigger_response///_") == 0){ bool shellRequestSuccess = false; auto shellRequest = checkForTriggerAndHandle(&shellRequestSuccess); switch (shellRequest) { case ShellRequest::TRIGGER_UNSET: break; default: return writerTriggerResponse(shellRequestSuccess); } } if(g_shell.watchState != E_WatchState::WITHIN_CMD){ shell_earlydbg("ignoring pathname %s (not WITHIN_CMD)", pathname); return g_shell.orig_open(pathname, flags, mode); } const auto absPath = mkAbsPath(pathname); // pass the resolved abs. path relative to shournal's root directory fd, // by omitting the initial '/'. // Users may further pass malformed file-paths such as //foo, so find the first // non-slash char. const char* actualPath = nullptr; for(size_t i=0; i < absPath.size(); i++){ if(absPath[i] != '/'){ actualPath = &absPath[i]; break; } } if(actualPath == nullptr || absPath.c_str() + absPath.size() - actualPath < 1){ // Get here on mkAbsPath-error or because user attempted to open "/" or "" // The shortest possible absolute FILEpath under linux is two chars long. // We may get here, if bash-user calls e.g. // while read line; do echo $line ; done < "/" logDebug << "no valid path" << absPath; return g_shell.orig_open(pathname, flags, mode); } logDebug << "about to open" << actualPath - 1; return openat(g_shell.shournalRootDirFd, actualPath, flags, mode); } ================================================ FILE: src/shell-integration-fanotify/event_open.h ================================================ #pragma once #include namespace event_open { int handleOpen(const char *pathname, int flags, mode_t mode, bool largeFile ); } ================================================ FILE: src/shell-integration-fanotify/event_process.cpp ================================================ #include #include #include #include #include #include "attached_bash.h" #include "cleanupresource.h" #include "logger.h" #include "event_process.h" #include "shell_globals.h" #include "settings.h" #include "excos.h" #include "qsimplecfg/exccfg.h" #include "subprocess.h" #include "osutil.h" #include "fdcommunication.h" #include "util.h" #include "commandinfo.h" #include "app.h" #include "shell_logger.h" #include "translation.h" #include "qoutstream.h" #include "shell_request_handler.h" static int execveUnobserved(const char *filename, char * const argv[], char * const envp[]){ auto& g_shell = ShellGlobals::instance(); logDebug << __func__ << filename; auto sockFlags = g_shell.shournalSockFdDescripFlags; setBitIn(sockFlags, FD_CLOEXEC); os::setFdDescriptorFlags(g_shell.shournalSocketNb, sockFlags); // in case execve fails, restore flags. auto resetCLOEXEC = finally([&g_shell] { try { os::setFdDescriptorFlags(g_shell.shournalSocketNb, g_shell.shournalSockFdDescripFlags); } catch (std::exception& e) { logCritical << e.what(); } }); return g_shell.orig_execve(filename, argv, envp); } pid_t event_process::handleFork() { auto& g_shell = ShellGlobals::instance(); if( g_shell.ignoreEvents.test_and_set()){ return g_shell.orig_fork(); } auto clearIgnEvents = finally([&g_shell] {g_shell.ignoreEvents.clear(); }); if( g_shell.inParentShell && g_shell.watchState == E_WatchState::INTERMEDIATE && dynamic_cast(g_shell.pAttchedShell) != nullptr && g_shell.pAttchedShell->cmdCounterJustIncremented()){ shell_request_handler::handlePrepareCmd(); } pid_t ret = g_shell.orig_fork(); if(ret == 0){ if(g_shell.shellParentPid != 0){ // our parent shell is initialized, so we can't be it g_shell.inParentShell = false; } } return ret; } int event_process::handleExecve(const char *filename, char * const argv[], char * const envp[]) { auto& g_shell = ShellGlobals::instance(); if(g_shell.ignoreEvents.test_and_set()){ return g_shell.orig_execve(filename, argv, envp); } auto clearIgnEvents = finally([&g_shell] {g_shell.ignoreEvents.clear(); }); if( g_shell.inParentShell || g_shell.watchState != E_WatchState::WITHIN_CMD){ shell_earlydbg("ignore execve of %s", filename); return g_shell.orig_execve(filename, argv, envp); } // No point in observing an unstattable executable. // Further, do not monitor suid-applications. Note: this is, of course, *not* // a security-feature, however, events by other users are in // relevant cases not recorded by shournal anyway. struct stat st; if(stat(filename, &st) == -1 || IsBitSet(st.st_mode, mode_t(S_ISUID) ) ){ return execveUnobserved(filename, argv, envp); } std::string filenameStr(filename); auto& sets = Settings::instance(); if(sets.ignoreCmdsRegardslessOfArgs().find(filenameStr) != sets.ignoreCmdsRegardslessOfArgs().end()){ return execveUnobserved(filename, argv, envp); } std::string fullCmd; QVarLengthArray args; args.push_back(app::SHOURNAL_RUN_FANOTIFY); args.push_back("--msenter"); std::string pid = std::to_string(g_shell.lastMountNamespacePid); args.push_back(pid.c_str()); args.push_back("--verbosity"); args.push_back(g_shell.shournalRunVerbosity.c_str()); args.push_back("--env"); // first value after --env is its size, which we don't know yet. args.push_back("DUMMY"); int envSizeIdx = args.size() -1; // set shournal socket only for observed processes (do not add to // shell env). const std::string shournalSocketNbStr = std::string(app::ENV_VAR_SOCKET_NB) + '=' + std::to_string(g_shell.shournalSocketNb); args.push_back(shournalSocketNbStr.c_str()); for(char* const *e = envp; *e != nullptr; e++) { args.push_back(*e); } // optimization in shournal-run... args.push_back("SHOURNAL_DUMMY_NULL=1"); std::string envSize = std::to_string(args.size() - envSizeIdx - 1); args[envSizeIdx] = envSize.c_str(); args.push_back("--exec-filename"); args.push_back(filename); args.push_back("--exec"); fullCmd += filenameStr + ' '; for(int i=0; ; i++) { // include final nullptr here args.push_back(argv[i]); if(argv[i] == nullptr){ break; } if(i > 0){ // for the ignore-list skip argv0 which should be the same // as filename in most cases anyway. fullCmd.append(argv[i]); fullCmd += ' '; } } // strip final whitespace fullCmd.pop_back(); if(sets.ignoreCmds().find(fullCmd) != sets.ignoreCmds().end()){ logDebug << "exec UNobserved:" << fullCmd.c_str(); return execveUnobserved(filename, argv, envp); } logDebug << "execvpe observed:" << fullCmd.c_str(); try { os::exec(args, envp); } catch (const os::ExcOs& e) { logCritical << qtr("Failed to launch %1 with external program. " "Please make sure %2 is in your PATH: %3. " "Running it unobserved instead...") .arg(filename, app::SHOURNAL_RUN_FANOTIFY, e.what()); } return execveUnobserved(filename, argv, envp); } ================================================ FILE: src/shell-integration-fanotify/event_process.h ================================================ #pragma once #include namespace event_process { pid_t handleFork(); int handleExecve(const char *filename, char *const argv[], char *const envp[]); } ================================================ FILE: src/shell-integration-fanotify/libshellwatch.version ================================================ { global: open; open64; fork; execve; strcpy; local: *; # hide everything else }; ================================================ FILE: src/shell-integration-fanotify/libshournal-shellwatch.cpp ================================================ // necessary for RTLD_NEXT in dlfcn.h #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include #include #include #include #include "cleanupresource.h" #include "event_open.h" #include "event_process.h" #include "staticinitializer.h" #include "shell_globals.h" #include "shell_logger.h" // cmake export-symbol control: #include "libshournal-shellwatch_export.h" /// Initalize the original functions close, fclose ... /// One might think it was a good idea, to initialize the functions /// in gcc's __attribute__((constructor)). This is too late, /// at that time fclose/close was already called several times. /// One has to be _extremely_ careful not to call anything which invokes /// one of the below preloaded functions (open, fork...) in here, /// otherwise we're lost. static void initSymIfNeeded(){ static StaticInitializer loader( [](){ try { ShellGlobals& g_shell = ShellGlobals::instance(); g_shell.orig_fork = reinterpret_cast(os::dlsym(RTLD_NEXT, "fork")); g_shell.orig_execve = reinterpret_cast(os::dlsym(RTLD_NEXT, "execve")); g_shell.orig_open = reinterpret_cast(os::dlsym(RTLD_NEXT, "open")); // globals.orig_fopen = reinterpret_cast(os::dlsym(RTLD_NEXT, "fopen")); g_shell.orig_strcpy = reinterpret_cast(os::dlsym(RTLD_NEXT, "strcpy")); return; } catch(const std::exception& ex){ fprintf(stderr, "shournal shell integration fatal error: " "failed to load original symbols, expect " "the worst: %s", ex.what()); } }); #ifndef NDEBUG // Ignoring events is maybe not strictly necessary here, // but better safe than sorry. ShellGlobals& g_shell = ShellGlobals::instance(); if(g_shell.ignoreEvents.test_and_set()){ return; } auto clearIgnEvents = finally([&g_shell] { g_shell.ignoreEvents.clear(); }); static StaticInitializer initPrintDbg( [](){ shell_earlydbg("initalizing libshournal-shellwatch.so for pid %d %s", os::getpid(), os::readlink("/proc/self/exe").c_str()); }); #endif } #ifdef __cplusplus extern "C" { #endif LIBSHOURNAL_SHELLWATCH_EXPORT int open(const char *pathname, int flags, mode_t mode) { // std::cerr << __func__ << "\n"; initSymIfNeeded(); try{ return event_open::handleOpen(pathname, flags, mode, false); } catch (const std::exception& ex ) { std::cerr << __func__ << " fatal: " << ex.what() << "\n"; } return ShellGlobals::instance().orig_open(pathname, flags, mode); } LIBSHOURNAL_SHELLWATCH_EXPORT int open64(const char *pathname, int flags, mode_t mode) { // std::cerr << __func__ << "\n"; initSymIfNeeded(); try{ // probably O_LARGEFILE should only be set, if we are running in 32 // bit mode (using open64). It seems to do no harm though (see handleOpen). return event_open::handleOpen(pathname, flags, mode, true); } catch (const std::exception& ex ) { std::cerr << __func__ << " fatal: " << ex.what() << "\n"; } return ShellGlobals::instance().orig_open(pathname, flags, mode); } // There seems to be no point in observing fopen - browsing the source-code // of bash, zsh, kash, csh, .. all relevant user file activity is handled via // the 'open' library-call. If one day it would be observed anyway: the shell's // seem to not make use of any (g)libc-fopen-mode-extensions like 'c' or ,ccs=string. // As such the translation of w,r,a etc. to O_WRONLY,O_RDONLY, etc. is pretty // straight forward. Otherwise things get more complicated - fdopen does not handle // all the cases. // LIBSHOURNAL_SHELLWATCH_EXPORT // FILE* fopen(const char *path, const char *mode) { // // std::cerr << __func__ << "\n"; // try{ // initSymIfNeeded(); // return event_open::handleFopen(path, mode); // } catch (const std::exception& ex ) { // std::cerr << __func__ << " fatal: " << ex.what() << "\n"; // } // return nullptr; // } // see comment for fopen. // LIBSHOURNAL_SHELLWATCH_EXPORT // FILE* fopen64(const char *path, const char *mode) { // // initIfNeeded(); // // FILE* f = orig_fopen64(path, mode); // // if(f != NULL){ // // handleOpen(fileno(f)); // // } // return f; // } LIBSHOURNAL_SHELLWATCH_EXPORT pid_t fork(){ initSymIfNeeded(); try { return event_process::handleFork(); } catch (const std::exception& ex ) { std::cerr << __func__ << " fatal: " << ex.what() << "\n"; } return ShellGlobals::instance().orig_fork(); } LIBSHOURNAL_SHELLWATCH_EXPORT int execve(const char *filename, char *const argv[], char *const envp[]){ initSymIfNeeded(); try { return event_process::handleExecve(filename, argv, envp); } catch (const std::exception& ex ) { std::cerr << __func__ << " fatal: " << ex.what() << "\n"; } return ShellGlobals::instance().orig_execve(filename, argv, envp); } #ifdef __cplusplus } #endif ================================================ FILE: src/shell-integration-fanotify/shell_globals.cpp ================================================ #include "app.h" #include "qoutstream.h" #include "shell_globals.h" #include "shell_logger.h" #include "staticinitializer.h" #include "translation.h" const char* ENV_VARNAME_SHELL_VERBOSITY = "_SHOURNAL_LIB_SHELL_VERBOSITY"; static bool updateShouranlRunVerbosityFromEnv(bool verboseIfUnset){ const char* VERB_VARNAME = "_SHOURNAL_VERBOSITY"; const char* verbosityValue = getenv(VERB_VARNAME); if(verbosityValue == nullptr){ if(verboseIfUnset){ logWarning << qtr("Required verbosity environment variable '%1' " "is unset.").arg(VERB_VARNAME); return false; } return true; } if(app::VERBOSITIES.find(verbosityValue) == app::VERBOSITIES.end()){ logWarning << qtr("Verbosity environment variable '%1' " "is invalid ('%2')").arg(VERB_VARNAME, verbosityValue); return false; } auto& g_shell = ShellGlobals::instance(); g_shell.shournalRunVerbosity = verbosityValue; return true; } ShellGlobals &ShellGlobals::instance() { static ShellGlobals s; return s; } ShellGlobals::ShellGlobals() { ignoreEvents.clear(); ignoreSigation.clear(); } bool ShellGlobals::performBasicInitIfNeeded(){ bool success = true; static StaticInitializer initOnFirstCall( [&success](){ app::setupNameAndVersion("shournal shell-integration"); try { if(! shournal_common_init()){ QIErr() << qtr("Fatal error: failed to initialize custom Qt conversion functions"); } shell_logger::setup(); translation::init(); updateVerbosityFromEnv(false); } catch (const std::exception& ex) { success = false; logCritical << ex.what(); } }); return success; } /// @param verboseIfUnset If true print a warning if the environment variables /// are unset. /// @return if verboseIfUnset: return false if unset or invalid /// else : return false if invalid bool ShellGlobals::updateVerbosityFromEnv(bool verboseIfUnset){ bool shournalRunSuccess = updateShouranlRunVerbosityFromEnv(verboseIfUnset); const char* VERB_VARNAME = ENV_VARNAME_SHELL_VERBOSITY; const char* verbosityValue = getenv(VERB_VARNAME); if(verbosityValue == nullptr){ if(verboseIfUnset){ logWarning << qtr("Required verbosity environment variable '%1' " "is unset.").arg(VERB_VARNAME); return false; } return shournalRunSuccess; } if(app::VERBOSITIES.find(verbosityValue) == app::VERBOSITIES.end()){ logWarning << qtr("Verbosity environment variable '%1' " "is invalid ('%2')").arg(VERB_VARNAME, verbosityValue); return false; } auto& g_shell = ShellGlobals::instance(); g_shell.verbosityLevel = logger::strToMsgType(verbosityValue); return shournalRunSuccess; } ================================================ FILE: src/shell-integration-fanotify/shell_globals.h ================================================ #pragma once #include #include #include #include #include #include #include "attached_shell.h" #include "fdcommunication.h" #include "logger.h" #include "os.h" #include "sessioninfo.h" #include "util.h" typedef pid_t (*fork_func_t)(); typedef int (*execve_func_t)(const char *filename, char *const argv[], char *const envp[]); typedef int (*open_func_t)(const char *pathname, int flags, mode_t mode); typedef char * (*strcpy_func_t)(char *, const char*); extern const char* ENV_VARNAME_SHELL_VERBOSITY; enum class E_WatchState {DISABLED, WITHIN_CMD, INTERMEDIATE, ENUM_END}; class ShellGlobals { public: static ShellGlobals& instance(); static bool performBasicInitIfNeeded(); static bool updateVerbosityFromEnv(bool verboseIfUnset); int shournalSocketNb {-1}; fork_func_t orig_fork {}; execve_func_t orig_execve {}; open_func_t orig_open {}; strcpy_func_t orig_strcpy {}; std::atomic_flag ignoreEvents{}; E_WatchState watchState {E_WatchState::DISABLED}; bool inParentShell {false}; fdcommunication::SocketCommunication shournalSocket; pid_t lastMountNamespacePid {-1}; struct sigaction origSigintAction{}; std::atomic_flag ignoreSigation{}; AttachedShell* pAttchedShell {}; QtMsgType verbosityLevel {QtMsgType::QtWarningMsg}; std::string shournalRunVerbosity {logger::msgTypeToStr(QtWarningMsg)}; int shournalSockFdDescripFlags {-1}; SessionInfo sessionInfo; int shournalRootDirFd {-1}; pid_t shellParentPid {0}; public: ~ShellGlobals() = default; Q_DISABLE_COPY(ShellGlobals) DISABLE_MOVE(ShellGlobals) private: ShellGlobals(); }; ================================================ FILE: src/shell-integration-fanotify/shell_logger.cpp ================================================ #include #include #include #include #include "app.h" #include "shell_logger.h" #include "qoutstream.h" #include "shell_globals.h" #include "fdcommunication.h" #include "logger.h" #include "socket_message.h" using socket_message::E_SocketMsg; namespace { struct ShellLogState { QString logPreamble; QVarLengthArray bufferedMessages; }; ShellLogState& sLogState(){ static ShellLogState s; return s; } void sendViaSock(QByteArray& msg){ try { ShellGlobals::instance().shournalSocket.sendMsg({int(E_SocketMsg::LOG_MESSAGE), msg} ); } catch (const os::ExcOs& e) { QIErr() << "Failed to send message via socket:" << e.what(); } } void messageHandler(QtMsgType msgType, const QMessageLogContext &context, const QString &msg) { auto& g_shell = ShellGlobals::instance(); int desiredVerbosity = logger::msgTypeToOrdinal(g_shell.verbosityLevel); int typeOrdinal = logger::msgTypeToOrdinal(msgType); #ifndef NDEBUG if (msgType == QtDebugMsg) { if(typeOrdinal >= desiredVerbosity){ QErr() << sLogState().logPreamble << " Dbg: " << "(" << QFileInfo(context.file).fileName() <<":" << context.line << ") " << "pid " << getpid() << ": " << msg << '\n' ; } return; } #else Q_UNUSED(context) #endif QString msgTypeStr = logger::msgTypeToStr(msgType); const QString dateTime = QDateTime::currentDateTime().toString( "yyyy-MM-dd HH:mm:ss"); if(typeOrdinal >= desiredVerbosity){ QErr() << sLogState().logPreamble << " " < sLogState().bufferedMessages.capacity()){ QErr() << sLogState().logPreamble << " " << qtr("Too many log-messages could not be sent " "to external %1-process, " "so some will be lost (not logged to disk). " "This is most likely a bug.").arg(app::SHOURNAL); sLogState().bufferedMessages.clear(); } sLogState().bufferedMessages.push_back(msgArr); } } } // namespace void shell_logger::setup() { sLogState().logPreamble = QString(app::SHOURNAL) + " shell-integration"; qInstallMessageHandler(messageHandler); } /// There is not always a socket to shournal open. /// In that case, store them here until flushed void shell_logger::flushBufferdMessages() { for(QByteArray& msg : sLogState().bufferedMessages){ sendViaSock(msg); } sLogState().bufferedMessages.clear(); } /// Instead of logDebug use this function which works without any /// complex initialization. Otherwise we might mess up global variables of /// the attached program, e.g. qInstallMessageHandler or /// QCoreApplication::setApplicationName ... /// Note however that we remove *this shared libaray from LD_PRELOAD _before_ /// calling qInstallMessageHandler etc. (from within the shell integration scripts), /// so we should be mostly safe. /// In general it is probably a good idea to not use foreign complex functions /// like qInstallMessageHandler from within the shell integration at all .. void __shell_earlydbg(const char* file, int line, const char *format, ...) { const char* verbosityValue = getenv(ENV_VARNAME_SHELL_VERBOSITY); if(verbosityValue == nullptr || strcmp(verbosityValue, "dbg") != 0){ return; } fprintf(stderr, "shournal shell integration Dbg: (%s:%d) pid %d: ", file, line, os::getpid()); va_list args; va_start(args, format); vfprintf(stderr, format, args); va_end(args); fprintf(stderr, "\n"); } ================================================ FILE: src/shell-integration-fanotify/shell_logger.h ================================================ #pragma once #ifndef __FILENAME__ #define __FILENAME__ (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) #endif /// The shell logger prints to stderr, /// and sends messages via socket to the external /// shournal process, where they are written to file. /// In case the socket is closed, the messages are buffered. namespace shell_logger { void setup(); void flushBufferdMessages(); } #ifndef NDEBUG void __shell_earlydbg(const char* file, int line, const char *format, ...); #define shell_earlydbg(format, args...) __shell_earlydbg(__FILENAME__, __LINE__, format, ## args) #else #define shell_earlydbg(format, args...) #endif ================================================ FILE: src/shell-integration-fanotify/shell_request_handler.cpp ================================================ #include #include #include #include #include #include "shell_request_handler.h" #include "cleanupresource.h" #include "logger.h" #include "event_process.h" #include "shell_globals.h" #include "settings.h" #include "excos.h" #include "qsimplecfg/exccfg.h" #include "subprocess.h" #include "osutil.h" #include "fdcommunication.h" #include "util.h" #include "commandinfo.h" #include "app.h" #include "shell_logger.h" #include "translation.h" #include "qoutstream.h" #include "attached_bash.h" #include "staticinitializer.h" #include "socket_message.h" #include "interrupt_handler.h" #include "conversions.h" using socket_message::E_SocketMsg; using socket_message::socketMsgToStr; using fdcommunication::SocketCommunication; using osutil::closeVerbose; using shell_request_handler::ShellRequest; // const char* shellRequestToStr(ShellRequest r){ // switch (r) { // case ShellRequest::ENABLE: // return "enable"; // case ShellRequest::DISABLE: // return "disable"; // case ShellRequest::PREPARE_CMD: // return "prepare_cmd"; // case ShellRequest::CLEANUP_CMD: // return "cleanup_cmd"; // case ShellRequest::PRINT_VERSION: // return "print_version"; // case ShellRequest::SIGINT_HANDLER_INSTALL: // return "sigint_install"; // case ShellRequest::SIGINT_HANDLER_RESTORE: // return "sigint_restore"; // case ShellRequest::DUMMY: // return "dummy"; // case ShellRequest::ENUM_END: // return "enumend!"; // } // return "unkown"; // } static bool initializeAttachedShellIfNeeded(){ auto& g_shell = ShellGlobals::instance(); if(g_shell.pAttchedShell != nullptr){ return true; } const char* attachedShellName = getenv("_SHOURNAL_SHELL_NAME"); if(attachedShellName == nullptr){ // should never happen logCritical << "shell name is not set in environment."; return false; } try { switch (attachedShellName[0]) { case 'b': // bash g_shell.pAttchedShell = new AttachedBash(); break; case 'z': // zsh - nothing special to do. g_shell.pAttchedShell = new AttachedShell(); break; default: logCritical << "unknown shell name:" << attachedShellName; return false; } } catch (const os::ExcOs& e) { logCritical << "Failed to initialize attached shell:" << e.what(); return false; } return true; } static bool loadSettings(){ try { // maybe_todo: copy file to another path and load the same file later in shournal-run: Settings::instance().load(); return true; } catch (const std::exception& e) { logCritical << e.what() << "\n"; } logCritical << "Because of that, the shell observation is disabled\n"; return false; } /// Shells usually start at low numbers for internal file descriptors (usually 10), /// we try to find the highest possible free fd /// If startFd != -1, start searching from that. static int verbose_findHighestFreeFd(int startFd=-1){ int fd = osutil::findHighestFreeFd(startFd, 30); if(fd == -1){ logWarning << qtr("Could not find a free file descriptor number. " "The max. number of open files for this process is %1.") .arg(osutil::getMaxCountOpenFiles()); } return fd; } /// Read update request from environment and check if the request /// is valid (log error on exit). static ShellRequest readCheckShellUpdateRequest(){ const char* TRIGGER_NAME = "_LIBSHOURNAL_TRIGGER"; const char* shellStateStr = getenv(TRIGGER_NAME); if(shellStateStr == nullptr){ // No update request return ShellRequest::TRIGGER_UNSET; } if(! ShellGlobals::performBasicInitIfNeeded()){ // should never happen. return ShellRequest::TRIGGER_UNSET; } uint shellRequestInt; try { qVariantTo_throw(shellStateStr, &shellRequestInt); } catch (const ExcQVariantConvert& ex) { logCritical << qtr("Cannot determine shell-request: ") << ex.descrip(); return ShellRequest::TRIGGER_MALFORMED; } if(shellRequestInt >= static_cast(ShellRequest::ENUM_END)){ logCritical << qtr("Invalid shell-request passed:") << shellRequestInt; return ShellRequest::TRIGGER_MALFORMED; } auto shellRequest = static_cast(shellRequestInt); // Note: this logDebug is called BEFORE initialize logging. // logDebug << "received shell request:" << int(shellRequest); // QIErr() << "received shell request:" << int(shellRequest) << "(current state:" // << int(ShellGlobals::instance().watchState) << ")"; return shellRequest; } static void verboseCloseShournalSocket(){ auto& g_shell = ShellGlobals::instance(); if(g_shell.shournalSocket.sockFd() >= 0 && close(g_shell.shournalSocket.sockFd()) == -1){ logWarning << "close of shournal-socket failed:" << translation::strerror_l(); } g_shell.shournalSocket.setSockFd(-1); } static void verboseCloseRootDirFd(){ auto& g_shell = ShellGlobals::instance(); if(g_shell.shournalRootDirFd >= 0 && close(g_shell.shournalRootDirFd) == -1){ logWarning << "close of shournal-root dir-fd failed:" << translation::strerror_l(); } g_shell.shournalRootDirFd = -1; } static bool updateShellPID(){ const char* _SHOURNAL_SHELL_PID = "_SHOURNAL_SHELL_PID"; const char* pidValue = getenv(_SHOURNAL_SHELL_PID); if(pidValue == nullptr){ logWarning << qtr("Required environment variable '%1' " "is unset.").arg(_SHOURNAL_SHELL_PID); return false; } pid_t pid; try { qVariantTo_throw(pidValue, &pid); } catch (const ExcQVariantConvert& ex) { logWarning << "Failed to convert pid:" << ex.descrip(); return false; } auto realPid = getpid(); if(pid != realPid){ logWarning << qtr("Apparently we were enabled from a subshell, which " "is not supported."); return false; } auto& g_shell = ShellGlobals::instance(); g_shell.shellParentPid = pid; g_shell.inParentShell = true; return true; } static bool handleDisableRequest(){ auto& g_shell = ShellGlobals::instance(); if(g_shell.watchState == E_WatchState::DISABLED){ logWarning << qtr("Received disable-request while shell observation " "was already disabled."); return false; } g_shell.watchState = E_WatchState::DISABLED; verboseCloseShournalSocket(); verboseCloseRootDirFd(); logDebug << "shell-integration disabled!"; return true; } static bool handleCleanupCmd(){ auto& g_shell = ShellGlobals::instance(); if(g_shell.watchState != E_WatchState::WITHIN_CMD){ // can for example happen, if // - a user presses enter while the command-string is empty, or Ctrl+C // is pressed without a currently executing command, // - being at the first prompt after SHOURNAL_ENABLE -> no command // was observed yet, nothing to clean up logDebug << "ignoring cleanup-request: not within command."; return false; } auto finalActions = finally([&g_shell] { g_shell.watchState = E_WatchState::INTERMEDIATE; verboseCloseShournalSocket(); verboseCloseRootDirFd(); }); QByteArray lastCommand = getenv("_SHOURNAL_LAST_COMMAND"); if(lastCommand.isNull()){ logWarning << "Failed to retrieve last command string from environment"; lastCommand = "UNKNOWN"; } const char* lastReturnValueStr = getenv("_SHOURNAL_LAST_RETURN_VALUE"); qint32 returnVal = CommandInfo::INVALID_RETURN_VAL; if(lastReturnValueStr == nullptr){ logWarning << qtr("Failed to retrieve last return-value from environment"); } else { try { qVariantTo_throw(lastReturnValueStr, &returnVal); } catch (const ExcQVariantConvert& ex) { logWarning << "Failed to convert last return value:" << ex.descrip(); } } logDebug << __func__ << "sending to shournal-run-fanotify" << "($?:" << returnVal << "):" << lastCommand.mid(0, 100); SocketCommunication::Messages messages; messages.push_back({int(E_SocketMsg::COMMAND), lastCommand}); messages.push_back({int(E_SocketMsg::RETURN_VALUE), qBytesFromVar(returnVal)}); g_shell.shournalSocket.sendMessages(messages); return true; } static bool handleEnableRequest(){ if(! initializeAttachedShellIfNeeded()){ return false; } ShellGlobals::updateVerbosityFromEnv(false); auto& g_shell = ShellGlobals::instance(); if(g_shell.watchState != E_WatchState::DISABLED){ logDebug << "received enable request while watchstate != DISABLED" << int(E_WatchState::DISABLED); } static StaticInitializer initOnFirstCall( [](){ // This shell might have been launched within an already observed shell. // Note that when the shell observation was launched, we already left // the observerd mount namespace. Thus all that remains to do is closing // respective fd (which hopefully does not belong to another program.. const char* socketNbStr = getenv(app::ENV_VAR_SOCKET_NB); if(socketNbStr == nullptr){ return ; } auto laterUnsetIt = finally([] { unsetenv(app::ENV_VAR_SOCKET_NB); }); int fdNb; try { qVariantTo_throw(socketNbStr, &fdNb); } catch (const ExcQVariantConvert& ex) { logCritical << qtr("Bad environment variable %1: ").arg(app::ENV_VAR_SOCKET_NB) << ex.descrip(); return; } if(osutil::fdIsOpen(fdNb)){ logDebug << "initially closing shournal-socket" << fdNb; closeVerbose(fdNb); } else { logInfo << QString("The environment variable %1 is set, but the socket " "%2 is not open").arg(app::ENV_VAR_SOCKET_NB).arg(fdNb); } }); g_shell.watchState = E_WatchState::INTERMEDIATE; bool madeSafe; g_shell.sessionInfo.uuid = make_uuid(&madeSafe); if(! madeSafe){ logInfo << __func__ << qtr("session uuid not created 'safe'. Is the uuidd-daemon running?"); } os::setenv(QByteArray("SHOURNAL_SESSION_ID"), g_shell.sessionInfo.uuid.toBase64()); g_shell.pAttchedShell->handleEnable(); logDebug << "shell-integration enabled!"; return true; } /// Launch external shournal (detached) and wait for it to finish unsharing /// the mount-NS and fanotify-marking the mounts. Since it is called in a new session /// (setsid), it survives the parent shell (*this process), furhter it receives no sigint, destinated /// for our shell, which could have caused it to terminate even before installing a SIGIGN-handler. /// Pass a socket to shournal, which is used for communication *and* to stop it /// (semi-)automatically. Each subsequentially launched process inherits it. Once /// all of them finished *and* we cleaned up (or died), external shournal stops. /// Note that for processes, which close passed file-descriptors /// before exit, shournal might quit too early, in which case file modfication events /// are lost bool shell_request_handler::handlePrepareCmd(){ ShellGlobals::updateVerbosityFromEnv(false); auto& g_shell = ShellGlobals::instance(); if(g_shell.watchState == E_WatchState::WITHIN_CMD){ // Happens e.g. if a previous cleanup request was ignored due to // an invalid command. logDebug << "Received setup-request while shell observation " "was already enabled (might be ok)."; return false; } if(! loadSettings()){ return false; } g_shell.shournalSocket.setSockFd(-1); try { g_shell.shournalSocketNb = verbose_findHighestFreeFd(); if( g_shell.shournalSocketNb == -1){ return false; } g_shell.shournalRootDirFd = verbose_findHighestFreeFd(g_shell.shournalSocketNb - 1); if( g_shell.shournalRootDirFd == -1){ return false; } auto sockets = os::socketpair(PF_UNIX, SOCK_STREAM); auto autocloseSocket0 = finally([&sockets] { close(sockets[0]); }); auto autocloseSocket1 = finally([&sockets] { close(sockets[1]); }); const char* BACKEND_FILENAME = app::SHOURNAL_RUN_FANOTIFY; subprocess::Args_t args = { BACKEND_FILENAME, "--socket-fd", std::to_string(sockets[0]), "--verbosity", g_shell.shournalRunVerbosity, "--shell-session-uuid", g_shell.sessionInfo.uuid.toBase64().data() }; const char* tmpdir = getenv("TMPDIR"); if(tmpdir != nullptr){ args.push_back("--tmpdir"); args.push_back(tmpdir); } g_shell.lastMountNamespacePid = -1; subprocess::Subprocess subproc; subproc.setInNewSid(true); // Survive parent shell exit // Pass the socket to the external shournal process for communication purposes. std::unordered_set forwardFs {sockets[0]}; if(app::inIntegrationTestMode()){ // forward a pipe to async shournal so integration-test knows when it finished const char* pipeFdStr = getenv("_SHOURNAL_INTEGRATION_TEST_PIPE_FD"); if(pipeFdStr == nullptr){ QIErr() << "app is set to integration test mode, but pipe-fd is not set..."; } else { int pipeFd = qVariantTo_throw(QByteArray(pipeFdStr)); if(! osutil::fdIsOpen(pipeFd)){ QIErr() << "_SHOURNAL_INTEGRATION_TEST_PIPE_FD set in env " "but fd" << pipeFd << "is not open"; } else { forwardFs.insert(pipeFd); } } } subproc.setForwardFdsOnExec(forwardFs); subproc.call(args); logDebug << "launched" << BACKEND_FILENAME << "(pid" << subproc.lastPid() << ")"; os::close(sockets[0]); autocloseSocket0.setEnabled(false); // avoid deadlock: close our write end // wait for reply from shournal g_shell.shournalSocket.setReceiveBufferSize(100); g_shell.shournalSocket.setSockFd(sockets[1]); auto messages=g_shell.shournalSocket.receiveMessages(); if(messages.size() != 1 ){ logCritical << qtr("Setup of external %1-process failed: " "expected one message but received %2") .arg(BACKEND_FILENAME) .arg(messages.size()); return false; } auto& socketMsg = messages.first(); if( E_SocketMsg(socketMsg.msgId) != E_SocketMsg::SETUP_DONE){ QString msg = (socketMsg.msgId < 0 || socketMsg.msgId >= int(E_SocketMsg::ENUM_END)) ? qtr("Bad response") : socketMsgToStr(E_SocketMsg(socketMsg.msgId)); logCritical << qtr("Setup of external %1-process failed, " "received message: %2 (%3)") .arg(BACKEND_FILENAME) .arg(msg) .arg(int(socketMsg.msgId)); return false; } g_shell.lastMountNamespacePid = varFromQBytes(socketMsg.bytes, static_cast(-1)); assert(socketMsg.fd != -1); if(socketMsg.fd != g_shell.shournalRootDirFd){ os::dup2(socketMsg.fd, g_shell.shournalRootDirFd); os::close(socketMsg.fd); } auto RootDirFlags = os::getFdDescriptorFlags(g_shell.shournalRootDirFd); setBitIn(RootDirFlags, FD_CLOEXEC); os::setFdDescriptorFlags(g_shell.shournalRootDirFd, RootDirFlags); autocloseSocket1.setEnabled(false); if(sockets[1] != g_shell.shournalSocketNb ){ // dup2 and close orig try { os::dup2(sockets[1], g_shell.shournalSocketNb); close(sockets[1]); } catch (const os::ExcOs& ex) { logCritical << "duplicating to shournal-wait-fd failed: " << ex.what(); close(sockets[1]); return false; } } g_shell.shournalSocket.setSockFd(g_shell.shournalSocketNb); g_shell.shournalSockFdDescripFlags = os::getFdDescriptorFlags(g_shell.shournalSocketNb); g_shell.watchState = E_WatchState::WITHIN_CMD; shell_logger::flushBufferdMessages(); return true; } catch(const os::ExcOs& ex){ logCritical << ex.what(); } catch (const std::exception& e) { logCritical << "Unknown std::exception occurred: " << e.what() << "\n"; } catch (...) { logCritical << "Unknown exception occurred\n"; } g_shell.shournalSocket.setSockFd(-1); g_shell.shournalRootDirFd = -1; return false; } /// If the environment variable '_LIBSHOURNAL_TRIGGER' is set, /// perform the set action (load settings, launch external shournal, etc.). /// @param success If a valid shell request occured AND was successful this /// variable is set to true. /// @return the request which occurred or TRIGGER_UNSET/TRIGGER_MALFORMED. ShellRequest shell_request_handler::checkForTriggerAndHandle(bool *success){ *success = false; ShellRequest request = readCheckShellUpdateRequest(); switch (request) { case ShellRequest::TRIGGER_UNSET: case ShellRequest::TRIGGER_MALFORMED: return request; default: break; } // Interrupt protect mostly applies to waiting for a shournal response, which is // still short enough to justify not being interruptible. InterruptProtect ip(SIGINT); auto& g_shell = ShellGlobals::instance(); if(g_shell.pAttchedShell == nullptr){ // not initialized yet: only allow some requests: switch (request) { case ShellRequest::ENABLE: case ShellRequest::PRINT_VERSION: case ShellRequest::UPDATE_VERBOSITY: break; default: QIErr() << int(request) << "occurred, although the " "attached shell was not initialized (bug?)"; return request; } } if(! updateShellPID()){ return ShellRequest::TRIGGER_MALFORMED; } switch (request) { case ShellRequest::ENABLE: *success = handleEnableRequest(); break; case ShellRequest::DISABLE: *success = handleDisableRequest(); break; case ShellRequest::PREPARE_CMD: *success = handlePrepareCmd(); break; case ShellRequest::CLEANUP_CMD: *success = handleCleanupCmd(); break; case ShellRequest::PRINT_VERSION: QOut() << "libshournal-shellwatch.so version " << app::version().toString() << "\n"; *success = true; break; case ShellRequest::UPDATE_VERBOSITY: *success = ShellGlobals::updateVerbosityFromEnv(true); break; default: QIErr() << "BUG! Unhandeld request occurred:" << int(request); } return request; } ================================================ FILE: src/shell-integration-fanotify/shell_request_handler.h ================================================ #pragma once namespace shell_request_handler { /// ENABLE: shell observation enabled /// DISABLE: shell observation disabled /// PREPARE_CMD: prepare observing the next command-sequence /// CLEANUP_CMD: stop monitoring the command-sequence and send command-info to external shournal /// PRINT_VERSION: print the version of *this* shared library /// UPDATE_VERBOSITY: update the verbosity from environment /// TRIGGER_UNSET: The trigger is not set in the environment /// TRIGGER_MALFORMED: The trigger is set in the environment but malformed enum class ShellRequest { // To be used by the shell integration-scripts ENABLE, DISABLE, PREPARE_CMD, CLEANUP_CMD, PRINT_VERSION, UPDATE_VERBOSITY, // Internal use in this shared library TRIGGER_UNSET, TRIGGER_MALFORMED, ENUM_END}; ShellRequest checkForTriggerAndHandle(bool *success); bool handlePrepareCmd(); } // namespace shell_request_handler ================================================ FILE: src/shournal/CMakeLists.txt ================================================ add_executable(${PROJECT_NAME} ../../html-export/dist/htmlexportres.qrc shournal.cpp argcontrol_dbdelete.cpp argcontrol_dbquery.cpp command_printer.cpp command_printer_html.cpp command_printer_human.cpp command_printer_json.cpp cmd_stats.cpp ) # To keep dependencies low, only generate the main.js # when developing the html export. # To do so, set HTML_EXPORT_DEV to ON: # cmake -DHTML_EXPORT_DEV:BOOL=ON # npm and webpack must already be installed if (${HTML_EXPORT_DEV}) add_custom_target( build_htmlplot_npm WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/html-export COMMAND npm install --production COMMAND npm run build ) add_dependencies(${PROJECT_NAME} build_htmlplot_npm) endif() target_link_libraries(${PROJECT_NAME} lib_shournal_common ) install( TARGETS ${PROJECT_NAME} RUNTIME DESTINATION bin PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE ) ================================================ FILE: src/shournal/argcontrol_dbdelete.cpp ================================================ #include #include "argcontrol_dbdelete.h" #include "argcontrol_dbquery.h" #include "qoutstream.h" #include "qoptargparse.h" #include "qoptsqlarg.h" #include "database/db_controller.h" #include "database/query_columns.h" #include "cpp_exit.h" #include "app.h" using argcontol_dbquery::addVariantSqlArgToQueryIfParsed; using argcontol_dbquery::addSimpleSqlArgToQueryIfParsed; using db_controller::QueryColumns; void argcontrol_dbdelete::parse(int argc, char *argv[]) { QOptArgParse parser; parser.setHelpIntroduction(qtr( "Delete commands (and all corresponding file events) from the database by id or date.")); QOptSqlArg argCmdId("cmdid", "command-id", qtr("Deletes command with given id."), {E_CompareOperator::EQ} ); parser.addArg(&argCmdId); QOptSqlArg argCmdText("cmdtxt", "command-text", qtr("Delete commands with matching command-string."), QOptSqlArg::cmpOpsText()); parser.addArg(&argCmdText); QOptSqlArg argCmdCwd("cwd", "command-working-dir", qtr("Delete commands with matching working-directory."), QOptSqlArg::cmpOpsText()); parser.addArg(&argCmdCwd); QOptSqlArg argCmdDate("cmded", "command-end-date", qtr("Deletes commands given by end-date. Example:\n" "%1 --delete --command-end-date -between " "2019-04-01 2019-04-02\n" "deletes all commands which finished between " "the first and second of April 2019.").arg(app::SHOURNAL), QOptSqlArg::cmpOpsAllButLike() ); parser.addArg(&argCmdDate); QOptArg argCmdOlderThan("", "older-than", qtr("Delete commands older than the given number of " "years, months, days, etc.. Example:\n" "%1 --delete --older-than 3y\n" "deletes all commands which were executed more than " "three years ago.").arg(app::SHOURNAL)); argCmdOlderThan.setIsRelativeDateTime(true, true); parser.addArg(&argCmdOlderThan); QOptArg argCmdYoungerThan("", "younger-than", qtr("Delete commands younger than the given number of " "years, months, days, etc.. Example:\n" "%1 --delete --younger-than 1h\n" "deletes all commands which were executed within " "the last hour.").arg(app::SHOURNAL)); argCmdYoungerThan.setIsRelativeDateTime(true, true); parser.addArg(&argCmdYoungerThan); parser.parse(argc, argv); SqlQuery query; auto & cols = QueryColumns::instance(); addVariantSqlArgToQueryIfParsed(query, argCmdId, cols.cmd_id); addVariantSqlArgToQueryIfParsed(query, argCmdDate, cols.cmd_endtime); addSimpleSqlArgToQueryIfParsed(query, argCmdText, cols.cmd_txt); addSimpleSqlArgToQueryIfParsed(query, argCmdCwd, cols.cmd_workingDir); if(argCmdOlderThan.wasParsed()){ auto olderThanDates = argCmdOlderThan.getVariantRelativeDateTimes(); query.addWithAnd(cols.cmd_starttime, olderThanDates, E_CompareOperator::LT ); } if(argCmdYoungerThan.wasParsed()){ auto youngerThanDates = argCmdYoungerThan.getVariantRelativeDateTimes(); query.addWithAnd(cols.cmd_starttime, youngerThanDates, E_CompareOperator::GT ); } if( parser.rest().len != 0){ QIErr() << qtr("Invalid parameters passed: %1.\n" "Show help with --delete --help"). arg(argvToQStr(parser.rest().len, parser.rest().argv)); cpp_exit(1); } if(query.isEmpty()){ QIErr() << qtr("No target fields given (empty query)."); cpp_exit(1); } QOut() << qtr("%1 command(s) deleted.").arg( db_controller::deleteCommand(query)) << "\n"; cpp_exit(0); } ================================================ FILE: src/shournal/argcontrol_dbdelete.h ================================================ #pragma once namespace argcontrol_dbdelete { [[noreturn]] void parse(int argc, char *argv[]); } ================================================ FILE: src/shournal/argcontrol_dbquery.cpp ================================================ #include #include #include #include "argcontrol_dbquery.h" #include "qoptargparse.h" #include "qoptsqlarg.h" #include "database/db_globals.h" #include "database/db_controller.h" #include "database/query_columns.h" #include "database/file_query_helper.h" #include "database/db_conversions.h" #include "app.h" #include "logger.h" #include "qoutstream.h" #include "cpp_exit.h" #include "command_printer.h" #include "command_printer_human.h" #include "command_printer_json.h" #include "command_printer_html.h" #include "console_dialog.h" #include "osutil.h" #include "translation.h" #include "conversions.h" using translation::TrSnippets; using db_controller::QueryColumns; [[noreturn]] static void queryCmdPrintAndExit(std::unique_ptr& cmdPrinter, SqlQuery& sqlQ, bool reverseResultIter ){ auto results = db_controller::queryForCmd(sqlQ, reverseResultIter); cmdPrinter->printCommandInfosEvtlRestore(results); cpp_exit(0); } [[noreturn]] static void restoreSingleReadFile(QOptArg& argRestoreRfileId){ auto fReadInfo = db_controller::queryReadInfo_byId( static_cast(argRestoreRfileId.getValue()) ); if(fReadInfo.idInDb == db::INVALID_INT_ID){ QIErr() << qtr("cannot restore file - no database-entry exists"); cpp_exit(1); } if(! fReadInfo.isStoredToDisk){ QIErr() << qtr("cannot restore file %1 - only meta-information (path, name, etc.) " "about the file is stored in the database but not the " "file itself.").arg(fReadInfo.name); cpp_exit(1); } QDir currentDir = QDir::current(); if(QFile::exists(currentDir.absoluteFilePath(fReadInfo.name)) && osutil::isTTYForegoundProcess(STDIN_FILENO) && ! console_dialog::yesNo(qtr("File %1 exists. Replace?").arg(fReadInfo.name)) ) { cpp_exit(0); } StoredFiles().restoreReadFileAtDIr(fReadInfo, currentDir); QOut() << qtr("File '%1' restored at current working directory.").arg(fReadInfo.name) << "\n"; cpp_exit(0); } static void addFileQuery(SqlQuery &query, const QOptArg& argFile, const QOptArg& argTakeFromFile, bool readFile){ SqlQuery fQuery; if(argTakeFromFile.wasParsed()){ bool mtime=false; bool hash=false; bool size=false; for(auto opt : argTakeFromFile.getOptions()){ switch(opt[0].toLatin1()){ case 'm': mtime = true; break; case 'h': hash = true; break; case 's': size = true; break; default: throw QExcProgramming("Bad "+argFile.name()+" option: "+opt); } } fQuery = file_query_helper::buildFileQuery(argFile.getValue(), readFile, mtime, hash, size); } else { fQuery = file_query_helper::buildFileQuerySmart( argFile.getValue(), readFile); } query.addWithAnd(fQuery); } void argcontol_dbquery::addBytesizeSqlArgToQueryIfParsed(SqlQuery &query, QOptSqlArg &arg, const QString &tableCol) { if(! arg.wasParsed()) return; query.addWithAnd(tableCol, arg.getVariantByteSizes(), arg.parsedOperator() ); } void argcontol_dbquery::parse(int argc, char *argv[]) { QOptArgParse parser; const std::unordered_set &TAKE_FROM_FILE_OPTIONS { "mtime", "hash", "size"}; parser.setHelpIntroduction(qtr( "Query the command/file-database for several parameters which are\n" "AND-connected. For several fields optional comparison-operators are supported.\n" "The operators are passed in shell-friendly syntax so e.g. " "-gt stands for 'greater than'.\n" "-like will allow for using sql wildcards (e.g. '%').\n" "Examples:\n" "%1 --query --wfile /tmp/foo123 - use existing file to find out, how it was created.\n" "%1 --query --wsize -gt 10KiB - print all commands which have written to files whose " "size is greater than 10KiB.\n" "%1 --query --wpath -like /home/user% - print all commands, which have written to files " "below /home/user and all subdirectories.\n" ).arg(app::SHOURNAL) + "\n"); QOptArg argHistory("", "history", qtr("Only display the last N commands, you may optionally " "filter by other parameters as well (like command-text)") ); parser.addArg(&argHistory); // ------------ wfile QOptArg argWFile("wf", "wfile", qtr("Pass an existing file(-path) to find out the command, " "which caused the creation/modification of a given file " "(wfile stands for 'written file'). Per default the query is performed on " "the basis of hash(es), mtime and size." )); parser.addArg(&argWFile); QOptArg argTakeFromWFile("", "take-from-wfile", qtr("Specify explicitly which properties to collect " "from the given file passed via %1. " "Typically you do not need this.").arg(argWFile.name()) ); argTakeFromWFile.addRequiredArg(&argWFile); argTakeFromWFile.setAllowedOptions(TAKE_FROM_FILE_OPTIONS); parser.addArg(&argTakeFromWFile); const QString wFilePreamble = qtr("Query for files written to "); QOptSqlArg argWName("wn", "wname", wFilePreamble + qtr("by filename."), QOptSqlArg::cmpOpsText()); parser.addArg(&argWName); QOptSqlArg argWPath("wp", "wpath", wFilePreamble + qtr("by (full) directory-path."), QOptSqlArg::cmpOpsText(), E_CompareOperator::LIKE); parser.addArg(&argWPath); QOptSqlArg argWSize("ws", "wsize", wFilePreamble + qtr("by filesize."), QOptSqlArg::cmpOpsAllButLike() ); argWSize.setIsByteSizeArg(true); parser.addArg(&argWSize); QOptSqlArg argWHash("wh", "whash", wFilePreamble + qtr("by hash."), QOptSqlArg::cmpOpsEqNe() ); parser.addArg(&argWHash); QOptSqlArg argWMtime("wm", "wmtime", wFilePreamble + qtr("by mtime."), QOptSqlArg::cmpOpsAllButLike() ); parser.addArg(&argWMtime); // ------------ rfile QOptArg argRFile("rf", "rfile", qtr("Pass an existing file(-path) to find out the command(s), " "which read from it " "(rfile stands for 'read file'). Per default the query is performed on " "the basis of hash(es), mtime and size." )); parser.addArg(&argRFile); QOptArg argTakeFromRFile("", "take-from-rfile", qtr("Specify explicitly which properties to collect " "from the given file passed via %1. " "Typically you do not need this.").arg(argRFile.name()) ); argTakeFromRFile.addRequiredArg(&argRFile); argTakeFromRFile.setAllowedOptions(TAKE_FROM_FILE_OPTIONS); parser.addArg(&argTakeFromRFile); const QString rFilePreamble = qtr("Query for read files "); QOptSqlArg argRName("rn", "rname", rFilePreamble + qtr("by filename."), QOptSqlArg::cmpOpsText()); parser.addArg(&argRName); QOptSqlArg argRPath("rp", "rpath", rFilePreamble + qtr("by (full) directory-path."), QOptSqlArg::cmpOpsText(), E_CompareOperator::LIKE); parser.addArg(&argRPath); QOptSqlArg argRSize("rs", "rsize", rFilePreamble + qtr("by filesize."), QOptSqlArg::cmpOpsAllButLike() ); argRSize.setIsByteSizeArg(true); parser.addArg(&argRSize); QOptSqlArg argRHash("rh", "rhash", rFilePreamble + qtr("by hash."), QOptSqlArg::cmpOpsEqNe() ); parser.addArg(&argRHash); QOptSqlArg argRMtime("rm", "rmtime", rFilePreamble + qtr("by mtime."), QOptSqlArg::cmpOpsAllButLike() ); parser.addArg(&argRMtime); QOptArg argMaxReadFileLines("", "max-rfile-lines", qtr("Display at most the first N lines for each " "read file.") ); parser.addArg(&argMaxReadFileLines); QOptArg argRestoreRfiles("", "restore-rfiles", qtr("Restore read files for the found commands at the system's " "temporary directory."), false ); parser.addArg(&argRestoreRfiles); QOptArg argRestoreRfilesAt("", "restore-rfiles-at", qtr("Restore read files for the found commands at the given " "path.") ); parser.addArg(&argRestoreRfilesAt); QOptArg argRestoreRfileId("", "restore-rfile-id", qtr("Restore the read file with the given id at the working directory. " "Please note that id's are not necessarily in " "an ascending order.") ); parser.addArg(&argRestoreRfileId); // ------------ cmd QOptSqlArg argCmdText("cmdtxt", "command-text", qtr("Query for commands with matching command-string."), QOptSqlArg::cmpOpsText(), E_CompareOperator::LIKE); parser.addArg(&argCmdText); QOptSqlArg argCmdCwd("cwd", "command-working-dir", qtr("Query for commands with matching working-directory."), QOptSqlArg::cmpOpsText(), E_CompareOperator::LIKE); parser.addArg(&argCmdCwd); QOptSqlArg argCmdId("cmdid", "command-id", qtr("Query for commands with matching ids. " "Please note that id's are not necessarily in " "an ascending order."), QOptSqlArg::cmpOpsAllButLike()); parser.addArg(&argCmdId); QOptSqlArg argCmdEndDate("cmded", "command-end-date", qtr("Query for commands based on " "the date (time) they finished."), QOptSqlArg::cmpOpsAllButLike()); parser.addArg(&argCmdEndDate); // ------------ QOptSqlArg argShellSessionId("sid", "shell-session-id", qtr("Query for all commands with a given shell-session-id."), QOptSqlArg::cmpOpsEqNe()); parser.addArg(&argShellSessionId); const uint DEFAULT_wfilesMaxCount = 10; QOptArg argWfilesMaxCount("wfc", "wfiles-max-count", qtr("Limit the number of rendered written files " "per command (default is %1)").arg(DEFAULT_wfilesMaxCount)); parser.addArg(&argWfilesMaxCount); const uint DEFAULT_rfilesMaxCount = 10; QOptArg argRfilesMaxCount("rfc", "rfiles-max-count", qtr("Limit the number of rendered read files " "per command (default is %1)").arg(DEFAULT_rfilesMaxCount)); parser.addArg(&argRfilesMaxCount); QOptArg argOutputFile("o", "output", qtr("Specify an output file where the report " "is written to. Otherwise it is printed " "to stdout")); parser.addArg(&argOutputFile); QOptArg argOutputFormat("", "output-format", qtr("Specify the output format (human is default). " "If 'html' is used, %1 must also be specified") .arg(argOutputFile.name())); const char* OUTPUT_FORMAT_HUMAN = "human"; argOutputFormat.setAllowedOptions({OUTPUT_FORMAT_HUMAN, "json", "html"}); parser.addArg(&argOutputFormat); QOptArg argStatCounts("", "stat-counts", qtr("Specify the min. and max. number of entries " "for the overall statistics (e.g. commands with most file modifications) " "as a comma-separated list, e.g. '5,10' to display at least 5 but not more than" "10 entries.")); parser.addArg(&argStatCounts); QOptArg argFileStat("", "stat", qtr("Report the current status of files compared to the " "database as U (up to date), " "M (modified), N (not exist) ERROR (in case of an error) or " "NA (not queried, only using json)."), false); parser.addArg(&argFileStat); // --------------------- End of Args ----------------------- parser.parse(argc, argv); auto & trSnips = TrSnippets::instance(); SqlQuery query; std::unique_ptr cmdPrinter; if(argOutputFormat.wasParsed()){ switch(argOutputFormat.getOptions(1).first()[1].toLatin1()){ case 'u': cmdPrinter = std::unique_ptr(new CommandPrinterHuman); break; case 's': cmdPrinter = std::unique_ptr(new CommandPrinterJson);break; case 't': cmdPrinter = std::unique_ptr(new CommandPrinterHtml);break; default: throw QExcProgramming("Bad output format:" + argOutputFormat.getOptions(1).first()); } } else { cmdPrinter = std::unique_ptr(new CommandPrinterHuman); } cmdPrinter->setQueryString(argvToQStr(argc, argv)); cmdPrinter->setMaxCountWfiles(argWfilesMaxCount.getValue(DEFAULT_wfilesMaxCount)); cmdPrinter->setMaxCountRfiles(argRfilesMaxCount.getValue(DEFAULT_rfilesMaxCount)); { const auto statCounts = argStatCounts.getValuesByDelim >(",", {5,5}, 2,2); if(statCounts[0] > statCounts[1]){ throw ExcOptArgParse(qtr("argument %1: min. cannot be greater than max. stat-count") .arg(argStatCounts.name())); } cmdPrinter->setMinCountOfStats(statCounts[0]); cmdPrinter->cmdStats().setMaxCountOfStats(statCounts[1]); } if(argOutputFile.wasParsed()){ cmdPrinter->outputFile().setFileName(argOutputFile.getValue()); } else { if(dynamic_cast(cmdPrinter.get()) != nullptr){ QIErr() << qtr("For html-reports, please specify an output file " "(arg %1).").arg(argOutputFile.name()); cpp_exit(1); } cmdPrinter->outputFile().open(stdout, QFile::OpenModeFlag::WriteOnly); } if(argMaxReadFileLines.wasParsed()){ auto cmdPrinterHuman = dynamic_cast(cmdPrinter.get()); if(cmdPrinterHuman == nullptr){ QIErr() << qtr("Argument %1 is only allowed with output-format '%2'") .arg(argMaxReadFileLines.name(), OUTPUT_FORMAT_HUMAN); cpp_exit(1); } cmdPrinterHuman->setMaxCountOfReadFileLines(int(argMaxReadFileLines.getValue(5))); } cmdPrinter->setRestoreReadFiles(argRestoreRfiles.wasParsed() || argRestoreRfilesAt.wasParsed()); if(argRestoreRfilesAt.wasParsed()){ QDir restoreDir(argRestoreRfilesAt.getValue()); if(! restoreDir.exists()){ QIErr() << qtr("Restore directory %1 does not exist.").arg(restoreDir.absolutePath()); cpp_exit(1); } restoreDir.setPath(pathJoinFilename(restoreDir.absolutePath(), trSnips.shournalRestore)); cmdPrinter->setRestoreDir(restoreDir); } if(argRestoreRfileId.wasParsed()){ restoreSingleReadFile(argRestoreRfileId); } QueryColumns & cols = QueryColumns::instance(); addSimpleSqlArgToQueryIfParsed(query, argWName, cols.wFile_name); addSimpleSqlArgToQueryIfParsed(query, argWPath, cols.wFile_path); addBytesizeSqlArgToQueryIfParsed(query, argWSize, cols.wFile_size); if(argWHash.wasParsed()){ HashValue hashVal(argWHash.getValue()); query.addWithAnd(cols.wFile_hash, db_conversions::fromHashValue(hashVal), argWHash.parsedOperator()); } addVariantSqlArgToQueryIfParsed(query, argWMtime, cols.wFile_mtime); addSimpleSqlArgToQueryIfParsed(query, argRName, cols.rFile_name); addSimpleSqlArgToQueryIfParsed(query, argRPath, cols.rFile_path); addBytesizeSqlArgToQueryIfParsed(query, argRSize, cols.rFile_size); addVariantSqlArgToQueryIfParsed(query, argRMtime, cols.rFile_mtime); if(argRHash.wasParsed()){ HashValue hashVal(argRHash.getValue()); query.addWithAnd(cols.rFile_hash, db_conversions::fromHashValue(hashVal), argRHash.parsedOperator()); } addVariantSqlArgToQueryIfParsed(query, argCmdId, cols.cmd_id); addSimpleSqlArgToQueryIfParsed(query, argCmdText, cols.cmd_txt); addSimpleSqlArgToQueryIfParsed(query, argCmdCwd, cols.cmd_workingDir); addVariantSqlArgToQueryIfParsed(query, argCmdEndDate, cols.cmd_endtime); if(argShellSessionId.wasParsed()){ auto shellSessionUUID = QByteArray::fromBase64(argShellSessionId.getValue()); query.addWithAnd(cols.session_id, shellSessionUUID, argShellSessionId.parsedOperator()); } if(argWFile.wasParsed()){ addFileQuery(query, argWFile, argTakeFromWFile, false); } if(argRFile.wasParsed()){ addFileQuery(query, argRFile, argTakeFromRFile, true); } if(argFileStat.wasParsed()){ cmdPrinter->setReportFileStatus(true); } // we always display commands in startDate-order, however, // to allow for a performant history query (where the last // N entries are queried) we traverse the result-set from // end -> reverseResultIter = true AND query.ascending = false. bool reverseResultIter=false; // argHistory *must* be last, in case of an otherwise empty // query, accept all (where 1). if(argHistory.wasParsed()){ reverseResultIter = true; query.setAscending(false); query.setLimit(static_cast(argHistory.getValue())); if(query.isEmpty()){ // accept everything query.setQuery(" 1 "); } } if( parser.rest().len != 0){ QIErr() << qtr("Invalid parameters passed: «%1».\n" "Show help with --query --help"). arg(argvToQStr(parser.rest().len, parser.rest().argv)); cpp_exit(1); } if(query.isEmpty()){ QIErr() << qtr("No target fields given (empty query)."); cpp_exit(1); } queryCmdPrintAndExit(cmdPrinter, query, reverseResultIter); } ================================================ FILE: src/shournal/argcontrol_dbquery.h ================================================ #pragma once #include "sqlquery.h" #include "qoptsqlarg.h" namespace argcontol_dbquery { [[noreturn]] void parse(int argc, char *argv[]); template void addSimpleSqlArgToQueryIfParsed(SqlQuery& query, QOptSqlArg& arg, const QString& tableCol); template void addVariantSqlArgToQueryIfParsed(SqlQuery& query, QOptSqlArg& arg, const QString& tableCol); void addBytesizeSqlArgToQueryIfParsed(SqlQuery& query, QOptSqlArg& arg, const QString& tableCol); } template void argcontol_dbquery::addSimpleSqlArgToQueryIfParsed(SqlQuery& query, QOptSqlArg& arg, const QString& tableCol){ if(! arg.wasParsed()){ return; } query.addWithAnd(tableCol, arg.getValue(), arg.parsedOperator() ); } template void argcontol_dbquery::addVariantSqlArgToQueryIfParsed(SqlQuery& query, QOptSqlArg& arg, const QString& tableCol){ if(! arg.wasParsed()){ return; } query.addWithAnd(tableCol, arg.getVariantValues(), arg.parsedOperator() ); } ================================================ FILE: src/shournal/cmd_stats.cpp ================================================ #include "cmd_stats.h" #include "cleanupresource.h" /// Do not collect more than that many entries of each category CmdStats::CmdStats() : m_maxCountOfStats(5) { m_cmdsWithMostFileModsQueue.setMaxSize(m_maxCountOfStats); } void CmdStats::setMaxCountOfStats(const int &val) { m_maxCountOfStats = val; m_cmdsWithMostFileModsQueue.setMaxSize(val); } void CmdStats::collectCmd(const CommandInfo &cmd) { auto incrementIdxLater = finally([this] { ++m_currentCmdIdx; }); if(! cmd.fileWriteInfos.isEmpty()){ MostFileModsEntry mostFileMods; mostFileMods.idx = m_currentCmdIdx; mostFileMods.idInDb = cmd.idInDb; mostFileMods.cmdTxt = cmd.text; mostFileMods.countOfFileMods = cmd.fileWriteInfos.size(); m_cmdsWithMostFileModsQueue.push(mostFileMods); } if(! cmd.sessionInfo.uuid.isNull()){ auto & el = m_sessionMostCmdsMap[cmd.sessionInfo.uuid]; if(el.idx == -1){ // remember the first cmd in this session el.idx = m_currentCmdIdx; el.idInDb = cmd.idInDb; } el.cmdUuid = cmd.sessionInfo.uuid; ++el.cmdCount; } { auto & cwdCmdCountEntry = m_cwdCmdCountMap[cmd.workingDirectory]; ++cwdCmdCountEntry.cmdCount; } for(const auto& info : cmd.fileReadInfos){ auto & dirIoEntry = m_dirIoCountMap[info.path]; ++dirIoEntry.readCount; } for(const auto& info : cmd.fileWriteInfos){ auto & dirIoEntry = m_dirIoCountMap[info.path]; ++dirIoEntry.writeCount; } } /// aggregate the collected commands -> meant to be called, after all commands /// were collected. This function may be called only once as it clears /// afterwards not needed data. void CmdStats::eval() { m_cmdsWithMostFileMods = m_cmdsWithMostFileModsQueue.popAll(true); limited_priority_queue sessionMostCmdsPq; sessionMostCmdsPq.setMaxSize(m_maxCountOfStats); for(const auto & el : m_sessionMostCmdsMap){ sessionMostCmdsPq.push(el); } m_sessionMostCmds = sessionMostCmdsPq.popAll(true); m_sessionMostCmdsMap.clear(); limited_priority_queue cwdCmdCountQueue; cwdCmdCountQueue.setMaxSize(m_maxCountOfStats); for(auto it=m_cwdCmdCountMap.begin(); it != m_cwdCmdCountMap.end(); ++it){ // was not yet assigned because here it has to be assigned only once per // working dir it.value().workingDir = it.key(); cwdCmdCountQueue.push(it.value()); } m_cwdCmdCounts = cwdCmdCountQueue.popAll(true); m_cwdCmdCountMap.clear(); limited_priority_queue dirIoCountQueue; dirIoCountQueue.setMaxSize(m_maxCountOfStats); for(auto it=m_dirIoCountMap.begin(); it != m_dirIoCountMap.end(); ++it){ it.value().dir = it.key(); dirIoCountQueue.push(it.value()); } m_dirIoCounts = dirIoCountQueue.popAll(true); m_dirIoCountMap.clear(); } const CmdStats::MostFileModsEntrys &CmdStats::cmdsWithMostFileMods() const { return m_cmdsWithMostFileMods; } const CmdStats::SessionMostCmds &CmdStats::sessionMostCmds() const { return m_sessionMostCmds; } const CmdStats::CwdCmdCounts &CmdStats::cwdCmdCounts() const { return m_cwdCmdCounts; } const CmdStats::DirIoCounts &CmdStats::dirIoCounts() const { return m_dirIoCounts; } ================================================ FILE: src/shournal/cmd_stats.h ================================================ #pragma once #include #include #include "commandinfo.h" #include "limited_priority_queue.h" class CmdStats { public: // Commands which modified the most files struct MostFileModsEntry { int idx; // zero based collectCmd index (first command -> 0...) qint64 idInDb; QString cmdTxt; int countOfFileMods; }; // Sessions where the most commands where executed in struct SessionMostCmdsEntry { int idx {-1}; // idx of the first cmd of this session qint64 idInDb{-1}; // id of the first cmd of this session int cmdCount{0}; // number of commands executed in this session QByteArray cmdUuid; }; // Count of commands executed in // a specific CurrentWorkingDirectory struct CwdCmdCount { QString workingDir; int cmdCount{0}; }; // Directories, where the most files were read and written struct DirIoCount { QString dir; qint64 readCount{0}; qint64 writeCount{0}; }; typedef QVector MostFileModsEntrys; typedef QVector SessionMostCmds; typedef QVector CwdCmdCounts; typedef QVector DirIoCounts; public: CmdStats(); void setMaxCountOfStats(const int &val); void collectCmd(const CommandInfo& cmd); void eval(); const MostFileModsEntrys& cmdsWithMostFileMods() const; const SessionMostCmds& sessionMostCmds() const; const CwdCmdCounts& cwdCmdCounts() const; const DirIoCounts& dirIoCounts() const; private: struct cmpFileModEntry { bool operator()(const MostFileModsEntry & e1, const MostFileModsEntry & e2) { return e1.countOfFileMods > e2.countOfFileMods; } }; struct cmpSessionMostCmdEntry { bool operator()(const SessionMostCmdsEntry & e1, const SessionMostCmdsEntry & e2) { return e1.cmdCount > e2.cmdCount; } }; struct cmpCwdCmdCount { bool operator()(const CwdCmdCount & e1, const CwdCmdCount & e2) { return e1.cmdCount > e2.cmdCount; } }; struct cmpDirIoCount { bool operator()(const DirIoCount & e1, const DirIoCount & e2) { return e1.readCount + e1.writeCount > e2.readCount + e2.writeCount; } }; limited_priority_queue m_cmdsWithMostFileModsQueue; MostFileModsEntrys m_cmdsWithMostFileMods; QHash m_sessionMostCmdsMap; SessionMostCmds m_sessionMostCmds; QHash m_cwdCmdCountMap; CwdCmdCounts m_cwdCmdCounts; QHash m_dirIoCountMap; DirIoCounts m_dirIoCounts; int m_maxCountOfStats; int m_currentCmdIdx{0}; }; ================================================ FILE: src/shournal/command_printer.cpp ================================================ #include #include #include #include #include #include #include "command_printer.h" #include "qformattedstream.h" #include "util.h" #include "qfilethrow.h" #include "db_controller.h" #include "logger.h" #include "file_query_helper.h" #include "excos.h" #include "os.h" #include "translation.h" using translation::TrSnippets; static QString buildRestorePath(){ return pathJoinFilename( QStandardPaths::writableLocation(QStandardPaths::TempLocation), TrSnippets::instance().shournalRestore + "-" + os::getUserName() ); } CommandPrinter::CommandPrinter() : m_restoreDir(buildRestorePath()) {} void CommandPrinter::createRestoreTopleveDirIfNeeded() { if(m_countOfRestoredFiles == 0){ // initially create restore dir if(! m_restoreDir.mkpath(m_restoreDir.absolutePath())){ throw QExcIo(qtr("Failed to the create top-level read-files restore directory at %1") .arg(m_restoreDir.absolutePath()), false); } } } void CommandPrinter::restoreReadFile_safe(const FileReadInfo &readInfo, const QString &cmdIdStr) { QFileThrow f(m_storedFiles.mkPathStringToStoredReadFile(readInfo)); f.open(QFile::ReadOnly); restoreReadFile_safe(readInfo, cmdIdStr, f); } void CommandPrinter::restoreReadFile_safe(const FileReadInfo &readInfo, const QString &cmdIdStr, const QFile &openReadFile) { QDir fullDirPath( pathJoinFilename(m_restoreDir.absoluteFilePath(qtr("command-id-") + cmdIdStr) ,readInfo.path)); const QString failMsg(qtr("Failed to restore read file with id %1:").arg(readInfo.idInDb)); try { if(! fullDirPath.mkpath(fullDirPath.absolutePath())){ throw QExcIo(qtr("Failed to create the read-files restore directory for command-id %1") .arg(cmdIdStr)); } m_storedFiles.restoreReadFileAtDIr(readInfo, fullDirPath, openReadFile); ++m_countOfRestoredFiles; } catch (const os::ExcOs& e) { logWarning << failMsg << e.what(); } catch(const QExcIo& e){ logWarning << failMsg << e.descrip(); } } /// Do not output statistics, if less than 'val' entries void CommandPrinter::setMinCountOfStats(int val) { m_minCountOfStats = val; } void CommandPrinter::setReportFileStatus(bool val) { m_reportFileStatus = val; } bool CommandPrinter::reportFileStatus() const { return m_reportFileStatus; } void CommandPrinter::setMaxCountRfiles(int maxCountRfiles) { m_maxCountRfiles = maxCountRfiles; } CmdStats &CommandPrinter::cmdStats() { return m_cmdStats; } void CommandPrinter::setMaxCountWfiles(int maxCountWfiles) { m_maxCountWfiles = maxCountWfiles; } void CommandPrinter::setQueryString(const QString &queryString) { m_queryString = queryString; } void CommandPrinter::setRestoreDir(const QDir &restoreDir) { m_restoreDir = restoreDir; } QFileThrow &CommandPrinter::outputFile() { return m_outputFile; } void CommandPrinter::setRestoreReadFiles(bool restoreReadFiles) { m_restoreReadFiles = restoreReadFiles; } ================================================ FILE: src/shournal/command_printer.h ================================================ #pragma once #include #include "storedfiles.h" #include "conversions.h" #include "qfilethrow.h" #include "cmd_stats.h" class CommandQueryIterator; class QFormattedStream; /// Base class for command-printers (human, json). /// Print command-infos and corresponding file events to stdout. /// Since we can potentially stream only once over the sql-result, /// restore read files on the fly, if configured so. class CommandPrinter { public: CommandPrinter(); virtual ~CommandPrinter() = default; virtual void printCommandInfosEvtlRestore(std::unique_ptr& cmdIter) = 0; virtual void setRestoreReadFiles(bool restoreReadFiles); virtual void setRestoreDir(const QDir &restoreDir); virtual QFileThrow& outputFile(); virtual void setQueryString(const QString &queryString); virtual void setMaxCountWfiles(int maxCountWfiles); virtual void setMaxCountRfiles(int maxCountRfiles); virtual CmdStats& cmdStats(); virtual void setMinCountOfStats(int val); virtual void setReportFileStatus(bool val); virtual bool reportFileStatus() const; protected: Q_DISABLE_COPY(CommandPrinter) void createRestoreTopleveDirIfNeeded(); void restoreReadFile_safe(const FileReadInfo& readInfo, const QString &cmdIdStr); void restoreReadFile_safe(const FileReadInfo& readInfo, const QString &cmdIdStr, const QFile &openReadFile); StoredFiles m_storedFiles; bool m_restoreReadFiles {false}; int m_countOfRestoredFiles {0}; QDir m_restoreDir; QFileThrow m_outputFile; QString m_queryString; // entered by user on commandline int m_maxCountWfiles{0}; // do not print more than that number of written files per command int m_maxCountRfiles{0}; CmdStats m_cmdStats; int m_minCountOfStats; bool m_reportFileStatus{false}; }; ================================================ FILE: src/shournal/command_printer_html.cpp ================================================ #include #include #include #include #include #include #include #include "command_printer_html.h" #include "command_query_iterator.h" #include "logger.h" #include "util.h" #include "cleanupresource.h" #include "stupidinject.h" #include "qresource_helper.h" #include "qoutstream.h" using qresource_helper::data_safe; void CommandPrinterHtml::printCommandInfosEvtlRestore(std::unique_ptr &cmdIter) { if( cmdIter->computeSize() == 0){ QOut() << qtr("No results found matching the query.\n"); return; } Q_INIT_RESOURCE(htmlexportres); QResource indexHtmlResource("://index.html"); QByteArray html_content = data_safe(indexHtmlResource); if(! m_outputFile.isOpen()){ m_outputFile.open(QFile::OpenModeFlag::WriteOnly); } QTextStream outstream(&m_outputFile); StupidInject inject; QTemporaryFile tmpCmdDataFile; tmpCmdDataFile.open(); FileReadInfoSet_t readFileIdSet; inject.addInjection( "", [this, &cmdIter, &tmpCmdDataFile, &readFileIdSet](QTextStream& outstream){ // json performance much better-> embed into html outstream << R"(\n"; // store the rest as plain js: outstream << "\n"; }); inject.addInjection("", [this, &tmpCmdDataFile, &readFileIdSet](QTextStream& outstream){ // currently the whole js is compiled into a single main.js file, // which we inject here for convenience. QResource mainJSResource("://main.js"); QByteArray mainJsContent = data_safe(mainJSResource); outstream << "\n"; // write the cmd-data of the tempfile to html if(! tmpCmdDataFile.seek(0)){ throw QExcIo("Failed to seek to 0 in cmdData tmpfile: " + tmpCmdDataFile.errorString()); } QByteArray line; uint linecounter = 0; while(! (line = tmpCmdDataFile.readLine()).isEmpty()){ // pop \n line.resize(line.size() - 1); outstream << "\n"; ++linecounter; } outstream << "\n\n"; }); inject.stream(html_content, outstream); } void CommandPrinterHtml::processSingleCommand(QTextStream& outstream, CommandInfo& cmd, QDateTime& finalCommandEndDate, QTemporaryFile &tmpCmdDataFile){ m_cmdStats.collectCmd(cmd); if(cmd.startTime.msecsTo(cmd.endTime) < 1){ // for the plot we need at least one millisecond time difference to draw a rect: cmd.endTime = cmd.endTime.addMSecs(1); } finalCommandEndDate = std::max(finalCommandEndDate, cmd.endTime); // to speed up loading of the html document (especially useful // for more than 2000 entries), we 'split' up the command // data, to first (quickly) render the session time-line and // load the rest afterwards. For the timeline and command-list, only id, start/end-date, uuid and // text are necessary. writeCmdStartup(cmd, outstream); // write the 'rest' to tempfile, line by line writeCmdData(cmd, tmpCmdDataFile); } void CommandPrinterHtml::writeCmdStartup(const CommandInfo &cmd, QTextStream &outstream) { QJsonObject jsonCmdStartup; CmdJsonWriteCfg cmdJsonStartup(false); cmdJsonStartup.maxCountRFiles = m_maxCountRfiles; cmdJsonStartup.maxCountWFiles = m_maxCountWfiles; cmdJsonStartup.idInDb = true; cmdJsonStartup.startEndTime = true; cmdJsonStartup.sessionInfo = true; cmdJsonStartup.text = true; cmd.write(jsonCmdStartup, m_writeDatesWithMillisec, cmdJsonStartup); outstream << QJsonDocument(jsonCmdStartup).toJson(QJsonDocument::Compact); } void CommandPrinterHtml::writeCmdData(const CommandInfo &cmd, QTemporaryFile &tmpCmdDataFile) { CmdJsonWriteCfg cmdJsonData(true); cmdJsonData.maxCountRFiles = m_maxCountRfiles; cmdJsonData.maxCountWFiles = m_maxCountWfiles; cmdJsonData.idInDb = false; cmdJsonData.startEndTime = false; cmdJsonData.sessionInfo = false; cmdJsonData.text = false; QJsonObject jsonCmdData; cmd.write(jsonCmdData, m_writeDatesWithMillisec, cmdJsonData); // since we may restrict the number of read/written files (to not generate // huge html-files), store the real number in any case: jsonCmdData["fileReadEvents_length"] = cmd.fileReadInfos.length(); jsonCmdData["fileWriteEvents_length"] = cmd.fileWriteInfos.length(); if(tmpCmdDataFile.write(QJsonDocument(jsonCmdData).toJson(QJsonDocument::Compact)) == -1){ throw QExcIo("Failed to write cmdData to tmpfile: " + tmpCmdDataFile.errorString()); } tmpCmdDataFile.write("\n"); } void CommandPrinterHtml::addScriptsToReadFilesSet(const FileReadInfos &infos, FileReadInfoSet_t &set) { int counter = 0; for(const auto& info : infos){ if(info.isStoredToDisk){ // don't check mimetype here, to avoid performing it multiple times // for the same script-id set.insert(info.idInDb); } ++counter; if(counter > m_maxCountRfiles){ break; } } } void CommandPrinterHtml::writeReadFileContentsToHtml(QTextStream &outstream, FileReadInfoSet_t &readFileIdSet) { // uniquely store each script in the html file outstream << "const readFileContentMap = new Map([\n"; auto autoCloseNewMap = finally([&outstream] { outstream << "]);\n"; }); for(const auto& id_ : readFileIdSet) { // javascript Maps can take 2d arrays in the constructor. // Each array entry has the format [key, value]. outstream << "[" << id_ << ","; auto autoCloseBracket = finally([&outstream] { outstream << "],\n"; }); QFileThrow f(m_storedFiles.mkPathStringToStoredReadFile(id_)); try { f.open(QFile::OpenModeFlag::ReadOnly); auto mtype = m_mimedb.mimeTypeForData(&f); if(! mtype.inherits("text/plain")){ outstream << "null"; // don't use 'undefined' here! continue; } outstream << "\""; auto autoSetQuote = finally([&outstream] { outstream << "\""; }); writeFileToStream(f, outstream); } catch (const QExcIo& e) { logWarning << qtr("Error writing read file with id %1 to html: %2") .arg(id_).arg(e.descrip()); } } } void CommandPrinterHtml::writeFileToStream(QFileThrow &f, QTextStream &outstream) { const int BUFSIZE = 9000; // MUST be divisible by 3, so we create no padding '=' // between base64-chunks (; char buf[BUFSIZE]; qint64 readCount; while( (readCount = f.read(buf, BUFSIZE)) > 0 ){ // we could be writing anything here to the js file - KISS, and use base64 outstream << QByteArray::fromRawData(buf, int(readCount)).toBase64(); } } void CommandPrinterHtml::writeStatistics(QTextStream &outstream) { { QJsonArray jsonMostFileMods; if(m_cmdStats.cmdsWithMostFileMods().size() >= m_minCountOfStats) { for(const auto& e : m_cmdStats.cmdsWithMostFileMods()){ QJsonObject o; o["idx"] = e.idx; o["countOfFileMods"] = e.countOfFileMods; jsonMostFileMods.append(o); } } outstream << "const mostFileMods = " << QJsonDocument(jsonMostFileMods).toJson(QJsonDocument::Compact) << "\n"; } { QJsonArray jsonSessionsMostCmds; if(m_cmdStats.sessionMostCmds().size() >= m_minCountOfStats) { for(const auto & e : m_cmdStats.sessionMostCmds()){ QJsonObject o; o["idxFirstCmd"] = e.idx; o["countOfCommands"] = e.cmdCount; jsonSessionsMostCmds.append(o); } } outstream << "const sessionsMostCmds = " << QJsonDocument(jsonSessionsMostCmds).toJson(QJsonDocument::Compact) << "\n"; } { QJsonArray json; if(m_cmdStats.cwdCmdCounts().size() >= m_minCountOfStats) { for(const auto & e : m_cmdStats.cwdCmdCounts()){ QJsonObject o; o["workingDir"] = e.workingDir; o["countOfCommands"] = e.cmdCount; json.append(o); } } outstream << "const cwdCmdCounts = " << QJsonDocument(json).toJson(QJsonDocument::Compact) << "\n"; } { QJsonArray json; if(m_cmdStats.dirIoCounts().size() >= m_minCountOfStats) { for(const auto & e : m_cmdStats.dirIoCounts()){ QJsonObject o; o["dir"] = e.dir; o["readCount"] = e.readCount; o["writeCount"] = e.writeCount; json.append(o); } } outstream << "const dirIoCounts = " << QJsonDocument(json).toJson(QJsonDocument::Compact) << "\n"; } } ================================================ FILE: src/shournal/command_printer_html.h ================================================ #pragma once #include #include #include "command_printer.h" #include "fileinfos.h" #include "commandinfo.h" #include "qfilethrow.h" #include "cmd_stats.h" class QTextStream; class QTemporaryFile; class CommandPrinterHtml : public CommandPrinter { public: CommandPrinterHtml() = default; void printCommandInfosEvtlRestore(std::unique_ptr& cmdIter) override; protected: Q_DISABLE_COPY(CommandPrinterHtml) typedef std::unordered_set FileReadInfoSet_t; void processSingleCommand(QTextStream& outstream, CommandInfo& cmd, QDateTime& finalCommandEndDate, QTemporaryFile& tmpCmdDataFile); void writeCmdStartup(const CommandInfo& cmd, QTextStream& outstream); void writeCmdData(const CommandInfo& cmd, QTemporaryFile& tmpCmdDataFile); void addScriptsToReadFilesSet(const FileReadInfos& infos, FileReadInfoSet_t& set); void writeReadFileContentsToHtml(QTextStream& outstream, FileReadInfoSet_t& readFileIdSet); void writeFileToStream(QFileThrow& f, QTextStream& outstream); void writeStatistics(QTextStream& outstream); QMimeDatabase m_mimedb; bool m_writeDatesWithMillisec{true}; }; ================================================ FILE: src/shournal/command_printer_human.cpp ================================================ #include "command_printer_human.h" #include #include #include #include #include #include #include #include #include "command_printer.h" #include "qformattedstream.h" #include "util.h" #include "qfilethrow.h" #include "db_controller.h" #include "logger.h" #include "file_query_helper.h" #include "excos.h" #include "os.h" #include "translation.h" #include "qoutstream.h" #include "commandinfo.h" void CommandPrinterHuman::printCommandInfosEvtlRestore(std::unique_ptr &cmdIter) { if( cmdIter->computeSize() == 0){ QOut() << qtr("No results found matching the query.\n"); return; } if(! m_outputFile.isOpen()){ m_outputFile.open(QFile::OpenModeFlag::WriteOnly); } QFormattedStream s(&m_outputFile); struct winsize termWinSize{}; if(ioctl(STDOUT_FILENO, TIOCGWINSZ, &termWinSize) == 0){ s.setMaxLineWidth((termWinSize.ws_col > 5) ? termWinSize.ws_col : 80 ); } else { // this happens e.g. when the output is piped to grep logDebug << "failed to determine terminal size, using max..."; s.setMaxLineWidth( std::numeric_limits::max() ); } const QString currentHostname = QHostInfo::localHostName(); while(cmdIter->next()){ m_cmdStats.collectCmd(cmdIter->value()); s.setLineStart(m_indentlvl0); // for indentlvl0 line-word-wrapping makes almost no // sense and hinders copy-pasting of long terminal commands. auto oldMaxLineWidth = s.maxLineWidth(); s.setMaxLineWidth(std::numeric_limits::max()); auto & cmd = cmdIter->value(); s << qtr("cmd-id %1").arg(cmd.idInDb); if(cmd.returnVal != CommandInfo::INVALID_RETURN_VAL){ s << qtr("$?=%1").arg(QString::number(cmd.returnVal)); } // Only print end-date/time, if different from start-date/time, ignoring seconds. dropFromTime(cmd.startTime, 's'); dropFromTime(cmd.endTime, 's'); const QString tformat = "hh:mm"; const QString dtformat = QString("yyyy-MM-dd") + " " + tformat; QString cmdEndTime = (cmd.startTime.date() != cmd.endTime.date()) ? " - " + cmd.endTime.toString(dtformat) : (cmd.startTime.time() != cmd.endTime.time()) ? "-" + cmd.endTime.time().toString(tformat) : ""; s << cmd.startTime.toString(dtformat) + cmdEndTime << "$" << cmd.text << "\n"; s << qtr("Working directory: %1\n").arg(cmd.workingDirectory); if(! cmd.sessionInfo.uuid.isNull()){ s << qtr("session-uuid") << cmd.sessionInfo.uuid.toBase64() << "\n"; } if(cmd.hostname != currentHostname){ s << qtr("Hostname: %1\n").arg(cmd.hostname); } s.setMaxLineWidth(oldMaxLineWidth); printWriteInfos(cmd, s); printReadInfos(s, cmd); } m_cmdStats.eval(); if(m_cmdStats.cmdsWithMostFileMods().size() >= m_minCountOfStats){ s.setLineStart(m_indentlvl0); s << qtr("\nCommands with most file modifications:\n"); s.setLineStart(m_indentlvl1); for(const auto& e : m_cmdStats.cmdsWithMostFileMods()){ s << qtr("cmd-id %1 modified %2 file(s) - %3\n") .arg(e.idInDb).arg(e.countOfFileMods).arg(e.cmdTxt); } } if(m_cmdStats.sessionMostCmds().size() >= m_minCountOfStats){ s.setLineStart(m_indentlvl0); s << qtr("\nSessions with most commands:\n"); s.setLineStart(m_indentlvl1); for(const auto& e : m_cmdStats.sessionMostCmds()){ s << qtr("session-uuid %1 - %2 command(s)\n") .arg(e.cmdUuid.toBase64().data()).arg(e.cmdCount); } } if(m_cmdStats.cwdCmdCounts().size() >= m_minCountOfStats){ s.setLineStart(m_indentlvl0); s << qtr("\nWorking directories with most commands:\n"); s.setLineStart(m_indentlvl1); for(const auto& e : m_cmdStats.cwdCmdCounts()){ s << qtr("%1 command(s) at %2\n") .arg(e.cmdCount).arg(e.workingDir); } } if(m_cmdStats.dirIoCounts().size() >= m_minCountOfStats){ s.setLineStart(m_indentlvl0); s << qtr("\nDirectories with most input/output-activity:\n"); s.setLineStart(m_indentlvl1); for(const auto& e : m_cmdStats.dirIoCounts()){ s << qtr("Total %1 (%2 written, %3 read) files at %4\n") .arg(e.writeCount + e.readCount).arg(e.writeCount) .arg(e.readCount).arg(e.dir); } } if(m_countOfRestoredFiles > 0){ s.setLineStart(m_indentlvl0); s << qtr("%1 file(s) restored at %2").arg(m_countOfRestoredFiles) .arg(m_restoreDir.absolutePath()) << "\n"; } } void CommandPrinterHuman::printReadFileEventEvtlRestore (const CommandInfo &cmd, QFormattedStream& s, const FileReadInfo& f, const QString &cmdIdStr){ auto fStatus = (reportFileStatus()) ? " "+f.currentStatus(cmd) : ""; s.setLineStart(m_indentlvl2); s << pathJoinFilename(f.path, f.name) << "(" + m_userStrConv.bytesToHuman(f.size) + ")" << qtr("Hash:") << ((f.hash.isNull()) ? "-" : QString::number(f.hash.value())) << "id" << QString::number(f.idInDb) + fStatus + "\n"; if(! f.isStoredToDisk){ // since shournal 2.1 it is possible to log only meta-information about // read files without storing them in the read files dir. return; } if(m_restoreReadFiles){ createRestoreTopleveDirIfNeeded(); } bool printFileContentSuccess {false}; QFileThrow file(m_storedFiles.mkPathStringToStoredReadFile(f)); try { file.open(QFile::OpenModeFlag::ReadOnly); auto mtype = m_mimedb.mimeTypeForData(&file); s.setLineStart(m_indentlvl3); if(! mtype.inherits("text/plain")){ s << qtr("Not printing content (mimetype %1)").arg(mtype.name()) << "\n"; return; } printReadFile(s, file); printFileContentSuccess = true; if(m_restoreReadFiles){ restoreReadFile_safe(f, cmdIdStr, file); } } catch (const QExcIo& e) { if(printFileContentSuccess){ logWarning << e.what(); } else { logWarning << qtr("Error while printing read file '%1' with id %2: %3") .arg(f.name).arg(f.idInDb).arg(e.descrip()); } return; } } void CommandPrinterHuman::printReadFile(QFormattedStream &s, QFile &f) { QTextStream fstream(&f); int nLinesPrinted = 0; while(! fstream.atEnd()){ QString line = fstream.readLine(); if(line.isEmpty()){ continue; } s << line << "\n"; if(++nLinesPrinted >= m_maxCountOfReadFileLines){ s << "...\n"; break; } } } void CommandPrinterHuman::printWriteInfos (const CommandInfo& cmd, QFormattedStream &s) { auto & fileWriteInfos = cmd.fileWriteInfos; if(fileWriteInfos.isEmpty()){ return; } s.setLineStart(m_indentlvl1); const char dotOrColon = (m_maxCountWfiles == 0) ? '.' : ':'; const QString fileStr = (fileWriteInfos.size() == 1) ? "file" : "files"; s << qtr("%1 written %2%3\n").arg(fileWriteInfos.size()).arg(fileStr).arg(dotOrColon); s.setLineStart(m_indentlvl2); int counter = 0; for(const auto& f : fileWriteInfos){ if(counter >= m_maxCountWfiles){ if(counter > 0){ s << qtr("... and %1 more files.\n") .arg(fileWriteInfos.size() - m_maxCountWfiles); } break; } auto fStatus = (reportFileStatus()) ? " "+f.currentStatus(cmd) : ""; s << pathJoinFilename(f.path, f.name) << "(" + m_userStrConv.bytesToHuman(f.size) + ")" << qtr("Hash:") << ((f.hash.isNull()) ? "-" : QString::number(f.hash.value())) + fStatus + "\n"; ++counter; } } void CommandPrinterHuman::printReadInfos(QFormattedStream &s, const CommandInfo &cmd) { if(cmd.fileReadInfos.isEmpty()){ return; } s.setLineStart(m_indentlvl1); const char dotOrColon = (m_maxCountRfiles == 0) ? '.' : ':'; const QString fileStr = (cmd.fileReadInfos.size() == 1) ? "file" : "files"; s << qtr("%1 read %2%3\n").arg(cmd.fileReadInfos.size()).arg(fileStr).arg(dotOrColon); const QString cmdIdStr = QString::number(cmd.idInDb); int counter = 0; for(const auto & f : cmd.fileReadInfos){ if(counter >= m_maxCountRfiles){ if(counter > 0){ s << qtr("... and %1 more files.\n") .arg(cmd.fileReadInfos.size() - m_maxCountRfiles); } break; } printReadFileEventEvtlRestore(cmd, s, f, cmdIdStr); ++counter; } } /// Do not print more than that number of lines for each read file void CommandPrinterHuman::setMaxCountOfReadFileLines(int maxCountOfReadFileLines) { m_maxCountOfReadFileLines = maxCountOfReadFileLines; } ================================================ FILE: src/shournal/command_printer_human.h ================================================ #pragma once #include "command_printer.h" #include "commandinfo.h" #include class CommandPrinterHuman : public CommandPrinter { public: CommandPrinterHuman() = default; void printCommandInfosEvtlRestore(std::unique_ptr& cmdIter) override; virtual void setMaxCountOfReadFileLines(int maxCountOfReadFileLines); protected: Q_DISABLE_COPY(CommandPrinterHuman) void printReadFileEventEvtlRestore(const CommandInfo &cmd, QFormattedStream& s, const FileReadInfo& readInfo, const QString& cmdIdStr); void printReadFile(QFormattedStream& s, QFile& f); void printWriteInfos(const CommandInfo &cmd, QFormattedStream& s); void printReadInfos(QFormattedStream& s, const CommandInfo& cmd); QMimeDatabase m_mimedb; const QString m_indentlvl0 {""}; const QString m_indentlvl1 {" "}; const QString m_indentlvl2 {" "}; const QString m_indentlvl3 {" "}; int m_maxCountOfReadFileLines {5}; Conversions m_userStrConv; }; ================================================ FILE: src/shournal/command_printer_json.cpp ================================================  #include #include #include "command_printer_json.h" #include "command_query_iterator.h" #include "logger.h" #include "util.h" void CommandPrinterJson::printCommandInfosEvtlRestore(std::unique_ptr &cmdIter) { if(! m_outputFile.isOpen()){ m_outputFile.open(QFile::OpenModeFlag::WriteOnly); } QTextStream outstream(&m_outputFile); { QJsonObject header; header["pathToReadFiles"] = StoredFiles::getReadFilesDir(); QJsonDocument doc(header); outstream << "HEADER:" << doc.toJson(QJsonDocument::Compact) << "\n"; } CmdJsonWriteCfg jsonCfg(true); jsonCfg.fileStatus = this->reportFileStatus(); while(cmdIter->next()){ QJsonObject cmdObject; cmdIter->value().write(cmdObject, false, jsonCfg); QJsonDocument doc(cmdObject); outstream << "COMMAND:" << doc.toJson(QJsonDocument::Compact) << "\n"; if(! m_restoreReadFiles){ continue; } for(const auto& readInfo : cmdIter->value().fileReadInfos){ if(readInfo.isStoredToDisk){ createRestoreTopleveDirIfNeeded(); restoreReadFile_safe(readInfo, QString::number(cmdIter->value().idInDb)); } } } { QJsonObject footer; footer["restorePath"] =QJsonValue::fromVariant( (m_countOfRestoredFiles == 0) ? QVariant() : m_restoreDir.absolutePath() ); footer["countOfRestoredFiles"] = m_countOfRestoredFiles; QJsonDocument doc(footer); outstream << "FOOTER:" << doc.toJson(QJsonDocument::Compact) << "\n"; } } ================================================ FILE: src/shournal/command_printer_json.h ================================================ #pragma once #include "command_printer.h" class CommandPrinterJson : public CommandPrinter { public: CommandPrinterJson() = default; void printCommandInfosEvtlRestore(std::unique_ptr& cmdIter) override; private: Q_DISABLE_COPY(CommandPrinterJson) }; ================================================ FILE: src/shournal/shournal.cpp ================================================ #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include #include #include #include #include #include "qoptargparse.h" #include "excoptargparse.h" #include "excos.h" #include "logger.h" #include "os.h" #include "exccfg.h" #include "cpp_exit.h" #include "settings.h" #include "db_connection.h" #include "util.h" #include "cleanupresource.h" #include "qoutstream.h" #include "util.h" #include "translation.h" #include "app.h" #include "qsqlquerythrow.h" #include "argcontrol_dbquery.h" #include "argcontrol_dbdelete.h" #include "qexcdatabase.h" #include "conversions.h" #include "console_dialog.h" #include "qfilethrow.h" #include "shournal_run_common.h" using namespace shournal_run_common; /// Uncaught exception handler void onterminate() { try { auto unknown = std::current_exception(); if (unknown) { std::rethrow_exception(unknown); } } catch (const std::exception& e) { logCritical << e.what() << "\n"; } catch (...) { logCritical << "unknown exception occurred\n"; } } [[noreturn]] void execShournalRun(const QByteArray& backendFilename, QOptArg::RawValues_t &cargs, bool withinOrigMountspace, QVarLengthArray forwardArgs); int shournal_main(int argc, char *argv[]) { app::setupNameAndVersion(app::SHOURNAL); if(! translation::init()){ QIErr() << "Failed to initialize translation"; } logger::setup(app::CURRENT_NAME); std::set_terminate(onterminate); if(! shournal_common_init()){ logCritical << qtr("Fatal error: failed to initialize custom Qt conversion functions"); cpp_exit(1); } // ignore first arg (command to this app) --argc; ++argv; auto & sets = Settings::instance(); QOptArgParse parser; parser.setHelpIntroduction(qtr("Launch a command and recursively observe in specific " "directories which files " "were modified by that process and its children. ") + "\n"); QOptArg argVersion("v", "version", qtr("Display version"), false); parser.addArg(&argVersion); QOptArg argExec("e", "exec", qtr("Execute and observe the passed program " "and its arguments (this argument has to be last). " "All further parameters starting with a minus " "are considered options for the shournal-run* backend until " "double dash -- or the first command " "(not starting with a minus) occurs, e.g.\n" "shournal -e --print-summary echo foobar\n" "-> --print-summary is an argument for shournal-run.\n" "shournal --exec-filename /bin/bash -e -- -bash\n" "on the other hand can be used for commands starting " "with a dash (e.g. login-shells)."), false); argExec.setFinalizeFlag(true); parser.addArg(&argExec); QOptArg argExecFilename("", "exec-filename", qtr("This is an advanced option. " "In most cases the first argument of a " "program is the program name. For " "example for login-shells this does " "not have to be the case. If this " "argument is provided, that filename " "is used instead of argv[0]")); parser.addArg(&argExecFilename); argExecFilename.addRequiredArg(&argExec); QOptArg argBackend("", "backend-filename", qtr("When executing a command (option %1) use " "the given filename as observation backend-command") .arg(argExec.name())); parser.addArg(&argBackend); argBackend.addRequiredArg(&argExec); QOptArg argMsenterOrig("", "msenter-orig-mountspace", qtr("Must be passed along with '%1'. Execute the " "given command in the 'original' mount-namespace " "created the first time %2 observed a process.") .arg(argExec.name(), app::SHOURNAL_RUN), false); argMsenterOrig.setInternalOnly(true); argMsenterOrig.addRequiredArg(&argExec); parser.addArg(&argMsenterOrig); QOptArg argEditCfg("c", "edit-cfg", qtr("Edit the config-file at %1 " "with your favourite text-editor:\n" "export EDITOR='...'").arg(sets.cfgFilepath()), false); parser.addArg(&argEditCfg); QOptArg argQuery("q", "query", qtr("Query %1's database for activities. Type " "--query --help for details.").arg(app::SHOURNAL), false); argQuery.setFinalizeFlag(true); parser.addArg(&argQuery); QOptArg argDelete("", "delete", qtr("delete (parts of) %1's command history from the" " database. Type " "--delete --help for details.").arg(app::SHOURNAL), false); argDelete.setFinalizeFlag(true); parser.addArg(&argDelete); QOptArg argPrintMime("", "print-mime", qtr("Print the mimetpye of an existing file(name) " "which can be used in shournal's config-" "file for setting file-event-rules.")); parser.addArg(&argPrintMime); QOptArg argVerbosity("", "verbosity", qtr("How much shall be printed to stderr. Note that " "for 'dbg' shournal must not be a 'Release'-build, " "dbug-messages are lost in Release-mode.")); argVerbosity.setAllowedOptions(app::VERBOSITIES); parser.addArg(&argVerbosity); QOptArg argValidateSettings("", "validate-settings", qtr("If the settings-file is well formed, " "return 0, else print an error and return " "a nonzero value"), false); parser.addArg(&argValidateSettings); QOptArg argLsOurPaths("", "ls-paths", qtr("Print shournal's application paths (database-dir, etc.)"), false); parser.addArg(&argLsOurPaths); auto argCfgDir = mkarg_cfgdir(); parser.addArg(&argCfgDir); auto argDataDir = mkarg_datadir(); parser.addArg(&argDataDir); // Forward these to shournal-run: QVarLengthArray forwardArgs = {&argVerbosity, &argExecFilename, &argCfgDir, &argDataDir}; try { parser.parse(argc, argv); auto & sets = Settings::instance(); if(argCfgDir.wasParsed()){ sets.setUserCfgDir(argCfgDir.getValue()); } if(argDataDir.wasParsed()){ sets.setUserDataDir(argDataDir.getValue()); } if(argVerbosity.wasParsed()){ QByteArray verbosity = argVerbosity.getOptions(1).first().toLocal8Bit(); logger::setVerbosityLevel(verbosity.constData()); } else { logger::setVerbosityLevel(QtMsgType::QtWarningMsg); } if(argExec.wasParsed()){ auto backendFilename = argBackend.getValue(); if(backendFilename.isEmpty()){ backendFilename = sets.chooseShournalRunBackend(); if(backendFilename.isEmpty()){ QIErr() << qtr("No backend-filename given and no valid " "backend found - exiting..."); cpp_exit(1); } } execShournalRun(backendFilename.toLocal8Bit(), parser.rest(), argMsenterOrig.wasParsed(), forwardArgs); } if(argVersion.wasParsed()){ QOut() << app::SHOURNAL << qtr(" version ") << app::version().toString() << "\n"; cpp_exit(0); } logger::enableLogToFile(app::SHOURNAL); sets.load(); if(argValidateSettings.wasParsed()){ cpp_exit(0); } if(argQuery.wasParsed()){ argcontol_dbquery::parse(parser.rest().len, parser.rest().argv); // never get here } if(argDelete.wasParsed()){ argcontrol_dbdelete::parse(parser.rest().len, parser.rest().argv); // never get here } if(argEditCfg.wasParsed()){ int ret = console_dialog::openFileInExternalEditor(sets.cfgFilepath()); cpp_exit(ret); } if(argPrintMime.wasParsed()){ QFileThrow f(argPrintMime.getValue()); f.open(QFile::OpenModeFlag::ReadOnly); auto mtype = QMimeDatabase().mimeTypeForData(&f); QOut() << mtype.name() << "\n"; cpp_exit(0); } if(argLsOurPaths.wasParsed()){ QOut() << qtr("Database directory: ") << db_connection::getDatabaseDir() << "\n" << qtr("Configuration directory: ") << splitAbsPath(sets.cfgFilepath()).first << "\n" << qtr("Cache directory (log-files): ") << logger::logDir() << "\n" ; cpp_exit(0); } if(parser.rest().len != 0){ QIErr() << qtr("Invalid parameters passed: %1.\n" "Show help with --help"). arg( argvToQStr(parser.rest().len, parser.rest().argv)); cpp_exit(1); } QIErr() << "No action specified"; } catch (const ExcOptArgParse & ex) { QIErr() << qtr("Commandline seems to be erroneous:") << ex.descrip(); } catch (const ExcConversion & ex) { QIErr() << ex.descrip(); } catch(const qsimplecfg::ExcCfg & ex){ QIErr() << qtr("Failed to load config file: ") << ex.descrip(); } catch (const QExcIo& ex){ QIErr() << qtr("IO-operation failed: ") << ex.descrip(); } catch (const os::ExcOs& ex){ QIErr() << ex.what(); } cpp_exit(1); } void execShournalRun(const QByteArray& backendFilename, QOptArg::RawValues_t& cargs , bool withinOrigMountspace, QVarLengthArray forwardArgs){ // setuid-programs change some parts of the environment for security reasons. // Therefor, pass the environment via argv and apply it later (after setuid // to original user) QVarLengthArray args; args.push_back(backendFilename); if(withinOrigMountspace){ args.push_back("--msenter-orig-mountspace"); } QByteArray verbosityStr; QVarLengthArray forwardArgsBuf; for(QOptArg* a : forwardArgs){ if(a->wasParsed()){ // we need another buffer for the char* args array. forwardArgsBuf.push_back(a->name().toLocal8Bit()); args.push_back(forwardArgsBuf.last()); args.push_back(a->vals().argv[0]); } } const char* tmpdir = getenv("TMPDIR"); if(tmpdir != nullptr){ args.push_back("--tmpdir"); args.push_back(tmpdir); } args.push_back("--env"); // first value after --env is its size, which we don't know yet. args.push_back("DUMMY"); int envSizeIdx = args.size() -1; for (char **env = environ; *env != nullptr; env++) { args.push_back(*env); } // optimization in shournal-run... args.push_back("SHOURNAL_DUMMY_NULL=1"); std::string envSize = std::to_string(args.size() - envSizeIdx - 1); args[envSizeIdx] = envSize.c_str(); // As long as arguments start with a minus those // are passed as options to the shournal-run backend, e.g. // shournal-run -e --no-db --print-summary echo ok int execIdx = -1; for(int i=0; i < cargs.len; i++){ if(execIdx != -1){ args.push_back(cargs.argv[i]); continue; } if(strcmp(cargs.argv[i], "--") == 0){ // option terminator -> all further options // will be directly passed to our backend. execIdx = args.size(); args.push_back("--exec"); } else if(cargs.argv[i][0] == '-'){ // option for shournal-run args.push_back(cargs.argv[i]); } else { // first non-option. execIdx = args.size(); args.push_back("--exec"); args.push_back(cargs.argv[i]); } } if(execIdx == -1 || execIdx == args.size()-1){ QIErr() << qtr("No executable found after parsing the commandline. " "Note that for commands starting with dashes (e.g. login-shells) " "--exec has to be terminated by double-dash --\n" "Current arguments:"); for(const auto& arg : args){ QErr() << arg << " "; } QErr() << "\n"; cpp_exit(1); } args.push_back(nullptr); os::exec(args); } int main(int argc, char *argv[]) { try { shournal_main(argc, argv); } catch (const ExcCppExit& e) { return e.ret(); } } ================================================ FILE: src/shournal-run/CMakeLists.txt ================================================ add_executable(shournal-run shournal-run.cpp fifocom.cpp filewatcher_shournalk.cpp mark_helper.cpp shournalk_ctrl.c ) target_link_libraries(shournal-run Qt5::Core Qt5::Sql lib_shournal_common pthread ) install( TARGETS shournal-run RUNTIME DESTINATION bin PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE ) ================================================ FILE: src/shournal-run/fifocom.cpp ================================================ #include "fifocom.h" #include #include #include #include #include "stdiocpp.h" #include "logger.h" #include "cleanupresource.h" #include "os.h" FifoCom::FifoCom(int fifo) : m_fifofd(fifo) { m_linebuf.reserve(PIPE_BUF); } /// Read a simple json message from our fifo containing only /// of messsage-type (int >= 0) and data-field. /// @param data: is filled with the data field of the message /// @return the message type or -1. int FifoCom::readJsonLine(QString &data) { if(! readLineRaw()){ return -1; } auto finallyClearLinebuf = finally([&] { m_linebuf.clear(); }); QJsonDocument d = QJsonDocument::fromJson(m_linebuf); QJsonObject rooObj = d.object(); int messageType = rooObj.value("msgType").toInt(-1); if(messageType == -1){ logWarning << "invalid fifo-message received (buggy client?):" << m_linebuf; return -1; } data = rooObj.value("data").toString(); return messageType; } /// Buffered line read from non-blocking fifo. /// push_back to our line buffer until we find a NEWLINE. /// To be compliant with O_NONBLOCK, return false on EAGAIN. /// @return if true, m_linebuf contains the read line. bool FifoCom::readLineRaw() { bool foundNewLine = false; if(m_bufIdx == 0){ m_bufTmp.resize(PIPE_BUF); auto count = read(m_fifofd, m_bufTmp.data(), m_bufTmp.size()); if(count == -1){ if(errno == EAGAIN || errno == EWOULDBLOCK){ return false; } throw os::ExcOs("read from fifo failed:"); } m_bufTmp.resize(int(count)); } for(; m_bufIdx < m_bufTmp.size(); m_bufIdx++){ if(m_bufTmp[m_bufIdx] == '\n'){ foundNewLine = true; m_bufIdx++; break; } m_linebuf.push_back(m_bufTmp[m_bufIdx]); } if(m_bufIdx >= m_bufTmp.size()){ // whole buffer consumed m_bufIdx = 0; } if(! foundNewLine){ // most likely the user sent a message greater than // PIPE_BUF. Get the rest on next call (do not clear // buffer) return false; } return true; } ================================================ FILE: src/shournal-run/fifocom.h ================================================ #pragma once #include #include "qfilethrow.h" /// Fifo-communication. Parse (json) /// messages sent to a given shournal-run instance class FifoCom { public: FifoCom(int fifo); int readJsonLine(QString& data); private: bool readLineRaw(); int m_fifofd; QByteArray m_bufTmp; int m_bufIdx{0}; QByteArray m_linebuf; }; ================================================ FILE: src/shournal-run/filewatcher_shournalk.cpp ================================================ #include #include "filewatcher_shournalk.h" #include #include #include #include #include #include #include #include #include #include #include #include #include "app.h" #include "cefd.h" #include "cpp_exit.h" #include "conversions.h" #include "commandinfo.h" #include "db_controller.h" #include "cleanupresource.h" #include "fdentries.h" #include "fifocom.h" #include "fileevents.h" #include "logger.h" #include "mark_helper.h" #include "os.h" #include "osutil.h" #include "qoutstream.h" #include "settings.h" #include "shournalk_ctrl.h" #include "shournal_run_common.h" #include "stdiocpp.h" #include "subprocess.h" #include "translation.h" const int PRIO_DATABASE_FLUSH = 10; const pid_t INVALID_PID = std::numeric_limits::max(); using subprocess::Subprocess; using std::shared_ptr; using std::make_shared; using ShournalK_ptr = Filewatcher_shournalk::ShournalK_ptr; static void handleFifoEvent(shared_ptr& fifoCom, CommandInfo* cmdInfo, ShournalK_ptr& shournalk){ enum { FIFO_RETURN_VAL=0, FIFO_UNMARK_PID}; QString data; for (int i=0; ; i++) { int msgType = fifoCom->readJsonLine(data); if(msgType == -1){ return; } switch (msgType) { case FIFO_RETURN_VAL: if(! qVariantTo(data, &cmdInfo->returnVal)){ logWarning << qtr("bad return value '%1' received").arg(data); } break; case FIFO_UNMARK_PID: pid_t pid; if(! qVariantTo(data, &pid)){ logWarning << qtr("bad pid '%1' received").arg(data); } else { try { shournalk->removePid(pid); } catch (const ExcShournalk& ex) { logWarning << ex.what(); } } break; default: logWarning << "Invalid fifo-message received:" << msgType << "with data" << data; break; } } } static int do_polling(ShournalK_ptr& shournalk, struct shournalk_run_result* run_result, const QByteArray& fifopath, CommandInfo* cmdInfo){ int fifo = -1; auto finallyCloseFifo = finally([&fifo] { if(fifo != -1) close(fifo); }); // Protect client from deadlock: first delete, then // close ("finally" reverses call order). auto finallydelFifo = finally([&fifopath] { if(!fifopath.isEmpty()){ // fail silently, as the shell integration // might be faster than us remove(fifopath); } }); shared_ptr fifoCom; QVector fds; shournalk->preparePollOnce(); pollfd shournalkfd; shournalkfd.fd = shournalk->kgrp()->pipe_readend; shournalkfd.events = POLLIN; fds.push_back(shournalkfd); if(! fifopath.isEmpty()){ pollfd fd; // open RDWR, to correctly get EAGAIN in case of no (other) writer. fifo = os::open(fifopath, os::OPEN_RDWR | os::OPEN_NONBLOCK | os::OPEN_EXCL); fd.fd = fifo; fd.events = POLLIN; fds.push_back(fd); fifoCom = make_shared(fifo); } while (1) { int poll_num = poll(fds.data(), nfds_t(fds.size()), -1); if (poll_num == -1) { if (errno == EINTR){ // Interrupted by a signal continue; // Restart poll() } logCritical << "Error during poll: " << translation::strerror_l(); return errno; } // 0 only on timeout, which is infinite assert(poll_num != 0); if (fds[0].revents & POLLIN) { auto read_count = os::read(shournalk->kgrp()->pipe_readend, run_result, sizeof(struct shournalk_run_result)); if(read_count != sizeof(struct shournalk_run_result)){ logCritical << qtr("Received bad run-result from kernel backend: " "expected %1 bytes but received %2.") .arg(sizeof(struct shournalk_run_result)).arg(read_count); return EPIPE; } return 0; } assert(fds.size() > 1); if(fds[1].revents & POLLIN){ handleFifoEvent(fifoCom, cmdInfo, shournalk); } else { // can never happen, because we opened the // fifo RDWR, so we get no events if a writer closes // (which is not the case if opened RDONLY) assert(false); } } } QByteArray Filewatcher_shournalk::fifopathForPid(pid_t pid) { QByteArray fifopath = pathJoinFilename(QDir::tempPath().toUtf8(), "shournal-run-fifo-" + QByteArray::number(pid)); return fifopath; } Filewatcher_shournalk::Filewatcher_shournalk() { } void Filewatcher_shournalk::setArgv(char **argv, int argc) { m_commandArgv = argv; m_commandArgc = argc; } void Filewatcher_shournalk::setPid(const pid_t &pid) { m_pid = pid; } void Filewatcher_shournalk::setCommandFilename(char *commandFilename) { m_commandFilename = commandFilename; } void Filewatcher_shournalk::setStoreToDatabase(bool storeToDatabase) { m_storeToDatabase = storeToDatabase; } void Filewatcher_shournalk::setShellSessionUUID(const QByteArray &shellSessionUUID) { m_shellSessionUUID = shellSessionUUID; } void Filewatcher_shournalk::setForkIntoBackground(bool value) { m_forkIntoBackground = value; } void Filewatcher_shournalk::setCmdString(const QString &cmdString) { m_cmdString = cmdString; } void Filewatcher_shournalk::setFifoname(const QByteArray &fifoname) { m_fifoname = fifoname; } void Filewatcher_shournalk::setPrintSummary(bool printSummary) { m_printSummary = printSummary; } CommandInfo Filewatcher_shournalk::runExec(ShournalK_ptr &shournalk, CEfd& toplvlEfd) { CommandInfo cmdInfo = CommandInfo::fromLocalEnv(); cmdInfo.sessionInfo.uuid = m_shellSessionUUID; if(m_commandFilename != nullptr){ cmdInfo.text += QString(m_commandFilename) + " "; // TODO: rather store cmdInfo.text only from &m_commandArgv[1] in case // of m_commandFilename != null ? } cmdInfo.text += argvToQStr(m_commandArgc, m_commandArgv); CEfd cefd; Subprocess proc; proc.setWaitForSetup(false); proc.setCallbackAsChild([&cefd]{ // Block until parent process did the setup cefd.recvMsg(); }); const char* cmdFilename = (m_commandFilename == nullptr) ? m_commandArgv[0] : m_commandFilename; cmdInfo.startTime = QDateTime::currentDateTime(); proc.call(cmdFilename, m_commandArgv); uint64_t markRet; try { shournalk->doMark(proc.lastPid()); markRet = CEfd::MSG_OK; } catch (const ExcShournalk& ex) { logWarning << ex.descrip(); markRet = CEfd::MSG_FAIL; } toplvlEfd.sendMsg(markRet); cefd.sendMsg(markRet); cefd.teardown(); try { cmdInfo.returnVal = proc.waitFinish(); } catch (const os::ExcProcessExitNotNormal& ex) { // return typical shell cpp_exit code cmdInfo.returnVal = 128 + ex.status(); } // do not set endTime here, but after poll for kgrp, so // all background-processes finished if(markRet != CEfd::MSG_OK){ cpp_exit(cmdInfo.returnVal); } return cmdInfo; } CommandInfo Filewatcher_shournalk::runMarkPid(ShournalK_ptr &shournalk, CEfd &toplvlEfd) { assert(! m_cmdString.isEmpty()); CommandInfo cmdInfo = CommandInfo::fromLocalEnv(); cmdInfo.sessionInfo.uuid = m_shellSessionUUID; // Start-time will likely be overwritten later cmdInfo.text = m_cmdString; cmdInfo.startTime = QDateTime::currentDateTime(); try { shournalk->doMark(m_pid, true); toplvlEfd.sendMsg(CEfd::MSG_OK); } catch (const ExcShournalk& ex) { logWarning << ex.descrip(); toplvlEfd.sendMsg(CEfd::MSG_FAIL); cpp_exit(1); } return cmdInfo; } void Filewatcher_shournalk::run() { auto shournalk = make_shared(); CommandInfo cmdInfo; CEfd toplvlEfd; if(m_forkIntoBackground){ // parent exits, child continues in new sid. // We wait for child to finish setup. if(os::fork() != 0){ logDebug << "forking into background"; auto ret = (toplvlEfd.recvMsg() == CEfd::MSG_OK) ? 0 : 1; exit(ret); } // child os::setsid(); } if(m_commandArgc != 0){ cmdInfo = runExec(shournalk, toplvlEfd); } else { cmdInfo = runMarkPid(shournalk, toplvlEfd); os::mkfifo(m_fifoname, 0600); } // Everything is ready and cmdInfo.workingDirectory is // also setup correctly. Try to act at least a bit like // a daemon by chdir("/"); e.g. to not block an unmount. // Note that in case we were launched from within the // shell integration the only open files should be // our own logfile and the eventlog-file, both // at locations which are usually never unmounted. os::chdir("/"); struct shournalk_run_result krun_result; auto poll_result = do_polling(shournalk, &krun_result, m_fifoname, &cmdInfo); cmdInfo.endTime = QDateTime::currentDateTime(); if(cmdInfo.returnVal == CommandInfo::INVALID_RETURN_VAL && krun_result.selected_exitcode != SHOURNALK_INVALID_EXIT_CODE){ if(krun_result.selected_exitcode < 0 || krun_result.selected_exitcode > 255){ // the kernel module currently strips higher bits, // so we should never get here. logWarning << qtr("Unusual exit-code %1 received. Please report.") .arg(krun_result.selected_exitcode); } logDebug << "using exitcode from kernel module:" << krun_result.selected_exitcode; cmdInfo.returnVal = krun_result.selected_exitcode; } if(poll_result != 0){ // should never happen. Return failure regardless // of launched command exit status cpp_exit(2); } if(krun_result.error_nb != 0){ // may rarely happen if target file got lost during // event processing (stored on NFS?) QString msg; switch (krun_result.error_nb) { case EIO: msg = qtr("%1. Maybe the target" "file resided on a NFS storage, which became unavailable?") .arg(translation::strerror_l(EIO)); break; default: msg = translation::strerror_l(krun_result.error_nb); break; } logWarning << qtr("Error %1 during file event processing " "(in kernel mode, most likely non-fatal): %2") .arg(krun_result.error_nb).arg(msg); // since it is nonfatal return cmd exit code } if(krun_result.lost_event_count != 0){ // TODO: insert cmd-id here. logInfo << qtr("%1 events where lost").arg(krun_result.lost_event_count); } if(m_printSummary){ shournal_run_common::print_summary( krun_result.w_event_count, krun_result.r_event_count, krun_result.lost_event_count, krun_result.stored_event_count, os::fstat(fileno(shournalk->tmpFileTarget())).st_size); } if(m_storeToDatabase){ // os::lseek(fileno_unlocked(tmpFileTarget), 0, SEEK_SET); stdiocpp::fseek(shournalk->tmpFileTarget(), 0, SEEK_SET); FileEvents fileEvents; fileEvents.setFile(shournalk->tmpFileTarget()); try { // Do not disturb other processes while we flush events to database os::setpriority(PRIO_PROCESS, 0, PRIO_DATABASE_FLUSH); } catch (const os::ExcOs&) { // This may happen regularly, e.g. if priority was already lowered. logDebug << "Failed to set priority before database flush"; } try { cmdInfo.idInDb = db_controller::addCommand(cmdInfo); db_controller::addFileEvents(cmdInfo, fileEvents); } catch (std::exception& e) { // May happen, e.g. if we run out of disk space... logCritical << qtr("Failed to store (some) file-events to disk: %1").arg(e.what()); } } shournalk.reset(); cpp_exit(cmdInfo.returnVal); } ================================================ FILE: src/shournal-run/filewatcher_shournalk.h ================================================ #pragma once #include #include #include "commandinfo.h" #include "mark_helper.h" extern const pid_t INVALID_PID; struct shournalk_group; class CEfd; class Filewatcher_shournalk { public: typedef std::shared_ptr ShournalK_ptr; public: static QByteArray fifopathForPid(pid_t pid); Filewatcher_shournalk(); void setArgv(char **argv, int argc); void setPid(const pid_t &pid); void setCommandFilename(char *commandFilename); void setStoreToDatabase(bool storeToDatabase); void setShellSessionUUID(const QByteArray &shellSessionUUID); void setForkIntoBackground(bool value); void setCmdString(const QString &cmdString); void setFifoname(const QByteArray &fifoname); void setPrintSummary(bool printSummary); [[noreturn]] void run(); private: CommandInfo runExec(ShournalK_ptr& shournalk, CEfd &toplvlEfd); CommandInfo runMarkPid(ShournalK_ptr& shournalk, CEfd &toplvlEfd); int m_commandArgc{}; char* m_commandFilename{}; char **m_commandArgv; bool m_forkIntoBackground{}; pid_t m_pid{INVALID_PID}; bool m_printSummary{false}; bool m_storeToDatabase{true}; QByteArray m_shellSessionUUID; QString m_cmdString; QByteArray m_fifoname; }; ================================================ FILE: src/shournal-run/mark_helper.cpp ================================================ #include "mark_helper.h" #include #include #include #include #include "app.h" #include "shournalk_ctrl.h" #include "stdiocpp.h" #include "translation.h" #include "os.h" #include "logger.h" using std::unordered_map; using std::string; ExcShournalk::ExcShournalk(const QString &text) : QExcCommon(text, false) {} using StrLightSet = Settings::StrLightSet; /// Build the kernel settings according to our own static shounalk_settings buildKSettings(){ auto & s = Settings::instance(); auto & w_sets = s.writeFileSettings(); auto & r_sets = s.readFileSettings(); auto & script_sets = s.readEventScriptSettings(); struct shounalk_settings ksettings{}; ksettings.w_exclude_hidden = w_sets.excludeHidden; ksettings.w_max_event_count = w_sets.maxEventCount; ksettings.r_only_writable = r_sets.onlyWritable; ksettings.r_exclude_hidden = r_sets.excludeHidden; ksettings.r_max_event_count = r_sets.maxEventCount; ksettings.r_store_only_writable = script_sets.onlyWritable; ksettings.r_store_max_size = unsigned(script_sets.maxFileSize); ksettings.r_store_max_count_of_files = uint16_t(script_sets.maxCountOfFiles); ksettings.r_store_exclude_hidden = script_sets.excludeHidden; if(s.hashSettings().hashEnable){ ksettings.hash_max_count_reads = s.hashSettings().hashMeta.maxCountOfReads; ksettings.hash_chunksize = s.hashSettings().hashMeta.chunkSize; } return ksettings; } ShournalkControl::ShournalkControl() { m_kgrp = shournalk_init(O_CLOEXEC); if(m_kgrp == nullptr){ throw ExcShournalk("init failed"); } shournalk_version kversion; if(shournalk_read_version(&kversion) != 0){ throw ExcShournalk(qtr("Failed to read version from file %1 - %2") .arg(shournalk_versionpath()) .arg(translation::strerror_l(errno))); } if(strcmp(SHOURNAL_VERSION, kversion.ver_str) != 0){ // Try to avoid unnecessary unloading of the kernel module shournalk // when a new version is installed: auto kver = QVersionNumber::fromString(kversion.ver_str); const auto minVersion = QVersionNumber{2,8}; if(kver < minVersion){ throw ExcShournalk(qtr("Version mismatch - kernel-module version is %1, but " "min. required version of %2 is %3") .arg(kversion.ver_str) .arg(app::SHOURNAL_RUN).arg(minVersion.toString())); } if(kver > app::version()){ logWarning <allPaths(); const auto & w_excl = s.writeFileSettings().excludePaths->allPaths(); const auto & r_incl = s.readFileSettings().includePaths->allPaths(); const auto & r_excl = s.readFileSettings().excludePaths->allPaths(); const auto & script_incl = s.readEventScriptSettings().includePaths->allPaths(); const auto & script_excl = s.readEventScriptSettings().excludePaths->allPaths(); markPaths(w_incl, SHOURNALK_MARK_W_INCL ); markPaths(w_excl, SHOURNALK_MARK_W_EXCL ); markPaths(all_excl, SHOURNALK_MARK_W_EXCL ); if(s.readFileSettings().enable){ markPaths(r_incl, SHOURNALK_MARK_R_INCL); markPaths(r_excl, SHOURNALK_MARK_R_EXCL); markPaths(all_excl, SHOURNALK_MARK_R_EXCL); } if(s.readEventScriptSettings().enable){ markPaths(script_incl, SHOURNALK_MARK_SCRIPT_INCL); markPaths(script_excl, SHOURNALK_MARK_SCRIPT_EXCL); markPaths(all_excl, SHOURNALK_MARK_SCRIPT_EXCL); const auto & exts = s.readEventScriptSettings().includeExtensions; if(exts.size()){ markExtensions(exts, SHOURNALK_MARK_SCRIPT_EXTS); } } if((ret = shournalk_commit(m_kgrp)) != 0){ throw ExcShournalk(qtr("failed to commit event target - %1") .arg(translation::strerror_l(ret))); } } catch (ExcShournalk& ex) { throw ExcShournalk(qtr("Failed to mark target process with pid " "%1 for observation - %2") .arg(pid).arg(ex.descrip())); } } void ShournalkControl::preparePollOnce() { if(shournalk_prepare_poll_ONCE(m_kgrp)){ throw ExcShournalk(qtr("failed to prepare poll")); } } void ShournalkControl::removePid(pid_t pid) { int ret; if((ret = shournalk_filter_pid(m_kgrp, SHOURNALK_MARK_REMOVE, pid)) != 0){ throw ExcShournalk( qtr("Failed to unmark pid for observation: %1").arg(translation::strerror_l(ret)) ); } } FILE *ShournalkControl::tmpFileTarget() const { return m_tmpFileTarget; } shournalk_group *ShournalkControl::kgrp() const { return m_kgrp; } void ShournalkControl::markPaths(const Settings::StrLightSet& paths, int path_tpye){ int ret; for(const auto& p : paths){ if((ret = shournalk_filter_string(m_kgrp, SHOURNALK_MARK_ADD, path_tpye, p.c_str())) != 0 ){ throw ExcShournalk(qtr("failed to mark path " "%1 - %2").arg(p.c_str()) .arg(translation::strerror_l(ret))); } } } void ShournalkControl::markExtensions (const Settings::StrLightSet& extensions, int ext_type){ StrLight extBuf; const StrLight::size_type BUF_SIZE = 4096; extBuf.reserve(BUF_SIZE); for(const auto & str : extensions){ // add extensions to a single long string // separated by slash. Flush, if bigger // than BUF_SIZE (unlikely) if(extBuf.size() + str.size() + 1 > BUF_SIZE){ doMarkExtensions(extBuf, ext_type); } extBuf += str + '/'; } if(! extBuf.empty()){ doMarkExtensions(extBuf, ext_type); } } void ShournalkControl::doMarkExtensions (const StrLight& extensions, int ext_type){ int ret; if((ret = shournalk_filter_string(m_kgrp, SHOURNALK_MARK_ADD, ext_type, extensions.c_str())) != 0 ){ throw ExcShournalk(qtr("failed to mark extensions - %1 " "- extensions-string: %2") .arg(translation::strerror_l(ret)) .arg(extensions.c_str())); } } ================================================ FILE: src/shournal-run/mark_helper.h ================================================ #pragma once #include #include "exccommon.h" #include "settings.h" struct shournalk_group; class ExcShournalk : public QExcCommon { public: ExcShournalk(const QString & text); }; /// c++ interface for shournal's kernel module. class ShournalkControl { public: ShournalkControl(); ~ShournalkControl(); void doMark(pid_t pid, bool collectExitcode=false); void preparePollOnce(); void removePid(pid_t pid); FILE *tmpFileTarget() const; shournalk_group *kgrp() const; private: Q_DISABLE_COPY(ShournalkControl) struct shournalk_group* m_kgrp; FILE* m_tmpFileTarget; void markPaths(const Settings::StrLightSet& paths, int path_tpye); void markExtensions(const Settings::StrLightSet& extensions, int ext_type); void doMarkExtensions(const StrLight &extensions, int ext_type); }; ================================================ FILE: src/shournal-run/shournal-run.cpp ================================================ #include #include #include #include #include #include #include "qoptargparse.h" #include "qoptvarlenarg.h" #include "excoptargparse.h" #include "os.h" #include "excos.h" #include "filewatcher_shournalk.h" #include "logger.h" #include "fdcommunication.h" #include "exccfg.h" #include "settings.h" #include "util.h" #include "qoutstream.h" #include "util.h" #include "translation.h" #include "app.h" #include "qexcdatabase.h" #include "cpp_exit.h" #include "db_connection.h" #include "storedfiles.h" #include "socket_message.h" using fdcommunication::SocketCommunication; using socket_message::E_SocketMsg; #include #include #include #include #include #include #include #include #include #include #include #include "os.h" #include "osutil.h" #include "fdentries.h" #include "qoutstream.h" #include "shournalk_ctrl.h" #include "shournal_run_common.h" using namespace shournal_run_common; /// Uncaught exception handler static void onterminate() { try { auto unknown = std::current_exception(); if (unknown) { std::rethrow_exception(unknown); } } catch (const std::exception& e) { logCritical << e.what() << "\n"; } catch (...) { logCritical << "unknown exception occurred\n"; } } static void closeFds(){ // close all file descriptors except stderr and // a potential integration test descriptor. auto keepFds = std::unordered_set{2, app::findIntegrationTestFd()}; for(int fd : osutil::FdEntries()){ if(keepFds.find(fd) == keepFds.end()){ os::close(fd); } } } static int shournal_run_main(int argc, char *argv[]) { // Since we are waiting for other processes to finish, ignore typical // signals. osutil::setInertSighandler(os::catchableTermSignals()); // Using app::SHOURNAL for several common paths (database, config) used // by QStandardPaths but app::CURRENT_NAME for others (log-filename) app::setupNameAndVersion(app::SHOURNAL_RUN); if(! translation::init()){ logWarning << "Failed to initialize translation"; } logger::setup(app::CURRENT_NAME); std::set_terminate(onterminate); if(! shournal_common_init()){ logCritical << qtr("Fatal error: failed to initialize custom Qt conversion functions"); cpp_exit(1); } // ignore first arg (command to this app) --argc; ++argv; QOptArgParse parser; parser.setHelpIntroduction(qtr("Observation backend for <%1> based " "on a custom kernel module." ).arg(app::SHOURNAL) + "\n"); QOptArg argVersion("v", "version", qtr("Display version"), false); parser.addArg(&argVersion); QOptArg argPid("", "pid", qtr("Mark the process with given pid for observation.")); parser.addArg(&argPid); QOptArg argPrintFifopath("", "print-fifopath-for-pid", qtr("Print the fifo path for a given pid " "and exit. A fifo is " "created, if argument %1 is given.") .arg(argPid.name())); parser.addArg(&argPrintFifopath); QOptArg argFifoname("", "fifoname", qtr("If arg %1 is also parsed, create " "the fifo under the given filename").arg(argPid.name())); parser.addArg(&argFifoname); argFifoname.addRequiredArg(&argPid); QOptArg argTmpDir("", "tmpdir", "NOT USED"); // Interface compatibility with shournal-run-fanotify. // Not being suid we can simply use $TMPDIR. argTmpDir.setInternalOnly(true); parser.addArg(&argTmpDir); QOptVarLenArg argEnv("", "env", "NOT USED"); // Interface compatibility with shournal-run-fanotify. argEnv.setInternalOnly(true); parser.addArg(&argEnv); QOptArg argFork("", "fork", qtr("Fork into background immediatly after marking " "a pid."), false); parser.addArg(&argFork); QOptArg argCmdString("", "cmd-string", qtr("Associate the recording of a process with the " "given command string. Only used, if arg %1 " "is given").arg(argPid.name())); argCmdString.addRequiredArg(&argPid); parser.addArg(&argCmdString); argPid.addRequiredArg(&argCmdString); QOptArg argCloseFds("", "close-fds", qtr("Advanced option: closes file " "descriptors except stderr."), false); parser.addArg(&argCloseFds); QOptArg argPrintSummary("", "print-summary", qtr("Print a short summary after " "event processing finished."), false); parser.addArg(&argPrintSummary); QOptArg argShournalkIsLoaded("", "shournalk-is-loaded", qtr("If shournal's kernel module is loaded, " "exit with zero, else nonzero"), false); parser.addArg(&argShournalkIsLoaded); QOptArg argExec("e", "exec", qtr("Execute and observe the passed program " "and its arguments (this argument has to be last)."), false); argExec.setFinalizeFlag(true); parser.addArg(&argExec); QOptArg argExecFilename("", "exec-filename", qtr("This is an advanced option. " "In most cases the first argument of a " "program is the program name. For " "example for login-shells this does " "not have to be the case. If this " "argument is provided, that filename " "is used instead of argv[0]")); parser.addArg(&argExecFilename); argExecFilename.addRequiredArg(&argExec); QOptArg argVerbosity("", "verbosity", qtr("How much shall be printed to stderr. Note that " "for 'dbg' shournal-run must not be a 'Release'-build.")); argVerbosity.setAllowedOptions(app::VERBOSITIES); parser.addArg(&argVerbosity); QOptArg argShellSessionUUID("", "shell-session-uuid", qtr("uuid as base64-encoded string")); argShellSessionUUID.setInternalOnly(true); parser.addArg(&argShellSessionUUID); QOptArg argMakeSessionUUID("", "make-session-uuid", qtr("print a unique uuid to stdout and " "exit"), false); argMakeSessionUUID.setInternalOnly(true); parser.addArg(&argMakeSessionUUID); QOptArg argNoDb("", "no-db", qtr("For debug purposes: do not write to " "database after event processing"), false); parser.addArg(&argNoDb); auto argCfgDir = mkarg_cfgdir(); parser.addArg(&argCfgDir); auto argDataDir = mkarg_datadir(); parser.addArg(&argDataDir); try { auto & sets = Settings::instance(); parser.parse(argc, argv); if(argCfgDir.wasParsed()){ sets.setUserCfgDir(argCfgDir.getValue()); } if(argDataDir.wasParsed()){ sets.setUserDataDir(argDataDir.getValue()); } if(argCloseFds.wasParsed()){ // Do this early before we open fds ourselves. closeFds(); } if(argVerbosity.wasParsed()){ QByteArray verbosity = argVerbosity.getOptions(1).first().toLocal8Bit(); logger::setVerbosityLevel(verbosity.constData()); } else { logger::setVerbosityLevel(QtMsgType::QtWarningMsg); } if(argPrintFifopath.wasParsed()){ QOut() << Filewatcher_shournalk::fifopathForPid( argPrintFifopath.getValue() ) << "\n"; cpp_exit(0); } if(argShournalkIsLoaded.wasParsed()){ cpp_exit(! shournalk_module_is_loaded()); } if(argMakeSessionUUID.wasParsed()){ bool madeSafe; auto uuid = make_uuid(&madeSafe); if(! madeSafe){ logInfo << qtr("session uuid not created 'safe'. Is the uuidd-daemon running?"); } QOut() << uuid.toBase64() << "\n"; cpp_exit(0); } if(argExec.wasParsed() && argPid.wasParsed() ) { QIErr() << qtr("%1 and %2 are mutually exclusive").arg(argExec.name(), argPid.name()); cpp_exit(1); } if(argVersion.wasParsed()){ QOut() << app::SHOURNAL_RUN << qtr(" version ") << app::version().toString() << "\n"; cpp_exit(0); } try { logger::enableLogToFile(app::SHOURNAL_RUN); sets.load(); StoredFiles::mkpath(); } catch(const qsimplecfg::ExcCfg & ex){ QIErr() << qtr("Failed to load config file: ") << ex.descrip(); cpp_exit(1); } catch(const QExcDatabase & ex){ QIErr() << qtr("Database-operation failed: ") << ex.descrip(); cpp_exit(1); } catch (const QExcIo& ex){ logCritical << qtr("IO-operation failed: ") << ex.descrip(); cpp_exit(1); } catch (const os::ExcOs& ex){ logCritical << ex.what(); cpp_exit(1); } Filewatcher_shournalk fwatcher; if(argExecFilename.wasParsed()){ // fwatcher command-filename must otherwise be null, to allow // for correct storing of command in db (no duplicate first arg // if not necessary!) fwatcher.setCommandFilename(argExecFilename.vals().argv[0]); } fwatcher.setForkIntoBackground(argFork.wasParsed()); fwatcher.setPrintSummary(argPrintSummary.wasParsed()); fwatcher.setStoreToDatabase(! argNoDb.wasParsed()); if(argShellSessionUUID.wasParsed()){ fwatcher.setShellSessionUUID( QByteArray::fromBase64(argShellSessionUUID.getValue())); } if(argExec.wasParsed()){ auto externCmd = parser.rest(); fwatcher.setArgv(externCmd.argv, externCmd.len); fwatcher.run(); } if(argPid.wasParsed()){ fwatcher.setPid(argPid.getValue(INVALID_PID)); fwatcher.setFifoname(argFifoname.getValue( Filewatcher_shournalk::fifopathForPid(getpid()) )); assert(argCmdString.wasParsed()); fwatcher.setCmdString(argCmdString.getValue()); fwatcher.run(); } if(parser.rest().len != 0){ QIErr() << qtr("Invalid parameters passed: %1.\n" "Show help with --help"). arg( argvToQStr(parser.rest().len, parser.rest().argv)); cpp_exit(1); } QIErr() << "No action specified"; } catch (const ExcOptArgParse & ex) { QIErr() << qtr("Commandline seems to be erroneous:") << ex.descrip(); } cpp_exit(1); } int main(int argc, char *argv[]) { try { shournal_run_main(argc, argv); } catch (const ExcCppExit& e) { return e.ret(); } } ================================================ FILE: src/shournal-run/shournalk_ctrl.c ================================================ #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include #include #include #include #include #include #include #include #include #include "shournalk_ctrl.h" #define SHOURNALK_CTRL_PATH "/sys/kernel/shournalk_root/shournalk_ctrl" // Unprivileged docker containers have a read-only sysfs filesystem. // So we look at below path where the hosts shournalk-control // might be bind-mounted (by the user) inside the container. #define SHOURNALK_CTRL_DOCKER_PATH "/tmp/shournalk-sysfs/shournalk_ctrl" #define SHOURNALK_MARK_PATH SHOURNALK_CTRL_PATH "/mark" #define SHOURNALK_DOCKER_MARK_PATH SHOURNALK_CTRL_DOCKER_PATH "/mark" #define SHOURNALK_VERSION_PATH SHOURNALK_CTRL_PATH "/version" static bool __file_exists(const char* filename){ struct stat buffer; return (stat (filename, &buffer) == 0); } static int __open_sysfs_mark(void){ const int o_flags = O_WRONLY | O_CLOEXEC; int fd = open(SHOURNALK_MARK_PATH, o_flags); if(fd >= 0) return fd; if(errno != EROFS){ perror("Failed to open shournalk's sysfs-interface at " SHOURNALK_MARK_PATH ". Is the kernel module loaded? modprobe shournalk"); return -1; } // likely inside docker or another container - try alternative path fd = open(SHOURNALK_DOCKER_MARK_PATH, o_flags); if(fd >= 0) return fd; perror("Failed to open shournalk's sysfs-interface at\n" SHOURNALK_MARK_PATH " and\n" SHOURNALK_DOCKER_MARK_PATH ". Is the kernel module loaded? modprobe shournalk"); return -1; } static int __shournalk_filter_common(struct shournalk_group* grp, unsigned int flags, int action) { grp->__mark_struct.flags = flags; grp->__mark_struct.action = action; ssize_t writeRet = write(grp->__sysfs_mark_fd, &grp->__mark_struct, sizeof (grp->__mark_struct)); if(writeRet != sizeof (grp->__mark_struct)){ return errno; } return 0; } bool shournalk_module_is_loaded(void){ return __file_exists(shournalk_versionpath()); } const char* shournalk_versionpath(void){ return SHOURNALK_VERSION_PATH; } /// @param flags: for creating the pipe, e.g. O_NONBLOCK struct shournalk_group* shournalk_init(unsigned int flags){ struct shournalk_group* grp; int sysfs_fd; int pip_descr[2]; if(pipe2(pip_descr, flags) == -1){ perror("pipe"); return NULL; } sysfs_fd = __open_sysfs_mark(); if(sysfs_fd < 0){ close(pip_descr[0]); close(pip_descr[1]); return NULL; } grp = (struct shournalk_group*)calloc(1, sizeof (struct shournalk_group)); if(grp == NULL){ close(pip_descr[0]); close(pip_descr[1]); return NULL; } grp->__mark_struct.target_fd = -1; grp->pipe_readend = pip_descr[0]; grp->__mark_struct.pipe_fd = pip_descr[1]; grp->__sysfs_mark_fd = sysfs_fd; return grp; } void shournalk_release(struct shournalk_group* grp){ if(close(grp->pipe_readend) == -1){ perror("shournalk_release close pipe readend"); } if(close(grp->__sysfs_mark_fd) == -1){ perror("shournalk_release sysfs_mark_fd"); } if(grp->__mark_struct.pipe_fd != -1){ if(close(grp->__mark_struct.pipe_fd)){ perror("shournalk_release close pipe writeend"); } } free(grp); } void shournalk_set_target_fd(struct shournalk_group* grp, int fd){ grp->__mark_struct.target_fd = fd; } void shournalk_set_settings(struct shournalk_group* grp, struct shounalk_settings* settings){ grp->__mark_struct.settings = *settings; } /// Make sure to set target_fd and settings beforehand int shournalk_filter_pid(struct shournalk_group* grp, unsigned int flags, pid_t pid) { grp->__mark_struct.pid = pid; return __shournalk_filter_common(grp, flags, SHOURNALK_MARK_PID); } /// @param str_tpye: one of SHOURNALK_MARK_* /// R_INCL, R_EXCL /// W_INCL, W_EXCL /// SCRIPT_INCL, SCRIPT_EXCL, SCRIPT_EXTS int shournalk_filter_string(struct shournalk_group* grp, unsigned int flags, int str_tpye, const char* str){ grp->__mark_struct.data = str; return __shournalk_filter_common(grp, flags, str_tpye); } int shournalk_commit(struct shournalk_group* grp){ return __shournalk_filter_common(grp, SHOURNALK_MARK_COMMIT, 0); } /// cĺose the pipe write end to avoid deadlock in poll. /// warning - may only be called once per shournalk-group. /// After that you are not allowed to call other functions but /// shournalk_release int shournalk_prepare_poll_ONCE(struct shournalk_group* grp){ assert(grp->__mark_struct.pipe_fd != -1); if(close(grp->__mark_struct.pipe_fd)){ perror("shournalk_prepare_poll_ONCE close pipe writeend"); return errno; } grp->__mark_struct.pipe_fd = -1; return 0; } /// Read the version-string from sysfs, returning 0 on success, /// else nonzero with errno set. int shournalk_read_version(struct shournalk_version* ver){ int fd; ssize_t ret = 0; fd = open(SHOURNALK_VERSION_PATH, O_RDONLY); if(fd < 0) return fd; ret = read(fd, ver->ver_str, sizeof (ver->ver_str)); if(ret < 0) goto out; if(ret == sizeof (ver->ver_str)){ fprintf(stderr, "shournalk_read_version - " "too large version string read (bug?)\n"); errno = EFBIG; ret = -1; goto out; } ver->ver_str[ret] = '\0'; ret = 0; // success out: close(fd); return (int)ret; } ================================================ FILE: src/shournal-run/shournalk_ctrl.h ================================================ /* Generic c-interface to control shournal's kernel module from * userspace. Besides the kernel module (header) it depends only on * system headers. */ #pragma once #include #include "shournalk_user.h" #ifdef __cplusplus extern "C" { #endif struct shournalk_group { int pipe_readend; /* kernel notifies us when done */ int __sysfs_mark_fd; struct shournalk_mark_struct __mark_struct; }; struct shournalk_version { char ver_str[256]; }; bool shournalk_module_is_loaded(void); const char* shournalk_versionpath(void); struct shournalk_group* shournalk_init(unsigned int flags); void shournalk_release(struct shournalk_group* grp); void shournalk_set_target_fd(struct shournalk_group* grp, int fd); void shournalk_set_settings(struct shournalk_group* grp, struct shounalk_settings* settings); int shournalk_filter_pid(struct shournalk_group* grp, unsigned int flags, pid_t pid); int shournalk_filter_string(struct shournalk_group* grp, unsigned int flags, int str_tpye, const char* str); int shournalk_commit(struct shournalk_group* grp); int shournalk_prepare_poll_ONCE(struct shournalk_group* grp); int shournalk_read_version(struct shournalk_version* ver); #ifdef __cplusplus } #endif ================================================ FILE: src/shournal-run-fanotify/CMakeLists.txt ================================================ # This program is somewhat performance-critical, # so disable exports. FIXME: benchmark it IF(CMAKE_BUILD_TYPE MATCHES Release) SET (CMAKE_ENABLE_EXPORTS FALSE) ENDIF() include_directories( ../common ../common/qoptargparse ../common/database ../common/qsimplecfg ../common/oscpp ../common/database ../common/qsqlthrow ../../ ) add_executable(shournal-run-fanotify shournal-run-fanotify.cpp fanotify_controller.cpp filewatcher_fan.cpp mount_controller.cpp msenter.cpp orig_mountspace_process.cpp ) target_link_libraries(shournal-run-fanotify lib_shournal_common pthread uuid cap # capabilites ) install( TARGETS shournal-run-fanotify RUNTIME DESTINATION bin PERMISSIONS SETUID OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE ) ================================================ FILE: src/shournal-run-fanotify/fanotify_controller.cpp ================================================ #ifndef _GNU_SOURCE #define _GNU_SOURCE // Needed to get O_LARGEFILE definition #endif #include #include #include #include #include #include #include #include #include #include #include #include #include "fanotify_controller.h" #include "util.h" #include "fileeventhandler.h" #include "excos.h" #include "cxxhash.h" #include "os.h" #include "osutil.h" #include "settings.h" #include "logger.h" #include "translation.h" #include "mount_controller.h" #include "db_connection.h" #include "storedfiles.h" using ExcCXXHash = CXXHash::ExcCXXHash; using os::ExcOs; using StringSet = Settings::StringSet; // Max number of fanotify events which can be consumed // with a single read(2). Note that the max number of open // fd's is also adjusted, however, since we already // have some other fd's open, the actual max number of // events will be a little lower. const int FANOTIFY_MAX_EVENT_COUNT = 4096; namespace { QString fanotifyEventMaskToStr(uint64_t m){ QString action; if(m & FAN_MODIFY){ action += "modified "; } if(m & FAN_CLOSE_WRITE){ action += "closed_write "; } if(m & FAN_CLOSE_NOWRITE){ action += "closed_nowrite"; } if(m & FAN_OPEN){ action += "open"; } if(action.isEmpty()){ action = "unhandled event: " + QString::number(m, 16); } return action; } bool fanotifyMarkWrapOnInit(int fanFd, uint64_t mask, const std::string& path_){ if (fanotify_mark(fanFd, FAN_MARK_ADD | FAN_MARK_MOUNT, mask, AT_FDCWD, path_.c_str()) == -1) { const auto msg = qtr("fanotify_mark: failed to add path %1. " "It will not be observed: %2 failed - %3(%4)") .arg(path_.c_str(), fanotifyEventMaskToStr(mask), translation::strerror_l()).arg(errno); if(Settings::instance().getMountIgnoreNoPerm() && errno == EACCES){ logDebug << msg; } else { logWarning << msg; } return false; } logDebug << "fanotify_mark" << fanotifyEventMaskToStr(mask) << path_; return true; } /// Fill param result with parentPaths and all sub-mountpaths, that is, /// all paths in allMountpaths, which are a sub-path of any parentPath, /// are added. void addPathsAndSubMountPaths(const std::shared_ptr& parentPaths, const std::shared_ptr& allMountpaths, StringSet& result){ for(const auto& p : *parentPaths){ // maybe_todo: avoid needless conversion result.insert(p.c_str()); for(auto mountIt = allMountpaths->subpathIter(p); mountIt != allMountpaths->end(); ++mountIt){ // maybe_todo: avoid needless conversion result.insert((*mountIt).constData()); } } } } // anonymous namespace /// Initialize fanotify's filedescriptor (requires root) /// @throws ExcOs FanotifyController::FanotifyController() : m_fanFd(-1), m_markLimitReached(false), m_ReadEventsUnregistered(false), r_wCfg(Settings::instance().writeFileSettings()), r_rCfg(Settings::instance().readFileSettings()), r_scriptCfg(Settings::instance().readEventScriptSettings()) { // Create the file descriptor for accessing the fanotify API m_fanFd = fanotify_init(FAN_CLOEXEC | FAN_NONBLOCK, O_RDONLY | O_LARGEFILE | O_CLOEXEC | O_NOATIME); if (m_fanFd == -1) { throw ExcOs("fanotify_init failed:"); } } FanotifyController::~FanotifyController(){ try { os::close(m_fanFd); } catch (const std::exception& e) { logCritical << __func__ << e.what(); } } int FanotifyController::fanFd() const { return m_fanFd; } int FanotifyController::getFanotifyMaxEventCount() const { return FANOTIFY_MAX_EVENT_COUNT; } void FanotifyController::setFileEventHandler(std::shared_ptr & feventHandler) { m_feventHandler = feventHandler; } /// fanotify_mark all paths of interest, that is all paths /// which shall be observed for read- or write-events. /// We unshared the mount-namespace before, so perform the /// mark by using mount-points. /// Also collect all mount-points, which are submounts of /// a desired path (if / shall be observed, e.g. /// mark filesystems under /media as well). /// Note that on marking a path, parent directories are possibly marked /// as well, if the mount-point lays above it. /// ( if mountpoint is /home and the dir /home/user/foo shall be observed, /// fanotify_mark marks /home, thus events occuring in /home and /home/user /// are also reported [and need to be filtered out later]). void FanotifyController::setupPaths(){ m_ReadEventsUnregistered = false; auto allMounts = mountController::generatelMountTree(); StringSet allWritePaths; addPathsAndSubMountPaths(r_wCfg.includePaths, allMounts, allWritePaths); StringSet allReadPaths; // Script files (which shall be stored) and 'normal' read files are treated differently later - // first mark unified paths from both categories for fanotify read-events. if(r_rCfg.enable){ addPathsAndSubMountPaths(r_rCfg.includePaths, allMounts, allReadPaths); } if(r_scriptCfg.enable){ addPathsAndSubMountPaths(r_scriptCfg.includePaths, allMounts, allReadPaths); } m_readMountPaths.reserve(allReadPaths.size()); uint64_t writeMask = FAN_CLOSE_WRITE; uint64_t readMask = FAN_CLOSE_NOWRITE; uint64_t readWriteMask = readMask | writeMask; for(const auto & p : allWritePaths){ auto pathInReadIt = allReadPaths.find(p); uint64_t m = writeMask; if(pathInReadIt != allReadPaths.end()){ // path interesting for both, read and write m = readWriteMask; allReadPaths.erase(pathInReadIt); } if(fanotifyMarkWrapOnInit(m_fanFd, m, p) && m & readMask){ // once the specified number of read files was collected, // the read paths shall be unregistered again. So store the paths. m_readMountPaths.push_back(p); } } // also add read paths not already marked above (along with write-paths). for(const auto & p : allReadPaths){ if(fanotifyMarkWrapOnInit(m_fanFd, readMask, p)){ m_readMountPaths.push_back(p); } } // ignore file events we generate ourselves ignoreOwnPath(db_connection::getDatabaseDir().toUtf8()); ignoreOwnPath(StoredFiles::getReadFilesDir().toUtf8()); ignoreOwnPath(logger::logDir().toUtf8()); assert(m_feventHandler != nullptr); ignoreOwnPath(m_feventHandler->getTmpDirPath().toUtf8()); } /// Handle fanotify events. /// For a general introduction please see man fanotify. bool FanotifyController::handleEvents() { struct fanotify_event_metadata *metadata; struct fanotify_event_metadata buf[FANOTIFY_MAX_EVENT_COUNT]; ssize_t len; // Loop while events can be read from fanotify file descriptor while(true) { // Read some events len = read(m_fanFd, buf, sizeof(buf)); if (unlikely(len == -1 && errno != EAGAIN)) { const auto preamble = qtr("read from fanotify file descriptor failed:"); // maybe_todo: file a bug to the fanotify-devs? According to man 7 fanotify // there should be no permission check, when the kernel repoens the file // for fanotify... // Furthermore it is unclear whether other events in the queue after a // bad fd are gone as well. switch (errno) { case ENOENT: break; // a deleted file is not of interest anyway. ignore. case EACCES: logInfo << preamble << qtr("EACCES most likely occurred, because a not readable " "file was closed on a NFS-storage, or similar."); break; default: logWarning << preamble << "(" + QString::number(errno) + ") -" << translation::strerror_l(); break; } return false; } // Check if end of available data reached if (len <= 0) { return true; } if(static_cast(len) / sizeof(fanotify_event_metadata) < FANOTIFY_MAX_EVENT_COUNT / 8 ){ // Avoid reading too few events at a time (read-overhead). This sleep ensures // the next read won't happen too soon. usleep(1000*50); } logDebug << "read" << len << "bytes (" << static_cast(len) / sizeof(fanotify_event_metadata) << "events)"; // Point to the first event in the buffer metadata = buf; // Loop over all events in the buffer while (FAN_EVENT_OK(metadata, len)) { // Check that run-time and compile-time structures match if (unlikely(metadata->vers != FANOTIFY_METADATA_VERSION)) { logCritical << qtr("Mismatch of fanotify metadata version - runtime: %1, " "compiletime: %2. " "No event-processing takes place. " "Please recompile the application against the current " "Kernel").arg(metadata->vers, FANOTIFY_METADATA_VERSION); // maybe_todo: unregister from all events? return false; } // metadata->fd contains either FAN_NOFD, indicating a // queue overflow, or a file descriptor (a nonnegative // integer). if (unlikely(metadata->fd < 0)) { logWarning << "fanotify: queue overflow"; m_overflowCount++; } else { handleSingleEvent(*metadata); ::close(metadata->fd); } // Advance to next event metadata = FAN_EVENT_NEXT(metadata, len); } // while (FAN_EVENT_OK(metadata, len)) } // while true } void FanotifyController::handleSingleEvent( const struct fanotify_event_metadata& metadata){ if(unlikely(metadata.mask & FAN_Q_OVERFLOW)){ logWarning << "fanotify: queue overflow"; m_overflowCount++; } #ifndef NDEBUG { auto st = os::fstat(metadata.fd); std::string path; try { path = os::readlink("/proc/self/fd/" + std::to_string(metadata.fd)); } catch (const os::ExcOs& ex) { logDebug << ex.what(); path = "UNKNOWN"; } auto action = fanotifyEventMaskToStr(metadata.mask); logDebug << action << "event-pid" << metadata.pid << path << "fd:" << metadata.fd << "uid: " << st.st_uid << " gid: " << st.st_gid; } #endif if(metadata.mask & FAN_CLOSE_NOWRITE){ handleCloseRead_safe(metadata); } if (metadata.mask & FAN_CLOSE_WRITE) { handleModCloseWrite_safe(metadata); } } /// Handle a 'read'-event. /// If read 'script' files shall be stored, but not general read files, /// unregister from read events, as soon as the specified number of script /// files was collected. void FanotifyController::handleCloseRead_safe(const fanotify_event_metadata &metadata){ if(unlikely(m_ReadEventsUnregistered)){ // Do not edit: even if successfully unregistered, // events in the fanotify event-queue may still need to be consumed. return; } if(unlikely(! r_rCfg.enable && // never unregister, if general read files are logged r_scriptCfg.enable && m_feventHandler->fileEvents().rStoredFilesCount() >= r_scriptCfg.maxCountOfFiles)) { unregisterAllReadPaths(); m_ReadEventsUnregistered = true; return; } try { m_feventHandler->handleCloseRead(metadata.fd); // The count of cached read (script-) files might have been incremented, // so we might be done with read events. For the sake // of code-shortness only check that the *next* time we consume a read event. } catch (const std::exception & e) { logCritical << e.what(); } } void FanotifyController::handleModCloseWrite_safe( const fanotify_event_metadata & metadata){ try { m_feventHandler->handleCloseWrite( metadata.fd ); } catch (const std::exception & e) { logCritical << e.what(); } } /// unregister read events for all previously marked paths. void FanotifyController::unregisterAllReadPaths() { logDebug << "enough read script-files collected. Unregistering..."; for(const auto& p : m_readMountPaths){ if (fanotify_mark(m_fanFd, FAN_MARK_REMOVE | FAN_MARK_MOUNT, FAN_CLOSE_NOWRITE, AT_FDCWD, p.c_str()) == -1) { logInfo << "fanotify_mark: failed to remove read-path " << p <<": " << translation::strerror_l(); } } } void FanotifyController::ignoreOwnPath(const QByteArray& p){ if (fanotify_mark(m_fanFd, FAN_MARK_ADD | FAN_MARK_IGNORED_MASK | FAN_MARK_IGNORED_SURV_MODIFY | FAN_MARK_ONLYDIR, FAN_ALL_EVENTS, -1, p.constData()) == -1){ // should never happen... logCritical << "fanotify_mark: failed to ignore our own path: " << translation::strerror_l(errno); } } uint FanotifyController::getOverflowCount() const { return m_overflowCount; } ================================================ FILE: src/shournal-run-fanotify/fanotify_controller.h ================================================ #pragma once #include #include #include "fileeventhandler.h" #include "util.h" struct fanotify_event_metadata; class FanotifyController { public: FanotifyController(); ~FanotifyController(); void setFileEventHandler(std::shared_ptr&); void setupPaths(); bool handleEvents(); int fanFd() const; int getFanotifyMaxEventCount() const; uint getOverflowCount() const; public: Q_DISABLE_COPY(FanotifyController) DISABLE_MOVE(FanotifyController) private: void handleSingleEvent(const fanotify_event_metadata &metadata); void handleCloseRead_safe(const fanotify_event_metadata &metadata); void handleModCloseWrite_safe(const fanotify_event_metadata &metadata); void unregisterAllReadPaths(); void ignoreOwnPath(const QByteArray& p); std::shared_ptr m_feventHandler; uint m_overflowCount{0}; int m_fanFd; bool m_markLimitReached; bool m_ReadEventsUnregistered; std::vector m_readMountPaths; // all mount paths initially marked for read-events const Settings::WriteFileSettings& r_wCfg; const Settings::ReadFileSettings& r_rCfg; const Settings::ScriptFileSettings& r_scriptCfg; }; ================================================ FILE: src/shournal-run-fanotify/filewatcher_fan.cpp ================================================ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "compat.h" #include "filewatcher_fan.h" #include "fanotify_controller.h" #include "mount_controller.h" #include "os.h" #include "osutil.h" #include "oscaps.h" #include "cleanupresource.h" #include "fdcommunication.h" #include "util.h" #include "logger.h" #include "subprocess.h" #include "excos.h" #include "db_globals.h" #include "db_connection.h" #include "db_controller.h" #include "commandinfo.h" #include "translation.h" #include "subprocess.h" #include "app.h" #include "pathtree.h" #include "fileeventhandler.h" #include "orig_mountspace_process.h" #include "cpp_exit.h" #include "qfilethrow.h" #include "storedfiles.h" #include "sys_ioprio.h" #include "qoutstream.h" #include "conversions.h" #include "socket_message.h" #include "shournal_run_common.h" #include "stdiocpp.h" #include "kernel/shournalk_user.h" using socket_message::E_SocketMsg; using SocketMessages = fdcommunication::SocketCommunication::Messages; using subprocess::Subprocess; using osutil::closeVerbose; const int PRIO_FANOTIFY_POLL = 2; const int PRIO_DATABASE_FLUSH = 10; static void unshareOrDie(){ try { os::unshare( CLONE_NEWNS); } catch (const os::ExcOs& e) { logCritical << e.what(); if(os::geteuid() != 0){ logCritical << qtr("Note that the effective userid is not 0 (root), so most probably %1 " "does not have the setuid-bit set. As root execute:\n" "chown root %1 && chmod u+s %1").arg(app::CURRENT_NAME); } cpp_exit(1); } } /// Other applications unsharing their mount-namespace might rely on the /// fact that they cannot be joined (except from root). Therefor shournal /// allows only joining of processes whose (effective) gid matches /// below group. static gid_t findMsenterGidOrDie(){ auto* groupInfo = getgrnam(app::MSENTER_ONLY_GROUP); if(groupInfo == nullptr){ logCritical << qtr("group %1 does not exist on your " "system but is required. Please add it:\n" "groupadd %1").arg(app::MSENTER_ONLY_GROUP); cpp_exit(1); } return groupInfo->gr_gid; } /// The childprocess's mount-namespace can be joined by shournal-run (msenter). /// It has a group-id which should be used solely for this purpose which /// serves as a permission check, so shournal-run cannot be used to join /// processes which were not 'created' by it. FileWatcher::MsenterChildReturnValue FileWatcher::setupMsenterTargetChildProcess(){ assert(os::geteuid() == os::getuid()); os::seteuid(0); // set ids before fork, so parent does not need to wait for child // (msenter uid and gid permission check!) os::setegid(m_msenterGid); os::seteuid(m_realUid); auto pipe_ = os::pipe(); auto msenterPid = os::fork(); if(msenterPid != 0){ // parent os::seteuid(0); os::setegid(os::getgid()); os::seteuid(m_realUid); os::close(pipe_[0]); return {msenterPid, pipe_[1]}; } // child if(m_sockFd != -1){ // the socket is used to wait for other processes, not this one, so: os::close(m_sockFd); } os::close(pipe_[1]); char c; // wait unitl parent-process closes its write-end os::read(pipe_[0], &c, 1); exit(0); } FileWatcher::FileWatcher() : m_sockFd(-1), m_msenterGid(std::numeric_limits::max()), m_commandArgc(0), m_commandFilename(nullptr), m_commandArgv(nullptr), m_commandEnvp(environ), m_realUid(os::getuid()), m_storeToDatabase(true) {} void FileWatcher::setupShellLogger() { m_shellLogger.setFullpath(logger::logDir() + "/log_" + app::SHOURNAL + "_shell_integration"); m_shellLogger.setup(); } std::shared_ptr FileWatcher::createFileEventHandler() { // fevent-handler sets up a temp dir in constructor - use user privileges. // ld.so clears TMPDIR from the envirnoment of suid-binaries for security reasons, // so set it temporarily: assert(os::geteuid() == os::getuid()); static const char* TMPDIR = "TMPDIR"; const char* oldTmp = getenv(TMPDIR); auto resetTmpDir = finally([&oldTmp] { if(oldTmp == nullptr) unsetenv(TMPDIR); else os::setenv(TMPDIR, oldTmp); }, false); if(! m_tmpDir.isEmpty()){ os::setenv(TMPDIR, m_tmpDir.constData()); resetTmpDir.setEnabled(true); } return std::make_shared(); } /// Unshare the mount-namespace and mark the interesting mounts with fanotify according /// to the paths specified in settings. /// Then either start a new process (passed argv) or wait until the passed socket is closed. /// In this case, we are in the shell observation mode. /// To allow other processes to join (--msenter), we fork off a child process with a /// special group id, which waits for us to finish. /// Process fanotify events until the observed process finishes (first case) or until /// all other instances of the passed socket are closed by the observed processes. /// See also code in directory 'shell-integration'. void FileWatcher::run() { m_msenterGid = findMsenterGidOrDie(); orig_mountspace_process::setupIfNotExist(); m_fEventHandler = createFileEventHandler(); os::seteuid(0); unshareOrDie(); auto fanotifyCtrl = FanotifyController_ptr(new FanotifyController); fanotifyCtrl->setFileEventHandler(m_fEventHandler); fanotifyCtrl->setupPaths(); // We process events (filedescriptor-receive- and fanotify-events) with the // effective uid of the caller, because read events for files, for which // only the owner has read permission, usually fail for // root in case of NFS-storages. See also man 5 exports, look for 'root squashing'. os::seteuid(m_realUid); // maybe_todo: change scheduler? // struct sched_param sched{}; // sched.sched_priority = 0; // if(sched_setscheduler(getpid(), SCHED_BATCH | SCHED_RESET_ON_FORK, &sched) == -1){ // logInfo << __FILE__ << "sched_setscheduler failed" << translation::strerror_l(errno) ; // } CommandInfo cmdInfo = CommandInfo::fromLocalEnv(); cmdInfo.sessionInfo.uuid = m_shellSessionUUID; int ret = 1; m_sockCom.setReceiveBufferSize(RECEIVE_BUF_SIZE); E_SocketMsg pollResult; if(m_commandArgc != 0){ if(m_commandFilename != nullptr){ cmdInfo.text += QString(m_commandFilename) + " "; } cmdInfo.text += argvToQStr(m_commandArgc, m_commandArgv); auto sockPair = os::socketpair(PF_UNIX, SOCK_STREAM | SOCK_CLOEXEC ); m_sockCom.setSockFd(sockPair[0]); Subprocess proc; proc.setAsRealUser(true); proc.setEnviron(m_commandEnvp); cmdInfo.startTime = QDateTime::currentDateTime(); // TOODO: evtl. allow to configure proc to not close one of our sockets, // to wait on grandchildren. // Remove SOCK_CLOEXEC for one of them in that case const char* cmdFilename = (m_commandFilename == nullptr) ? m_commandArgv[0] : m_commandFilename; proc.call(cmdFilename, m_commandArgv); // *Must* be called after fork (resurce limits, etc.) std::future thread = std::async(&FileWatcher::fan_pollUntilStopped, this, std::ref(cmdInfo), std::ref(fanotifyCtrl)); try { cmdInfo.returnVal = proc.waitFinish(); } catch (const os::ExcProcessExitNotNormal& ex) { // return typical shell cpp_exit code cmdInfo.returnVal = 128 + ex.status(); } ret = cmdInfo.returnVal; // that should stop the polling event loop: os::close(sockPair[1]); thread.wait(); os::close(sockPair[0]); pollResult = thread.get(); } else if(m_sockFd != -1){ MsenterChildReturnValue msenterChildRet = setupMsenterTargetChildProcess(); auto closeMsenterWritePipe = finally([&msenterChildRet] { os::close(msenterChildRet.pipeWriteEnd); os::waitpid(msenterChildRet.pid); }); m_sockCom.setSockFd(m_sockFd); // should be overwritten later, null-constraint in db... cmdInfo.startTime = QDateTime::currentDateTime(); setupShellLogger(); int rootDirFd = os::open("/", O_RDONLY | O_DIRECTORY); auto closeRootDir = finally([&rootDirFd] { closeVerbose(rootDirFd);} ); SocketMessages sockMesgs; m_sockCom.sendMsg({int(E_SocketMsg::SETUP_DONE), qBytesFromVar(msenterChildRet.pid), rootDirFd}); pollResult = fan_pollUntilStopped(cmdInfo, fanotifyCtrl); ret = 0; } else { pollResult = E_SocketMsg::ENUM_END; assert(false); } cmdInfo.endTime = QDateTime::currentDateTime(); logDebug << "polling finished - about to cleanup and exit"; auto fanOveflowCount = fanotifyCtrl->getOverflowCount(); fanotifyCtrl.reset(); switch (pollResult) { case E_SocketMsg::EMPTY: break; // Normal case case E_SocketMsg::ENUM_END: logCritical << qtr("Because an error occurred, processing of " "fanotify/socket-events was " "stopped"); cpp_exit(ret); default: logWarning << "unhandled case for pollResult: " << int(pollResult); break; } QStringList missingFields; if(cmdInfo.text.isEmpty()){ // An empty command text should only occur, if the observed shell-session // exits. Discard this command. logDebug << "command-text is empty, " "not pushing to database..."; cpp_exit(ret); } if(cmdInfo.returnVal == CommandInfo::INVALID_RETURN_VAL){ missingFields += qtr("return value"); } if(! missingFields.isEmpty()){ logDebug << "The following fields are empty: " << missingFields.join(", "); } if(m_printSummary){ auto & fevents = m_fEventHandler->fileEvents(); QString overflowEvents = (fanOveflowCount) ? ">= " + QString::number(fanOveflowCount) : "0"; QErr() << qtr("=== %1 summary ===\n" "number of write-events: %2\n" "number of read-events: %3\n" "number of lost events: %4\n" "number of stored read files: %5\n" "size of tmp-file: %6\n") .arg(app::CURRENT_NAME) .arg(fevents.wEventCount()) .arg(fevents.rEventCount()) .arg(overflowEvents) .arg(fevents.rStoredFilesCount()) .arg(Conversions().bytesToHuman( os::fstat(fileno(fevents.file())).st_size)); } if(m_storeToDatabase){ flushToDisk(cmdInfo); } cpp_exit(ret); } void FileWatcher::setShellSessionUUID(const QByteArray &shellSessionUUID) { m_shellSessionUUID = shellSessionUUID; } void FileWatcher::setArgv(char **argv, int argc) { m_commandArgv = argv; m_commandArgc = argc; } void FileWatcher::setCommandEnvp(char **commandEnv) { m_commandEnvp = commandEnv; } void FileWatcher::setSockFd(int sockFd) { m_sockFd = sockFd; } int FileWatcher::sockFd() const { return m_sockFd; } void FileWatcher::setStoreToDatabase(bool storeToDatabase) { m_storeToDatabase = storeToDatabase; } void FileWatcher::setPrintSummary(bool printSummary) { m_printSummary = printSummary; } void FileWatcher::setTmpDir(const QByteArray &tmpDir) { m_tmpDir = tmpDir; } void FileWatcher::setCommandFilename(char *commandFilename) { m_commandFilename = commandFilename; } /// @return E_SocketMsg::EMPTY, if processing shall be stopped E_SocketMsg FileWatcher::processSocketEvent( CommandInfo& cmdInfo ){ m_sockCom.receiveMessages(&m_sockMessages); E_SocketMsg returnMsg = E_SocketMsg::ENUM_END; for(auto & msg : m_sockMessages){ if(msg.bytes.size() > RECEIVE_BUF_SIZE - 1024*10){ logWarning << "unusual large message received"; } if(msg.msgId == -1){ return E_SocketMsg::EMPTY; } assert(msg.msgId >=0 && msg.msgId < int(E_SocketMsg::ENUM_END)); returnMsg = E_SocketMsg(msg.msgId); logDebug << "received message:" << socket_message::socketMsgToStr(E_SocketMsg(msg.msgId)) << msg.bytes; switch (E_SocketMsg(msg.msgId)) { case E_SocketMsg::COMMAND: { cmdInfo.text = msg.bytes; break; } case E_SocketMsg::CMD_START_DATETIME: { cmdInfo.startTime = QDateTime::fromString(QString::fromUtf8(msg.bytes), Conversions::dateIsoFormatWithMilliseconds()); assert(! cmdInfo.startTime.isNull()); break; } case E_SocketMsg::RETURN_VALUE: { cmdInfo.returnVal = varFromQBytes(msg.bytes); break; } case E_SocketMsg::LOG_MESSAGE: m_shellLogger.stream() << msg.bytes << Qt::endl; break; case E_SocketMsg::CLEAR_EVENTS: m_fEventHandler->clearEvents(); // maybe_todo: also clear fanotify overflow events // (which occurred very unlikely in this case) cmdInfo.startTime = QDateTime::currentDateTime(); break; default: { // application bug? returnMsg = E_SocketMsg::EMPTY; logCritical << qtr("invalid message received - : %1").arg(int(msg.msgId)); break; } } } assert(returnMsg != E_SocketMsg::ENUM_END); return returnMsg; } void FileWatcher::flushToDisk(CommandInfo& cmdInfo){ assert(os::getegid() == os::getgid()); assert(os::geteuid() == os::getuid()); // Do not disturb other processes while we flush events to database os::setpriority(PRIO_PROCESS, 0, PRIO_DATABASE_FLUSH); try { cmdInfo.idInDb = db_controller::addCommand(cmdInfo); StoredFiles::mkpath(); stdiocpp::fseek(m_fEventHandler->fileEvents().file(), 0, SEEK_SET); db_controller::addFileEvents(cmdInfo, m_fEventHandler->fileEvents()); } catch (std::exception& e) { // May happen, e.g. if we run out of disk space... logCritical << qtr("Failed to store (some) file-events to disk: %1").arg(e.what()); } } /// @return: EMPTY, if stopped regulary /// ENUM_END in case of an error E_SocketMsg FileWatcher::fan_pollUntilStopped(CommandInfo& cmdInfo, FanotifyController_ptr& fanotifyCtrl){ // To allow for more fanotify-events read at a time, increase // RLIMIT_NOFILE struct rlimit rlim{}; getrlimit(RLIMIT_NOFILE, &rlim); const auto NO_FILE = fanotifyCtrl->getFanotifyMaxEventCount(); rlim.rlim_cur = NO_FILE; if(setrlimit(RLIMIT_NOFILE, &rlim) == -1){ logInfo << qtr("Failed to set number of open files to %1 - %2") .arg(NO_FILE) .arg(translation::strerror_l(errno)); } // At least on centos 7 with Kernel 3.10 CAP_SYS_PTRACE is required, otherwise // EACCES occurs on readlink of the received file descriptors // Warning: changing euid from 0 to nonzero resets the effective capabilities, // so don't do that until processing finished. auto caps = os::Capabilites::fromProc(); const os::Capabilites::CapFlags eventProcessingCaps { CAP_SYS_PTRACE, CAP_SYS_NICE }; caps->setFlags(CAP_EFFECTIVE, { eventProcessingCaps }); auto resetEventProcessingCaps = finally([&caps, &eventProcessingCaps] { caps->clearFlags(CAP_EFFECTIVE, eventProcessingCaps); }); os::setpriority(PRIO_PROCESS, 0, PRIO_FANOTIFY_POLL); auto resetPriority = finally([] { os::setpriority(PRIO_PROCESS, 0, 0); }); if(syscall(SYS_ioprio_set, IOPRIO_WHO_PROCESS, 0, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 6))){ logWarning << "Failed to set io-priority:" << strerror(errno); } int poll_num; const nfds_t nfds = 2; struct pollfd fds[nfds]; fds[0].fd = m_sockCom.sockFd(); fds[0].events = POLLIN; // Fanotify input fds[1].fd = fanotifyCtrl->fanFd(); fds[1].events = POLLIN; while (true) { // cleanly cpp_exit poll: // poll for two file descriptors: the fanotify descriptor and // another one, which receives an cpp_exit-message). poll_num = poll(fds, nfds, -1); if (poll_num == -1) { if (errno == EINTR){ // Interrupted by a signal continue; // Restart poll() } logCritical << qtr("poll failed (%1) - %2").arg(errno) .arg(translation::strerror_l()); return E_SocketMsg::ENUM_END; } // 0 only on timeout, which is infinite assert(poll_num != 0); // Important: first handle fanotify events, then check the socket if we are done. // Otherwise final fanotify-events might get lost! if (fds[1].revents & POLLIN) { // Fanotify events are available logDebug << "new fanotify events..."; fanotifyCtrl->handleEvents(); } if (fds[0].revents & POLLIN) { if(processSocketEvent(cmdInfo) == E_SocketMsg::EMPTY){ return E_SocketMsg::EMPTY; } } } } ================================================ FILE: src/shournal-run-fanotify/filewatcher_fan.h ================================================ #pragma once #include "logger.h" #include "fileeventhandler.h" #include "fanotify_controller.h" #include "socket_message.h" #include "fdcommunication.h" class FanotifyController; struct CommandInfo; class FileWatcher { public: FileWatcher(); void setupShellLogger(); [[noreturn]] void run(); void setShellSessionUUID(const QByteArray &shellSessionUUID); void setArgv(char** argv, int argc); void setCommandEnvp(char **commandEnv); void setCommandFilename(char *commandFilename); void setSockFd(int sockFd); int sockFd() const; void setStoreToDatabase(bool storeToDatabase); void setPrintSummary(bool printSummary); void setTmpDir(const QByteArray &tmpDir); private: struct MsenterChildReturnValue { MsenterChildReturnValue(pid_t p, int pipeWrite) : pid(p), pipeWriteEnd(pipeWrite){} pid_t pid; int pipeWriteEnd; }; typedef std::unique_ptr FanotifyController_ptr; static const int RECEIVE_BUF_SIZE = 1024*1024; int m_sockFd; logger::LogRotate m_shellLogger; std::shared_ptr m_fEventHandler; QByteArray m_tmpDir; gid_t m_msenterGid; fdcommunication::SocketCommunication m_sockCom; QByteArray m_shellSessionUUID; int m_commandArgc; char* m_commandFilename; char **m_commandArgv; char ** m_commandEnvp; uid_t m_realUid; bool m_printSummary{}; fdcommunication::SocketCommunication::Messages m_sockMessages; bool m_storeToDatabase; std::shared_ptr createFileEventHandler(); MsenterChildReturnValue setupMsenterTargetChildProcess(); socket_message::E_SocketMsg fan_pollUntilStopped(CommandInfo& cmdInfo, FanotifyController_ptr& fanotifyCtrl); socket_message::E_SocketMsg processSocketEvent( CommandInfo& cmdInfo ); void flushToDisk(CommandInfo& cmdInfo); }; ================================================ FILE: src/shournal-run-fanotify/mount_controller.cpp ================================================ #include #include #include #include #include #include #include #include #include #include #include "mount_controller.h" #include "util.h" #include "settings.h" #include "excos.h" #include "os.h" #include "pathtree.h" #include "logger.h" #include "cleanupresource.h" /// Return all mountpaths from /proc/self/mounts except the /// ones marked to be ignored (settings). std::shared_ptr mountController::generatelMountTree(){ auto & ignoreMountPaths = Settings::instance().getMountIgnorePaths(); PathTree ignoreMountTree; for(const auto & path : ignoreMountPaths){ ignoreMountTree.insert(path); } // iterate over all of our mounts FILE* mounts = setmntent ("/proc/self/mounts", "r"); if (mounts == nullptr) { throw ExcMountCtrl("setmntent /proc/self/mounts failed"); } auto closeLater = finally([&mounts] { endmntent (mounts); }); // Determine which submounts shall be ignored and // collect the others auto mountTree = std::make_shared(); struct mntent* mnt_; while ((mnt_ = getmntent (mounts)) != nullptr) { const StrLight mntDir(mnt_->mnt_dir); if(ignoreMountTree.isSubPath(mntDir, true)){ logDebug << "ignoring mountpath" << mntDir.c_str(); continue; } mountTree->insert(mntDir); } return mountTree; } ================================================ FILE: src/shournal-run-fanotify/mount_controller.h ================================================ #pragma once #include #include #include #include "exccommon.h" #include "settings.h" class ExcMountCtrl : public QExcCommon { public: using QExcCommon::QExcCommon; }; namespace mountController { std::shared_ptr generatelMountTree(); } ================================================ FILE: src/shournal-run-fanotify/msenter.cpp ================================================ /* Allow joining a mount-namespace created by shournal. * The most elegant solution would have been to pass the mnt-fd to the observed shell * process and let that one call setns before each command sequence. That is * however not allowed. TODO: suggest the Kernel devs, to skip permission checks * for setns if the respective fd in /proc/$pid/ns was opened by root. * Instead we call setns before exec, which we can, because being a setuid-program. * Thus we should perform some permission checks, to only allow joining mount-namespaces created * by shournal. * The target pid's user-id must be the same as the caller, its group *must* be * app::MSENTER_ONLY_GROUP. * Reenter the working directory (checked race condition, because using the dirfd does not work). * */ #include "msenter.h" #include #include #include #include #include #include #include "os.h" #include "excos.h" #include "qoutstream.h" #include "util.h" #include "translation.h" #include "osutil.h" #include "pidcontrol.h" #include "cleanupresource.h" #include "app.h" #include "logger.h" #include "fdentries.h" using osutil::closeVerbose; /// @overload void msenter::run(pid_t targetPid, const char* filename, char *commandArgv[], char **envp) { int targetprocDirFd = os::open("/proc/" + std::to_string(targetPid), O_DIRECTORY); auto closeFdLater = finally([&targetprocDirFd] { closeVerbose(targetprocDirFd); }); run(filename, commandArgv, envp, targetprocDirFd); } /// @param targetprocDirFd: an open directory descriptor of the target process. /// The caller is responsible, for closing it, if desired void msenter::run(const char* filename, char *commandArgv[], char **envp, int targetprocDirFd) { struct stat targetPidSt = os::fstat(targetprocDirFd); auto* allowedGroupInfo = getgrnam(app::MSENTER_ONLY_GROUP); if(allowedGroupInfo == nullptr){ logCritical << qtr("group %1 does not exist on your " "system but is required. Please add it.").arg(app::MSENTER_ONLY_GROUP); exit(1); } auto realUid = os::getuid(); if (realUid != targetPidSt.st_uid ) { logCritical << qtr("Target process belongs to a " "different user."); exit(1); } if( allowedGroupInfo->gr_gid != targetPidSt.st_gid){ logCritical << qtr("The group of the target process is not '%1'") .arg(app::MSENTER_ONLY_GROUP); exit(1); } bool setnsSuccess = false; bool openOrigCwdSuccess = false; try { // Remember the old working dir (setns changes it). Note that it is // not possible to fchdir back to oldWdFd, because doing // so leads also back to the original mountspace... // Opening the dir as *real* user is essential on NFS -> permissions. os::seteuid(realUid); const int oldWdFd = os::open(".", O_DIRECTORY); openOrigCwdSuccess = true; os::seteuid(os::getsuid()); int mntFd = os::openat(targetprocDirFd , "ns/mnt" , O_RDONLY); os::setns(mntFd, CLONE_NEWNS); setnsSuccess = true; os::close(mntFd); // Drop root privileges, irrevocable. os::setuid(realUid); try { const int newWdFd = osutil::reopenFdByPath( oldWdFd, O_DIRECTORY, true, false); auto closeNewWdLater = finally([&newWdFd] { closeVerbose(newWdFd); }); // reenter the working directory in new mount namespace which is surely the same // filesystem entry os::fchdir(newWdFd); } catch (const os::ExcOs& e) { // Should almost never happen. In that case // enter the working dir in the original mount-namespace. // File-events, referring to relative paths might then be lost. logWarning << qtr("Failed to enter working directory within the new mount-namespace. " "Entering the original one instead. Some file-events might " "be lost. Reason: %1").arg(e.what()); os::fchdir(oldWdFd); } os::close(oldWdFd); } catch (const os::ExcOs & ex) { logCritical << ex.what(); if(! openOrigCwdSuccess){ QString moreInfo; if( ex.errorNumber() == ESTALE){ moreInfo = qtr("Most probably it was deleted. "); } logCritical << qtr("Failed to open the working directory. %1").arg(moreInfo); } if(! setnsSuccess){ // most probably setns failed, because we are not suid. Since this program is execve'd // itself, above error message might not be visible for the user. So be gentle and // do not exit here. logCritical << qtr("Entering the mount-namespace at %1 failed, file events are not " "captured...") .arg(osutil::findPathOfFd(targetprocDirFd).data()); } os::setuid(os::getuid()); } execvpe(filename, commandArgv, envp); int err = errno; // Only get here on error. // Failed to launch the executable - print error and mimic shell-return-codes. translation::init(); QErr() << filename << ": " << translation::strerror_l(err) << "\n"; // In bash and zsh at least the following special cases exist: switch (err) { case EACCES: exit(126); case ENOENT: exit(127); // TODO: 128 + errno? default: exit(1); } } ================================================ FILE: src/shournal-run-fanotify/msenter.h ================================================ #pragma once #include namespace msenter { [[noreturn]] void run(pid_t targetPid, const char *filename, char *commandArgv[], char **envp); [[noreturn]] void run(const char* filename, char *commandArgv[], char **envp, int targetprocDirFd); } ================================================ FILE: src/shournal-run-fanotify/orig_mountspace_process.cpp ================================================ /* * To allow for leaving an observed mount namespace, which is e.g. helpful, * if another *interactive and observed* shell is launched within an observed * mount-namespace, the first time shournal starts (for a gien user) * a seperate process is started, * whose only task is to wait forever (until signaled). The process creates * a PID-file at the temporary dir, belongs to the real user and has the msenter group. * */ #include #include #include #include #include "orig_mountspace_process.h" #include "os.h" #include "app.h" #include "qoutstream.h" #include "logger.h" #include "msenter.h" #include "cleanupresource.h" #include "osutil.h" #include "fdentries.h" using osutil::closeVerbose; namespace { const char* LOCK = "_LOCK"; QString userPidPath(){ return pathJoinFilename(QStandardPaths::writableLocation(QStandardPaths::TempLocation), QString(app::SHOURNAL) + "-orig-mountnamespace-" + os::getUserName()) ; } /// The grandchild sets up a never ending process with the real userid /// and the msenter-groupid [[noreturn]] void setupIfNotExistAsChild(const QString& pidPath) { auto* allowedGroupInfo = getgrnam(app::MSENTER_ONLY_GROUP); if(allowedGroupInfo == nullptr){ logCritical << qtr("group %1 does not exist on your " "system but is required to setup the original " "mountnamespace-process. Please add it.").arg(app::MSENTER_ONLY_GROUP); exit(1); } os::seteuid(0); os::setgid(allowedGroupInfo->gr_gid); os::setuid(os::getuid()); QLockFile lockFile(pidPath + LOCK); if(! lockFile.lock()){ logCritical << qtr("failed to obtain lock on %1").arg(pidPath + LOCK); } if(QFileInfo::exists(pidPath)){ // file was created meanwhile exit(0); } logDebug << "creating new pid-file for the original mount-namespace"; QFile pidFile(pidPath); { if(! pidFile.open(QFile::OpenModeFlag::WriteOnly)){ // should never happen logCritical << "Failed to open pidfile" << pidPath; exit(1); } QTextStream stream(&pidFile); stream << os::getpid(); } pidFile.close(); lockFile.unlock(); // we are a daemon. Good practice: enter /, close all fds os::chdir("/"); logger::disableLogToFile(); for(const int fd : osutil::FdEntries()){ try { os::close(fd); } catch (const os::ExcOs& e) { QIErr() << e.what(); } } // wait for typical signals to exit osutil::waitForSignals(); pidFile.remove(); exit(0); } [[noreturn]] void execAsRealsUser(const char* filename, char *commandArgv[], char **envp){ os::setuid(os::getuid()); os::exec(filename, commandArgv, envp); // never get here } } // namespace /// Create a grandchild-process, which creates a pid-file (in a race-free way). void orig_mountspace_process::setupIfNotExist() { const QString pidPath = userPidPath(); if(QFileInfo::exists(pidPath)){ return; } auto pid = os::fork(); if(pid != 0){ // parent returns return; } // Prevent receiving signals for the process-group os::setsid(); setupIfNotExistAsChild(pidPath); // never get here } /// Enter a 'original' mount-namespace which was created before (setupIfNotExist) /// and excute a command in it. If there is no pid-file, assume, there is no such /// process yet and simply execve as 'real' user. /// In case of an error (pid-file points to invalid process, etc.) remove /// the pid-file and execve as 'real' user. void orig_mountspace_process::msenterOrig(const char *filename, char *commandArgv[], char **envp) { QFile pidFile(userPidPath()); if(! pidFile.open(QFile::OpenModeFlag::ReadOnly)){ // may happen, if file does not exist (we are the first process). // If it exists, report the error (yes, short race-condition here...) if(pidFile.exists()){ logWarning << qtr("Failed to open pidfile although" " it exists, not joining original namespace:") << pidFile.fileName(); } else { logDebug << "pidPath does not exist, not joining original namespace"; } execAsRealsUser(filename, commandArgv, envp); } QTextStream stream(&pidFile); try { auto targetPid = qVariantTo_throw(stream.readLine()); int targetprocDirFd = os::open("/proc/" + std::to_string(targetPid), O_DIRECTORY); auto closeTargetprocDirFdLater = finally([&targetprocDirFd] { closeVerbose(targetprocDirFd); }); msenter::run(filename, commandArgv, envp, targetprocDirFd); } catch (const std::exception& e) { logWarning << qtr("Cannot join original mount-namespace: %1. " "Maybe shournal-run was killed? Removing obsolete " "pid-file at %2...").arg(e.what()).arg(pidFile.fileName()); if(! pidFile.remove()){ logWarning << qtr("Removing pid-file %2 failed.").arg(pidFile.fileName()); } execAsRealsUser(filename, commandArgv, envp); } } ================================================ FILE: src/shournal-run-fanotify/orig_mountspace_process.h ================================================ #pragma once namespace orig_mountspace_process { void setupIfNotExist(); [[noreturn]] void msenterOrig(const char* filename, char *commandArgv[], char **envp); } ================================================ FILE: src/shournal-run-fanotify/shournal-run-fanotify.cpp ================================================ #include #include #include #include #include #include "qoptargparse.h" #include "qoptvarlenarg.h" #include "excoptargparse.h" #include "os.h" #include "osutil.h" #include "excos.h" #include "filewatcher_fan.h" #include "msenter.h" #include "logger.h" #include "fdcommunication.h" #include "exccfg.h" #include "settings.h" #include "util.h" #include "qoutstream.h" #include "util.h" #include "translation.h" #include "app.h" #include "qexcdatabase.h" #include "mount_controller.h" #include "orig_mountspace_process.h" #include "cpp_exit.h" #include "db_connection.h" #include "storedfiles.h" #include "socket_message.h" #include "shournal_run_common.h" using fdcommunication::SocketCommunication; using socket_message::E_SocketMsg; using namespace shournal_run_common; namespace { /// Uncaught exception handler void onterminate() { try { auto unknown = std::current_exception(); if (unknown) { std::rethrow_exception(unknown); } } catch (const std::exception& e) { logCritical << e.what() << "\n"; } catch (...) { logCritical << "unknown exception occurred\n"; } } [[noreturn]] void callFilewatcherSafe(FileWatcher& fwatcher){ try { fwatcher.run(); } catch (const os::ExcOs & ex) { logCritical << qtr("Sorry, need to close: ") << ex.what(); } catch(const ExcMountCtrl & ex){ logCritical << qtr("mount failed: ") << ex.descrip(); } if(fwatcher.sockFd() != -1){ SocketCommunication fdCom; fdCom.setSockFd(fwatcher.sockFd()); fdCom.sendMsg(int(E_SocketMsg::SETUP_FAIL)); } cpp_exit(1); } } // namespace int shournal_run_main(int argc, char *argv[]) { // Since we are waiting for other processes to finish, ignore typical // signals. osutil::setInertSighandler(os::catchableTermSignals()); // Using app::SHOURNAL for several common paths (database, config) used // by QStandardPaths but app::CURRENT_NAME for others (log-filename) app::setupNameAndVersion(app::SHOURNAL_RUN_FANOTIFY); if(! translation::init()){ logWarning << "Failed to initialize translation"; } if(os::geteuid() != 0){ QIErr() << qtr("%1 seems to lack the suid-bit (SETUID) for root. You can correct " "that by\n" "chown root %1 && chmod u+s %1").arg(app::CURRENT_NAME); // but continue to allow for e.g. msenter-orig to exec the process anyway... } logger::setup(app::CURRENT_NAME); std::set_terminate(onterminate); if(! shournal_common_init()){ logCritical << qtr("Fatal error: failed to initialize custom Qt conversion functions"); cpp_exit(1); } // ignore first arg (command to this app) --argc; ++argv; QOptArgParse parser; parser.setHelpIntroduction(qtr("Observation backend for <%1>. " "Not meant to be called by users directly, " "you would rather call %1 without trailing '-run'." ).arg(app::SHOURNAL) + "\n"); QOptArg argVersion("v", "version", qtr("Display version"), false); parser.addArg(&argVersion); // for communication with the shell-integration: QOptArg argSocketFd("", "socket-fd", "" ); argSocketFd.setInternalOnly(true); parser.addArg(&argSocketFd); QOptArg argExec("e", "exec", qtr("Execute and observe the passed program " "and its arguments (this argument has to be last)."), false); argExec.setFinalizeFlag(true); parser.addArg(&argExec); QOptArg argExecFilename("", "exec-filename", qtr("This is an advanced option. " "In most cases the first argument of a " "program is the program name. For " "example for login-shells this does " "not have to be the case. If this " "argument is provided, that filename " "is used instead of argv[0]")); parser.addArg(&argExecFilename); argExecFilename.addRequiredArg(&argExec); // Some variables like $TMPDIR are cleared in setuid-programs. // We allow them to be passed within argv and apply them during execve. // *after* setuid to original user. // see e.g. http://lists.gnu.org/archive/html/bug-glibc/2003-08/msg00076.html QOptVarLenArg argEnv("", "env", qtr("Specify an arbitrary number of environment variables. " "The first entry (integer) specifies the count " "of all the latter. The last entry is used internally and " "must be the string 'SHOURNAL_DUMMY_NULL=1'")); parser.addArg(&argEnv); // TODO: currently only interface compatibility. // We could indeed fork as well, at least in the exec-case. QOptArg argFork("", "fork", qtr("NOT USED"), false); argFork.setInternalOnly(true); parser.addArg(&argFork); QOptArg argTmpDir("", "tmpdir", qtr("Use the given TMPDIR (for non security-relevant stuff). " "As a setuid binary some variables are cleared from " "the environment (see also man 8 ld.so)")); // We expect to be called by the binary 'shournal' or the // shell integration, which both pass $TMPDIR using --tmpdir, so no need // to make it public. argTmpDir.setInternalOnly(true); parser.addArg(&argTmpDir); QOptArg argMsenter("", "msenter", qtr(". Must be passed along with '%1'. Execute the " "given command in an existing mountspace which was " "previously created via %2 %1") .arg(argExec.name(), app::CURRENT_NAME)); argMsenter.addRequiredArg(&argExec); parser.addArg(&argMsenter); QOptArg argMsenterOrig("", "msenter-orig-mountspace", qtr("Must be passed along with '%1'. Execute the " "given command in the 'original' mount-namespace " "created the first time %2 observed a process.") .arg(argExec.name(), app::CURRENT_NAME), false); argMsenterOrig.addRequiredArg(&argExec); parser.addArg(&argMsenterOrig); QOptArg argVerbosity("", "verbosity", qtr("How much shall be printed to stderr. Note that " "for 'dbg' shournal-run must not be a 'Release'-build.")); argVerbosity.setAllowedOptions(app::VERBOSITIES); parser.addArg(&argVerbosity); QOptArg argShellSessionUUID("", "shell-session-uuid", qtr("uuid as base64-encoded string")); argShellSessionUUID.setInternalOnly(true); parser.addArg(&argShellSessionUUID); QOptArg argNoDb("", "no-db", qtr("For debug purposes: do not write to" "database after event processing"), false); parser.addArg(&argNoDb); QOptArg argPrintSummary("", "print-summary", qtr("Print a short summary after " "event processing finished."), false); parser.addArg(&argPrintSummary); auto argCfgDir = mkarg_cfgdir(); parser.addArg(&argCfgDir); auto argDataDir = mkarg_datadir(); parser.addArg(&argDataDir); try { parser.parse(argc, argv); if(argVerbosity.wasParsed()){ QByteArray verbosity = argVerbosity.getOptions(1).first().toLocal8Bit(); logger::setVerbosityLevel(verbosity.constData()); } else { logger::setVerbosityLevel(QtMsgType::QtWarningMsg); } char** cmdEnv; if(argEnv.wasParsed()){ cmdEnv = argEnv.vals().argv; auto envArgc = argEnv.vals().len; assert(strcmp(cmdEnv[envArgc-1], "SHOURNAL_DUMMY_NULL=1" ) == 0); cmdEnv[envArgc-1] = nullptr; } else { cmdEnv = environ; } char* cmdFilename = nullptr; char** cmdArgv = nullptr; if(argExec.wasParsed()){ cmdArgv = parser.rest().argv; if(argExecFilename.wasParsed()){ cmdFilename = argExecFilename.vals().argv[0]; } else { cmdFilename = parser.rest().argv[0]; } } // Don't waste time, msenter has to run as early as possible if(argMsenter.wasParsed()){ msenter::run(argMsenter.getValue(), cmdFilename, cmdArgv, cmdEnv); } if(argExec.wasParsed() && argSocketFd.wasParsed() ) { QIErr() << qtr("%1 and %2 are mutually exclusive").arg(argExec.name(), argSocketFd.name()); cpp_exit(1); } if(argVersion.wasParsed()){ QOut() << app::CURRENT_NAME << qtr(" version ") << app::version().toString() << "\n"; cpp_exit(0); } auto & sets = Settings::instance(); if(argCfgDir.wasParsed()){ sets.setUserCfgDir(argCfgDir.getValue()); } if(argDataDir.wasParsed()){ sets.setUserDataDir(argDataDir.getValue()); } // has to be before argExec if(argMsenterOrig.wasParsed()){ // do not crash here if called from shell-integration, if // we forgot to make shournal suid... bool weAreAnotheUser = os::geteuid() != os::getuid(); if(weAreAnotheUser) os::seteuid(os::getuid()); logger::enableLogToFile(app::CURRENT_NAME); if(weAreAnotheUser) os::seteuid(0); // [[noreturn]] orig_mountspace_process::msenterOrig(cmdFilename, cmdArgv, cmdEnv); } // ------------------------------ // // In file observation mode (or invalid commandline-input) // Starting from this line, we effectively work as non-root // and only (shortly) switch to root to unshare the mount namespace // and initialize fanotify. // Also load settings and enable logging already, *before* unsharing the // mount-namespace, so we do not log events, we create ourselves. os::seteuid(os::getuid()); FileWatcher fwatcher; fwatcher.setCommandEnvp(cmdEnv); if(argExecFilename.wasParsed()){ // fwatcher command-filename must otherwise be null, to allow // for correct storing of command in db (no duplicate first arg // if not necessary!) fwatcher.setCommandFilename(argExecFilename.vals().argv[0]); } if(argTmpDir.wasParsed()){ fwatcher.setTmpDir(argTmpDir.getValue()); } fwatcher.setStoreToDatabase(! argNoDb.wasParsed()); fwatcher.setPrintSummary(argPrintSummary.wasParsed()); try { logger::enableLogToFile(app::CURRENT_NAME); sets.load(); StoredFiles::mkpath(); } catch(const qsimplecfg::ExcCfg & ex){ QIErr() << qtr("Failed to load config file: ") << ex.descrip(); cpp_exit(1); } catch(const QExcDatabase & ex){ QIErr() << qtr("Database-operation failed: ") << ex.descrip(); cpp_exit(1); } catch (const QExcIo& ex){ logCritical << qtr("IO-operation failed: ") << ex.descrip(); cpp_exit(1); } catch (const os::ExcOs& ex){ logCritical << ex.what(); cpp_exit(1); } if(argShellSessionUUID.wasParsed()){ fwatcher.setShellSessionUUID( QByteArray::fromBase64(argShellSessionUUID.getValue())); } if(argSocketFd.wasParsed()){ int socketFd = argSocketFd.getValue(-1); os::setFdDescriptorFlags(socketFd, FD_CLOEXEC); fwatcher.setSockFd(socketFd); callFilewatcherSafe(fwatcher); // [[noreturn]] } if(argExec.wasParsed()){ assert(!argMsenterOrig.wasParsed()); auto externCmd = parser.rest(); fwatcher.setArgv(externCmd.argv, externCmd.len); callFilewatcherSafe(fwatcher); // [[noreturn]] } if(parser.rest().len != 0){ QIErr() << qtr("Invalid parameters passed: %1.\n" "Show help with --help"). arg( argvToQStr(parser.rest().len, parser.rest().argv)); cpp_exit(1); } QIErr() << "No action specified"; } catch (const ExcOptArgParse & ex) { QIErr() << qtr("Commandline seems to be erroneous:") << ex.descrip(); } cpp_exit(1); } int main(int argc, char *argv[]) { try { shournal_run_main(argc, argv); } catch (const ExcCppExit& e) { return e.ret(); } } ================================================ FILE: test/CMakeLists.txt ================================================ include_directories( ../kernel ../src/common ../src/common/qsimplecfg ../src/common/oscpp ../src/common/util ../src/common/qsqlthrow ) enable_testing() find_package(Qt5Test REQUIRED) add_definitions( -DSHOURNALTEST_SQLITE_v_2_2="${CMAKE_CURRENT_SOURCE_DIR}/sqlite_sample_db_v2_2") add_executable(runTests main.cpp autotest.h test_cfg.cpp test_pathtree.cpp test_db_controller.cpp test_cxxhash.cpp test_fileeventhandler.cpp test_fdcommunication.cpp test_osutil.cpp test_qformattedstream.cpp test_qoptargparse.cpp test_util.cpp integration_test_shell.cpp helper_for_test.cpp ) add_test(NAME tests COMMAND runTests) target_link_libraries(runTests Qt5::Test lib_shournal_common ) # run tests post build: # add_custom_command( TARGET runTests # COMMENT "Run tests" # POST_BUILD # WORKING_DIRECTORY ${CMAKE_BINARY_DIR} # COMMAND runTests # ) set(CMAKE_AUTOMOC ON) set(CMAKE_INCLUDE_CURRENT_DIR ON) ================================================ FILE: test/autotest.h ================================================ #pragma once #include #include #include #include #include #include #include "compat.h" #include "qoutstream.h" #include "subprocess.h" #include "qoptargparse/qoptargparse.h" #include "app.h" #include "helper_for_test.h" #include "settings.h" #include "logger.h" class ShournalTestGlobals { public: subprocess::Args_t integrationShellArgs; std::string integrationSetupCommand; }; namespace AutoTest { inline ShournalTestGlobals& globals(){ static ShournalTestGlobals globals; return globals; } typedef QList TestList; inline TestList& testList() { static TestList list; return list; } inline bool findObject(QObject* object) { TestList& list = testList(); if (list.contains(object)) { return true; } foreach (QObject* test, list) { if (test->objectName() == object->objectName()) { return true; } } return false; } inline void addTest(QObject* object) { TestList& list = testList(); if (!findObject(object)) { list.append(object); } } inline int run(int argc, char *argv[]) { if(! shournal_common_init()){ QIErr() << qtr("Fatal error: failed to initialize custom Qt conversion functions"); exit(1); } logger::setup("shournal-test"); logger::setVerbosityLevel(QtMsgType::QtWarningMsg); // ignore first arg (command to this app) --argc; ++argv; QOptArgParse parser; QOptArg argVerbosity("", "verbosity", qtr("How much shall be printed to stderr. Note that " "for 'dbg' shournal must not be a 'Release'-build, " "dbg-messages are lost in Release-mode.")); argVerbosity.setAllowedOptions({"dbg", #if QT_VERSION >= QT_VERSION_CHECK(5, 5, 0) "info", #endif "warning", "critical"}); parser.addArg(&argVerbosity); QOptArg argIntegrationTest("", "integration", "Run integration tests, instead of normal tests", false); parser.addArg(&argIntegrationTest); QOptArg argShell("", "shell", "The shell used for the intgeration tests, including" " arguments, separated by whitespace"); argShell.addRequiredArg(&argIntegrationTest); parser.addArg(&argShell); parser.parse(argc, argv); if(argVerbosity.wasParsed()){ QByteArray verbosity = argVerbosity.getOptions(1).first().toLocal8Bit(); logger::setVerbosityLevel(verbosity.constData()); } if(argIntegrationTest.wasParsed()){ os::setenv("_SHOURNAL_IN_INTEGRATION_TEST_MODE", "true"); app::setupNameAndVersion("shournal-integration-test"); if(! app::inIntegrationTestMode()){ QIErr() << "Failed to enable integration test mode."; exit(1); } if(! argShell.wasParsed()){ QIErr() << "missing argument" << argShell.name(); exit(1); } const auto shellArgs = argShell.getValue().split(" ", Qt::SkipEmptyParts); if(shellArgs.first().startsWith("bash")){ globals().integrationSetupCommand = "export HISTFILE=/dev/null"; } else if(shellArgs.first().startsWith("zsh")){ globals().integrationSetupCommand = "unset HISTFILE\n" // search for 42 in integration_test_shell.cpp for the rationale "[ $_shournal_run_backend='shournal-run-fanotify' ] && " "[ -z \"${_shournal_is_running+x}\" ] && exit 42\n"; } else { QIErr() << "currently only bash and zsh are supported."; exit(1); } for(const QString& s : shellArgs){ globals().integrationShellArgs.push_back(s.toStdString()); } } else { QCoreApplication::setApplicationName(QString(app::SHOURNAL) + "-test"); QCoreApplication::setApplicationVersion( app::version().toString()); } QStandardPaths::setTestModeEnabled(true); // delete remaining paths from last test (if any) testhelper::deletePaths(); int ret = 0; foreach (QObject* test, testList()) { if(argIntegrationTest.wasParsed()){ if(! test->objectName().startsWith("IntegrationTest")){ continue; } } else{ if(test->objectName().startsWith("IntegrationTest")){ continue; } } ret += QTest::qExec(test, {}); } if(ret != 0){ QErr() << "\n**** AT LEAST ONE TEST FAILED! ****\n\n"; } return ret; } } template class Test { public: QSharedPointer child; Test(const QString& name) : child(new T) { child->setObjectName(name); AutoTest::addTest(child.data()); } }; #define DECLARE_TEST(className) static Test t(#className); #define TEST_MAIN \ int main(int argc, char *argv[]) \ { \ return AutoTest::run(argc, argv); \ } ================================================ FILE: test/helper_for_test.cpp ================================================ #include #include #include #include #include "helper_for_test.h" #include "util.h" #include "app.h" #include "exccommon.h" #include "qfilethrow.h" namespace { const QList& locations(){ static const QList locs = { QStandardPaths::ConfigLocation, QStandardPaths::DataLocation, QStandardPaths::CacheLocation}; return locs; } } // namespace /// Set application-name in a unique way and enable test mode in QStandardPaths, /// so application stuff is saved somewhere else. In the end, remove the /// respective directories (cleanupPaths). void testhelper::setupPaths() { for(const auto& l : locations()){ const QString path = QStandardPaths::writableLocation(l); QDir d(path); if( ! d.mkpath(path)){ throw QExcIo(QString("Failed to create %1").arg(path)); } } } void testhelper::deletePaths() { if(! QStandardPaths::isTestModeEnabled()){ throw QExcProgramming(QString(__func__) + " called while test mode disabled"); } for(const auto& l : locations()){ const QString path = QStandardPaths::writableLocation(l); QDir d(path); d.removeRecursively(); } } void testhelper::deleteDatabaseDir() { if(! QStandardPaths::isTestModeEnabled()){ throw QExcProgramming(QString(__func__) + " called while test mode disabled"); } const QString path = QStandardPaths::writableLocation(QStandardPaths::DataLocation); QDir d(path); d.removeRecursively(); } std::shared_ptr testhelper::mkAutoDelTmpDir() { auto pDir = std::make_shared(); if (! pDir->isValid()) { throw QExcIo("Failed to mk temp dir"); } pDir->setAutoRemove(true); return pDir; } void testhelper::writeStringToFile(const QString &filepath, const QString &str) { QFileThrow f(filepath); f.open(QFile::WriteOnly | QFile::Text); QTextStream stream(&f); stream << str; } /// Write repeated string pattern of len to the file at path void testhelper::writeStuffToFile(const QString &fpath, int len){ const QByteArray stuff("ABCDEFGHIJKLMNOPQRSTUVWXYZ"); QFileThrow f(fpath); f.open(QFile::WriteOnly | QFile::Text); for(int i=0; i < len / stuff.size(); i++){ f.write(stuff); } int rest = len % stuff.size(); if(rest){ auto stuffrest = QByteArray::fromRawData(stuff, rest); f.write(stuffrest); } } QString testhelper::readStringFromFile(const QString &fpath) { QFileThrow f(fpath); f.open(QFile::ReadOnly | QFile::Text); QTextStream stream(&f); return stream.readAll(); } bool testhelper::copyRecursively(const QString &srcFilePath, const QString &tgtFilePath) { QFileInfo srcFileInfo(srcFilePath); if (srcFileInfo.isDir()) { QDir targetDir(tgtFilePath); targetDir.cdUp(); if (!targetDir.mkdir(QFileInfo(tgtFilePath).fileName())) return false; QDir sourceDir(srcFilePath); QStringList fileNames = sourceDir.entryList(QDir::Files | QDir::Dirs | QDir::NoDotAndDotDot | QDir::Hidden | QDir::System); foreach (const QString &fileName, fileNames) { const QString newSrcFilePath = srcFilePath + QLatin1Char('/') + fileName; const QString newTgtFilePath = tgtFilePath + QLatin1Char('/') + fileName; if (!copyRecursively(newSrcFilePath, newTgtFilePath)) return false; } } else { if (!QFile::copy(srcFilePath, tgtFilePath)) return false; } return true; } ================================================ FILE: test/helper_for_test.h ================================================ #pragma once #include #include namespace testhelper { void setupPaths(); void deletePaths(); void deleteDatabaseDir(); std::shared_ptr mkAutoDelTmpDir(); void writeStringToFile(const QString& filepath, const QString& str); void writeStuffToFile(const QString &fpath, int len); QString readStringFromFile(const QString& fpath); bool copyRecursively(const QString &srcFilePath, const QString &tgtFilePath); } ================================================ FILE: test/integration_test_shell.cpp ================================================ #include #include "autotest.h" #include "qoutstream.h" #include "util.h" #include "osutil.h" #include "helper_for_test.h" #include "database/db_connection.h" #include "database/db_controller.h" #include "database/file_query_helper.h" #include "database/query_columns.h" #include "qsimplecfg/cfg.h" #include "settings.h" #include "database/storedfiles.h" #include "safe_file_update.h" using subprocess::Subprocess; using db_controller::QueryColumns; namespace { void writeLine(int fd, const std::string& line){ os::write(fd, line + "\n"); } os::Pipes_t prepareHighFdNumberPipe(){ auto pipe_ = os::pipe(0, false); // CLOEXEC irrelevant, dup2 below... int highFd = osutil::findHighestFreeFd(); os::dup2(pipe_[0], highFd); os::close(pipe_[0]); pipe_[0] = highFd; highFd = osutil::findHighestFreeFd(highFd - 1); os::dup2(pipe_[1], highFd); os::close(pipe_[1]); pipe_[1] = highFd; os::setenv("_SHOURNAL_INTEGRATION_TEST_PIPE_FD", QByteArray::number(pipe_[1])); return pipe_; } /// @return write-end of the pipe passed to the shell-process int callWithRedirectedStdin(Subprocess& proc){ int oldStdIn = os::dup(STDIN_FILENO); // CLOEXEC irrelevant, dup2 below... auto pipe_ = os::pipe(0); os::dup2(pipe_[0], STDIN_FILENO); os::close(pipe_[0]); proc.call(AutoTest::globals().integrationShellArgs); // restore stdin os::dup2(oldStdIn, STDIN_FILENO); os::close(oldStdIn); return pipe_[1]; } } // anonymous namespace class IntegrationTestShell : public QObject { Q_OBJECT private: void writeReadSettingsToCfg(const QString& readIncludeDir, qsimplecfg::Cfg& cfg){ auto sectRead = cfg[Settings::SECT_READ_NAME]; sectRead->getValue(Settings::SECT_READ_KEY_ENABLE, true, true); sectRead->getValue(Settings::SECT_READ_KEY_INCLUDE_PATHS, readIncludeDir, true); } void writeScriptSettingToCfg(const QString& includePath, const QStringList& fileExtensions, qsimplecfg::Cfg& cfg){ auto sectRead = cfg[Settings::SECT_SCRIPTS_NAME]; sectRead->getValue(Settings::SECT_SCRIPTS_ENABLE, true, true); sectRead->getValue(Settings::SECT_SCRIPTS_INCLUDE_PATHS, includePath, true); sectRead->getValues(Settings::SECT_SCRIPTS_INCLUDE_FILE_EXTENSIONS, fileExtensions, true, "\n"); } /// @param cmd: the command to be executed /// @param setupCommand: command executed before SHOURNAL_ENABLE void executeCmdInbservedShell(const std::string& cmd, const std::string& setupCommand){ auto pipe_ = prepareHighFdNumberPipe(); Subprocess proc; // pass pipe write end -> wait for async shournal grand-child process proc.setForwardFdsOnExec({pipe_[1]}); int writeFd = callWithRedirectedStdin(proc); os::close(pipe_[1]); if(! setupCommand.empty()){ writeLine(writeFd, setupCommand); } writeLine(writeFd, "SHOURNAL_ENABLE"); writeLine(writeFd, "SHOURNAL_SET_VERBOSITY " + std::string(logger::msgTypeToStr(logger::getVerbosityLevel()))); writeLine(writeFd, cmd); writeLine(writeFd, "SHOURNAL_DISABLE"); writeLine(writeFd, "exit 123"); os::close(writeFd); auto proc_ret = proc.waitFinish(); if(proc_ret == 42){ QIErr() << QString::fromUtf8("As of zsh 5.7.1 the fanotify shell integration backend " "*must* be enabled during zsh-startup (e.g .zshrc) for the " "integration-tests to succeed. Otherwise the first zsh-process " "*may* consume stdin so no commands remain after «exec zsh» " "which is called to preload libshournal-shellwatch.so. " "See also my email 'Unexpected stdin-behavior' from 2021-10-21 " "on the zsh mailing list zsh-workers@zsh.org. Note that zsh " "does not follow posix shell behaviour here: " "«When the shell is using " "standard input and it invokes a command that also uses standard " "input, the shell shall ensure that the standard input file " "pointer points directly after the command it has read when " "the command begins execution»."); throw QExcIllegalArgument("Bad integration-test environment (see above)"); } QCOMPARE(proc_ret, 123); char c; // wait for shournal grand-child process to finish (close it's write end) os::read(pipe_[0], &c, 1); os::close(pipe_[0]); } /// @overload void executeCmdInbservedShell(const QString& cmd, const std::string& setupCommand){ executeCmdInbservedShell(cmd.toStdString(), setupCommand); } void cmdWrittenFileCheck(const std::string& cmd, const std::string& fpath, const std::string& setupCommand){ executeCmdInbservedShell(cmd, setupCommand); auto query = file_query_helper::buildFileQuerySmart( QString::fromStdString(fpath), false); auto cmdIter = db_controller::queryForCmd(query); auto dbCleanup = finally([] { db_connection::close(); }); QVERIFY(cmdIter->next()); QCOMPARE(cmdIter->value().text, QString::fromStdString(cmd)); QVERIFY(QFile(QString::fromStdString(fpath)).remove()); } private slots: void initTestCase(){ logger::setup(__FILE__); } /// Called for each test. void init(){ testhelper::deletePaths(); // Load settings and delete the config-file. That way, // The version of the cfg-file is also set appropriately. Settings::instance().load(); QFile(Settings::instance().cfgFilepath()).remove(); } void cleanup(){ } void testWrite() { auto pTmpDir = testhelper::mkAutoDelTmpDir(); auto tmpDirPath = pTmpDir->path().toStdString(); QVERIFY(tmpDirPath != "/"); // otherwise this test must be changed auto tmpDirNoLeadingSlash(tmpDirPath); tmpDirNoLeadingSlash.erase(tmpDirNoLeadingSlash.begin()); auto r510 = pTmpDir->path() + "/510"; testhelper::writeStuffToFile(r510, 510); auto r4096 = pTmpDir->path() + "/4096"; testhelper::writeStuffToFile(r4096, 4096); auto r21567 = pTmpDir->path() + "/21567"; testhelper::writeStuffToFile(r21567, 21567); auto r101978 = pTmpDir->path() + "/101978"; testhelper::writeStuffToFile(r101978, 101978); std::string filepath = tmpDirPath + "/f1"; std::vector cmds { "echo '%' > " + filepath, // percent unveiled a printf format bug in shournal 0.7 "(echo foo2 ) > " + filepath, "(echo foo3 > " + filepath + ")", "/bin/echo foo4 > " + filepath, "sh -c 'echo foo5 > " + filepath + "'", "echo foo6 > " + filepath + " & wait", "(echo foo7 & wait ) > " + filepath, "(echo foo8 > " + filepath + ") & wait", "/bin/echo foo9 > " + filepath + " & wait", "sh -c 'echo foo10 > " + filepath + " & wait'", // malformed filepath with multiple slash // "echo foo11 > //" + filepath, // special case root dir "cd /; echo foo11 > //" + filepath, // relative paths must also work: "cd " + tmpDirPath + "; echo hi > f1", "cd " + tmpDirPath + "; echo hi > ./f1", "cd " + tmpDirPath + "; echo hi > ../" + splitAbsPath(tmpDirPath).second + "/f1", // special case root dir "cd /; echo hi > " + tmpDirNoLeadingSlash + "/f1", // test also if partial hashing works for bigger files "cat " + r510.toStdString() + " > " + filepath, "cat " + r4096.toStdString() + " > " + filepath, "cat " + r21567.toStdString() + " > " + filepath, "cat " + r101978.toStdString() + " > " + filepath, }; const auto setupCmd = AutoTest::globals().integrationSetupCommand; for(const auto& cmd : cmds){ testhelper::deleteDatabaseDir(); cmdWrittenFileCheck(cmd, filepath, setupCmd); } } void testRead(){ const auto setupCmd = AutoTest::globals().integrationSetupCommand; auto pTmpDir = testhelper::mkAutoDelTmpDir(); // for read events only include our tempdir qsimplecfg::Cfg cfg; writeReadSettingsToCfg(pTmpDir->path(), cfg); auto & sets = Settings::instance(); SafeFileUpdate cfgUpd8(sets.cfgFilepath()); cfgUpd8.write([&cfg, &cfgUpd8]{ cfg.store(cfgUpd8.file()); }); const QString fname = "foo1"; const QString fullPath = pTmpDir->path() + '/' + fname; QFileThrow(fullPath).open(QFile::WriteOnly); QString cmdTxt = "cat " + fullPath; executeCmdInbservedShell(cmdTxt, setupCmd); SqlQuery query; const auto & cols = QueryColumns::instance(); query.addWithAnd(cols.rFile_path, pTmpDir->path()); query.addWithAnd(cols.rFile_name, fname); auto cmdIter = db_controller::queryForCmd(query); auto dbCleanup = finally([] { db_connection::close(); }); QVERIFY(cmdIter->next()); auto cmdInfo = cmdIter->value(); QCOMPARE(cmdInfo.text, cmdTxt); QCOMPARE(cmdInfo.fileReadInfos.size(), 1); auto fReadInfo = cmdInfo.fileReadInfos.first(); QCOMPARE(fReadInfo.name, fname); QCOMPARE(fReadInfo.path, pTmpDir->path()); QCOMPARE(fReadInfo.isStoredToDisk, false); QVERIFY(! cmdIter->next()); cmdIter.reset(); // Test exec (collected as read files). Copy system echo-binary to out tmppath auto echoPath = QStandardPaths::findExecutable("echo").toLocal8Bit(); QVERIFY(! echoPath.isEmpty()); auto echoInTmp = pathJoinFilename(pTmpDir->path().toLocal8Bit(), QByteArray("echo")); os::sendfile(echoInTmp, echoPath, os::stat(echoPath).st_size); os::chmod(echoInTmp, 0755); // Check if called binaries are tracked when directly called or indirectly, via // env. for(QString cmdTxt : QStringList{echoInTmp + " exec_trace_test", "env " + echoInTmp + " exec_trace_test"}){ executeCmdInbservedShell(cmdTxt, setupCmd); query.clear(); query.addWithAnd(cols.cmd_txt, cmdTxt); // Have to explicitly free cmdIter at loop-end to avoid database locks. // On first loop, the transaction of cmdIter may still be active, so // the database would be locked, when we execute the next command in // the shell (!). auto cmdIter = db_controller::queryForCmd(query); QVERIFY(cmdIter->next()); cmdInfo = cmdIter->value(); QCOMPARE(cmdInfo.text, cmdTxt); QCOMPARE(cmdInfo.fileReadInfos.size(), 1); fReadInfo = cmdInfo.fileReadInfos.first(); QCOMPARE(fReadInfo.name, QString("echo")); QCOMPARE(fReadInfo.path, pTmpDir->path()); } } void testReadScript(){ const auto setupCmd = AutoTest::globals().integrationSetupCommand; auto pTmpDir = testhelper::mkAutoDelTmpDir(); // for read events only include our tempdir qsimplecfg::Cfg cfg; writeScriptSettingToCfg(pTmpDir->path(), {"sh"}, cfg); writeReadSettingsToCfg(pTmpDir->path(), cfg); auto & sets = Settings::instance(); SafeFileUpdate cfgUpd8(sets.cfgFilepath()); cfgUpd8.write([&cfg, &cfgUpd8]{ cfg.store(cfgUpd8.file()); }); const QString fname = "foo1.sh"; const QString fullPath = pTmpDir->path() + '/' + fname; const QString content("abcdefg"); testhelper::writeStringToFile(fullPath, content); const QString cmdTxt = "cat " + fullPath; executeCmdInbservedShell(cmdTxt, setupCmd); SqlQuery query; const auto & cols = QueryColumns::instance(); query.addWithAnd(cols.rFile_path, pTmpDir->path()); query.addWithAnd(cols.rFile_name, fname); auto cmdIter = db_controller::queryForCmd(query); auto dbCleanup = finally([] { db_connection::close(); }); QVERIFY(cmdIter->next()); auto cmdInfo = cmdIter->value(); QCOMPARE(cmdInfo.text, cmdTxt); QCOMPARE(cmdInfo.fileReadInfos.size(), 1); const auto& fReadInfo = cmdInfo.fileReadInfos.first(); QCOMPARE(fReadInfo.name, fname); QCOMPARE(fReadInfo.path, pTmpDir->path()); QCOMPARE(fReadInfo.isStoredToDisk, true); StoredFiles storedFiles; const QString pathToFileInDb = StoredFiles::getReadFilesDir() + "/" + QString::number(fReadInfo.idInDb); QFileThrow fInDb(pathToFileInDb); QVERIFY(fInDb.exists()); fInDb.open(QFile::ReadOnly); QCOMPARE(content, testhelper::readStringFromFile(fullPath)); QVERIFY(! cmdIter->next()); } }; DECLARE_TEST(IntegrationTestShell) #include "integration_test_shell.moc" ================================================ FILE: test/main.cpp ================================================ #include "autotest.h" TEST_MAIN ================================================ FILE: test/sqlite_sample_db_v2_2/readFiles/3 ================================================ s1 ================================================ FILE: test/sqlite_sample_db_v2_2/readFiles/4 ================================================ s2 ================================================ FILE: test/test_cfg.cpp ================================================ #include #include #include #include #include #include "cfg.h" #include "autotest.h" using qsimplecfg::Cfg; class CfgTest : public QObject { Q_OBJECT const QString CONFIG_TXT = R"SOMERANDOMTEXT( # Initial # comment [section1] # section1 # comment section1_key1=val1 section1_key2 = val2 [section2] # section2 # comment section2_key1 = '''first second third''' section2_key2=''' one two three ''' [section3] section3_key1 = ''' ''' section3_key2 = ''' foo ''' )SOMERANDOMTEXT"; void verifyStdCfg(Cfg& cfg){ QVERIFY(cfg.m_parsedNameSectionHash.find("section1") != cfg.m_parsedNameSectionHash.end()); auto sect1 = cfg["section1"]; sect1->setComments("section1\ncomment"); QCOMPARE(sect1->getValue("section1_key1"), QString("val1") ); QCOMPARE(sect1->getValue("section1_key2"), QString("val2") ); QVERIFY(cfg.m_parsedNameSectionHash.find("section2") != cfg.m_parsedNameSectionHash.end()); auto sect2 = cfg["section2"]; QCOMPARE(sect2->getValues("section2_key1", {},false, "\n"), (QStringList{"first", "second", "third"}) ); QCOMPARE(sect2->getValues("section2_key2", {},false, "\n"), (QStringList{"one", "two", "three"}) ); QVERIFY(cfg.m_parsedNameSectionHash.find("section3") != cfg.m_parsedNameSectionHash.end()); auto sect3 = cfg["section3"]; QCOMPARE(sect3->getValues("section3_key1", {}, false, "\n"), QStringList()); QCOMPARE(sect3->getValues("section3_key2", {}, false, "\n"), (QStringList{"foo"}) ); QVERIFY(cfg.generateNonReadSectionKeyPairs().isEmpty()); } std::unique_ptr writeToTmpConfigFile(const QString& txt){ auto file = std::unique_ptr(new QTemporaryFile); if(! file->open()){ return file; } QTextStream stream(file.get()); stream << txt; file->close(); return file; } private slots: void initTestCase(){ logger::setup(__FILE__); } void tgeneral() { auto file = writeToTmpConfigFile(CONFIG_TXT); QVERIFY(! file->fileName().isEmpty()); Cfg cfg; // parse, verify, store and verify again cfg.parse(file->fileName()); verifyStdCfg(cfg); cfg.store(file->fileName()); cfg.parse(file->fileName()); verifyStdCfg(cfg); } void tRenameSection(){ auto file = writeToTmpConfigFile(CONFIG_TXT); QVERIFY(! file->fileName().isEmpty()); Cfg cfg; cfg.parse(file->fileName()); QVERIFY(cfg.renameParsedSection("section1", "section1_renamed")); auto sect1 = cfg["section1_renamed"]; QCOMPARE(sect1->getValue("section1_key1"), QString("val1") ); QCOMPARE(sect1->getValue("section1_key2"), QString("val2") ); cfg.store(file->fileName()); // now that it's renamed, skip the renaming and check if values still correct cfg.parse(file->fileName()); sect1 = cfg["section1_renamed"]; QCOMPARE(sect1->getValue("section1_key1"), QString("val1") ); QCOMPARE(sect1->getValue("section1_key2"), QString("val2") ); } void tRenameKey(){ auto file = writeToTmpConfigFile(CONFIG_TXT); QVERIFY(! file->fileName().isEmpty()); Cfg cfg; cfg.parse(file->fileName()); auto sect = cfg.getParsedSectionIfExist("section1"); QVERIFY(sect != nullptr); QVERIFY(sect->renameParsedKey("section1_key1", "section1_key1_renamed")); // assign it again -> only sections accessed via opertor[] are stored // to disk later... sect = cfg["section1"]; QCOMPARE(sect->getValue("section1_key1_renamed"), QString("val1") ); QCOMPARE(sect->getValue("section1_key2"), QString("val2") ); cfg.store(file->fileName()); cfg.parse(file->fileName()); sect = cfg["section1"]; QCOMPARE(sect->getValue("section1_key1_renamed"), QString("val1") ); QCOMPARE(sect->getValue("section1_key2"), QString("val2") ); // now that it's renamed, skip the renaming and check if values still correct } }; DECLARE_TEST(CfgTest) #include "test_cfg.moc" ================================================ FILE: test/test_cxxhash.cpp ================================================ #include #include #include #include "autotest.h" #include "cxxhash.h" class CXXHashTest : public QObject { Q_OBJECT bool test_hash(int fd, const std::string& str, int chunksize, int seekstep, int maxCountOfReads, int bufsize, CXXHash& h, std::string expected=""){ ftruncate(fd, 0); lseek(fd, 0, SEEK_SET); os::write(fd, str); lseek(fd, 0, SEEK_SET); h.resizeBuf(bufsize); struct partial_xxhash_result res = h.digestFile(fd, chunksize, seekstep, maxCountOfReads); if(expected.empty()){ // delete all underscores (jumped over during partial hash) expected = str; expected.erase(std::remove(expected.begin(), expected.end(), '_'), expected.end()); } return res.hash == XXH64(expected.c_str(), expected.size(), 0 ); } private slots: void initTestCase(){ logger::setup(__FILE__); } void testDigestFile() { QTemporaryFile tmpFile; tmpFile.open(); int fd = tmpFile.handle(); CXXHash h; const int ignoreMaxReads = std::numeric_limits::max(); // result should be independent from buffer size for(int bufSize=1; bufSize < 128; bufSize++){ QVERIFY(test_hash(fd, "aa__bb__cc__dd", 2, 4, ignoreMaxReads, bufSize, h)); // change maxCountOfReads QVERIFY(test_hash(fd, "aa__bb__cc__dd", 2, 4, 2, bufSize, h, "aabb")); // effectively digest everything QVERIFY(test_hash(fd, "aa__bb__cc__dd", 2, 2, ignoreMaxReads, bufSize, h, "aa__bb__cc__dd")); // corner case: seekstep == chunksize + 1 QVERIFY(test_hash(fd, "aa__bb__cc__dd", 2, 3, ignoreMaxReads, bufSize, h, "aa_b__c_dd")); // uneven character length should also work QVERIFY(test_hash(fd, "aa__bb__cc__d", 2, 4, ignoreMaxReads, bufSize, h)); // larger chunks QVERIFY(test_hash(fd, "aaaa__bbbb__cccc__dddd", 4, 6, ignoreMaxReads, bufSize, h)); } } }; DECLARE_TEST(CXXHashTest) #include "test_cxxhash.moc" ================================================ FILE: test/test_db_controller.cpp ================================================ #include #include #include #include #include "autotest.h" #include "compat.h" #include "osutil.h" #include "helper_for_test.h" #include "util.h" #include "database/fileinfos.h" #include "fileevents.h" #include "database/db_controller.h" #include "database/db_connection.h" #include "database/query_columns.h" #include "database/db_conversions.h" #include "database/storedfiles.h" #include "cleanupresource.h" #include "settings.h" #include "qfilethrow.h" #include "stdiocpp.h" using db_controller::QueryColumns; using db_controller::queryForCmd; template typename ContainerT::value_type* __fInfoById(ContainerT& infos, qint64 id_){ for(auto& f : infos){ if(f.idInDb == id_) return &f; } return nullptr; } /// Stored read files may be moved /// from cache dir to shournal's db, /// to simplify comparison here, store /// the content in 'bytes'. class FileReadEventForTest { public: FileReadEventForTest(const QByteArray& bytes) : m_bytes(bytes) { m_file.open(); m_file.write(bytes); m_file.seek(0); } FileEvent e{}; const QTemporaryFile& file(){ return m_file; } const QByteArray& bytes(){ return m_bytes; } private: QTemporaryFile m_file; QByteArray m_bytes; }; typedef std::unique_ptr FileReadEventForTest_ptr; CommandInfo generateCmdInfo(){ static int id_ = 1; CommandInfo cmd; cmd.text = QByteArray::number(id_); cmd.hashMeta.chunkSize = 2048; cmd.hashMeta.maxCountOfReads = 20; cmd.hostname = "myhost"; cmd.username = "myuser"; cmd.returnVal = 42; cmd.startTime = Qt::datetimeFromDate(QDate(2019,1, id_ % 28)); cmd.endTime = Qt::datetimeFromDate(QDate(2019,1, id_ % 28)); cmd.workingDirectory = "/home/user"; id_++; return cmd; } void push_back_writeEvent(FileEvents& fEvents, const FileEvent& e){ struct stat st{}; st.st_mtime = e.mtime(); st.st_size = e.size(); st.st_mode = mode_t(e.mode()); fEvents.write(e.flags(), e.path(), st, e.hash()); } void push_back_readEvent(FileEvents& fEvents, const FileReadEventForTest_ptr& e){ struct stat st = os::fstat(e->file().handle()); fEvents.write(e->e.flags(), e->e.path(), st, e->e.hash(), e->file().handle()); } // sort them by filesize void sortFileWriteInfos(FileWriteInfos & fInos){ std::sort(fInos.begin(), fInos.end(), [](const FileWriteInfo& f1, const FileWriteInfo& f2){ return f1.size < f2.size; }); } // sort them by filesize void sortFileReadInfos(FileReadInfos & fInos){ std::sort(fInos.begin(), fInos.end(), [](const FileReadInfo& f1, const FileReadInfo& f2){ return f1.size < f2.size; }); } int countStoredFiles(){ return QDir(StoredFiles::getReadFilesDir()).entryList(QDir::Filter::NoDotDot | QDir::Files).size(); } int deleteCommandInDb(qint64 id) { SqlQuery q; q.addWithAnd(QueryColumns::instance().cmd_id, id, E_CompareOperator::EQ); return db_controller::deleteCommand(q); } void db_addFileEventsWrapper(const CommandInfo &cmd, FileEvents &fileEvents){ fseek(fileEvents.file(), 0, SEEK_SET); db_controller::addFileEvents(cmd, fileEvents); } class DbCtrlTest : public QObject { Q_OBJECT FileWriteInfo fileWriteEventToWriteInfo(const FileEvent& e){ FileWriteInfo i; assert(! e.m_close_event.hash_is_null); i.hash = e.hash(); auto splittedPah = splitAbsPath(QString(e.path())); i.path = splittedPah.first; i.name = splittedPah.second; i.size = e.size(); i.mtime = db_conversions::fromMtime(e.mtime()).toDateTime(); return i; } FileReadInfo fileReadEventToReadInfo(const FileReadEventForTest_ptr& e){ FileReadInfo i; i.mode = mode_t(e->e.mode()); auto splittedPah = splitAbsPath(QString(e->e.path())); i.path = splittedPah.first; i.name = splittedPah.second; i.size = e->e.size(); i.mtime = db_conversions::fromMtime(e->e.mtime()).toDateTime(); i.hash = e->e.hash(); return i; } FileEvent generateFileWriteEvent(){ static auto hash_ = std::numeric_limits::max(); static int id_ = 1; FileEvent e{}; e.m_close_event.flags = O_WRONLY; e.m_close_event.mtime = Qt::datetimeFromDate(QDate(2019,1, id_ % 28)).toTime_t(); e.m_close_event.size = id_; e.m_close_event.mode = 0; e.m_close_event.hash = hash_; e.m_close_event.hash_is_null = false; e.m_close_event.bytes = 0; std::string fullpath = "/tmp/" + std::to_string(id_) + ".txt"; e.setPath(fullpath.c_str()); hash_--; id_++; return e; } FileReadEventForTest_ptr generateFileReadEvent(){ static auto hash_ = std::numeric_limits::max(); static int id_ = 1; QByteArray fileContent(QByteArray::number(id_), id_); auto e = FileReadEventForTest_ptr(new FileReadEventForTest(fileContent)); auto st = os::fstat(e->file().handle()); e->e.m_close_event.flags = O_RDONLY; e->e.m_close_event.mtime = st.st_mtime; e->e.m_close_event.size = st.st_size; e->e.m_close_event.mode = st.st_mode; e->e.m_close_event.hash = hash_; e->e.m_close_event.hash_is_null = false; e->e.m_close_event.bytes = st.st_size; std::string fullpath = "/tmp/" + std::to_string(id_) + ".txt"; e->e.setPath(fullpath.c_str()); --hash_; ++id_; return e; } private slots: void initTestCase(){ logger::setup(__FILE__); } void init(){ testhelper::setupPaths(); } void cleanup(){ testhelper::deletePaths(); } void tWriteOnly() { FILE* tmpFile = stdiocpp::tmpfile(); auto closeTmpFile = finally([&tmpFile] { fclose(tmpFile); }); FileEvents fileEvents; fileEvents.setFile(tmpFile); auto fInfo1 = generateFileWriteEvent(); push_back_writeEvent(fileEvents, fInfo1); auto fInfo2 = generateFileWriteEvent(); push_back_writeEvent(fileEvents, fInfo2); CommandInfo cmd1 = generateCmdInfo(); cmd1.idInDb = db_controller::addCommand(cmd1); auto closeDb = finally([] { db_connection::close(); }); db_addFileEventsWrapper(cmd1, fileEvents); QueryColumns & queryCols = QueryColumns::instance(); SqlQuery q1; q1.addWithAnd(queryCols.wFile_size, int(fInfo1.size()) ); auto cmd1Back = queryForCmd(q1); QVERIFY(cmd1Back->next()); cmd1.fileWriteInfos = { fileWriteEventToWriteInfo(fInfo1), fileWriteEventToWriteInfo(fInfo2) }; sortFileWriteInfos(cmd1Back->value().fileWriteInfos); QCOMPARE(cmd1Back->value(), cmd1); q1.clear(); q1.addWithAnd(queryCols.wFile_hash, qBytesFromVar(fInfo1.hash().value()) ); cmd1Back.reset(); cmd1Back = queryForCmd(q1); QVERIFY(cmd1Back->next()); sortFileWriteInfos(cmd1Back->value().fileWriteInfos); QCOMPARE(cmd1Back->value(), cmd1); // TODO: test with a hash of null } void tRead(){ FILE* tmpFile = stdiocpp::tmpfile(); auto closeTmpFile = finally([&tmpFile] { fclose(tmpFile); }); FileEvents fileEvents; fileEvents.setFile(tmpFile); auto readEvent1 = generateFileReadEvent(); push_back_readEvent(fileEvents, readEvent1); auto readEvent2 = generateFileReadEvent(); push_back_readEvent(fileEvents, readEvent2); CommandInfo cmd1 = generateCmdInfo(); cmd1.idInDb = db_controller::addCommand(cmd1); auto closeDb = finally([] { db_connection::close(); }); fflush(tmpFile); db_addFileEventsWrapper(cmd1, fileEvents); cmd1.fileReadInfos = {fileReadEventToReadInfo(readEvent1), fileReadEventToReadInfo(readEvent2)}; QueryColumns & queryCols = QueryColumns::instance(); SqlQuery q1; q1.addWithAnd(queryCols.rFile_size, int(readEvent1->e.size()) ); auto cmd1Back = queryForCmd(q1); QVERIFY(cmd1Back->next()); sortFileReadInfos(cmd1Back->value().fileReadInfos); QCOMPARE(cmd1Back->value(), cmd1); q1.clear(); q1.addWithAnd(queryCols.rFile_size, int(readEvent2->e.size()) ); cmd1Back.reset(); cmd1Back = queryForCmd(q1); QVERIFY(cmd1Back->next()); sortFileReadInfos(cmd1Back->value().fileReadInfos); QCOMPARE(cmd1Back->value(), cmd1); // TODO: also check bytes of the file?! } void tDuplicates(){ FILE* tmpFile = stdiocpp::tmpfile(); auto closeTmpFile = finally([&tmpFile] { fclose(tmpFile); }); FileEvents fileEvents; fileEvents.setFile(tmpFile); auto wInfo1 = generateFileWriteEvent(); push_back_writeEvent(fileEvents, wInfo1); push_back_writeEvent(fileEvents, wInfo1); push_back_writeEvent(fileEvents, wInfo1); auto rInfo1 = generateFileReadEvent(); push_back_readEvent(fileEvents, rInfo1); push_back_readEvent(fileEvents, rInfo1); push_back_readEvent(fileEvents, rInfo1); CommandInfo cmd1 = generateCmdInfo(); cmd1.idInDb = db_controller::addCommand(cmd1); auto closeDb = finally([] { db_connection::close(); }); db_addFileEventsWrapper(cmd1, fileEvents); QueryColumns & queryCols = QueryColumns::instance(); SqlQuery q1; q1.addWithAnd(queryCols.cmd_id, int(cmd1.idInDb) ); auto cmd1Back = queryForCmd(q1); QVERIFY(cmd1Back->next()); cmd1.fileWriteInfos = { fileWriteEventToWriteInfo(wInfo1)}; cmd1.fileReadInfos = { fileReadEventToReadInfo(rInfo1)}; QCOMPARE(cmd1Back->value(), cmd1); } void tDeleteCommand(){ FILE* tmpFile = stdiocpp::tmpfile(); auto closeTmpFile = finally([&tmpFile] { fclose(tmpFile); }); FileEvents fileEvents; fileEvents.setFile(tmpFile); auto readEvent1 = generateFileReadEvent(); push_back_readEvent(fileEvents, readEvent1); auto readEvent2 = generateFileReadEvent(); push_back_readEvent(fileEvents, readEvent2); auto writeEvent1 = generateFileWriteEvent(); push_back_writeEvent(fileEvents, writeEvent1); auto writeEvent2 = generateFileWriteEvent(); push_back_writeEvent(fileEvents, writeEvent2); CommandInfo cmd1 = generateCmdInfo(); cmd1.idInDb = db_controller::addCommand(cmd1); auto closeDb = finally([] { db_connection::close(); }); db_addFileEventsWrapper(cmd1, fileEvents ); cmd1.fileReadInfos = {fileReadEventToReadInfo(readEvent1), fileReadEventToReadInfo(readEvent2)}; cmd1.fileWriteInfos = { fileWriteEventToWriteInfo(writeEvent1), fileWriteEventToWriteInfo(writeEvent2) }; QCOMPARE(deleteCommandInDb(cmd1.idInDb), 1); QueryColumns & queryCols = QueryColumns::instance(); SqlQuery q1; // should return all commands q1.addWithAnd(queryCols.rFile_size, 0, E_CompareOperator::GE ); auto cmd1Back = queryForCmd(q1); QVERIFY(! cmd1Back->next()); auto query = db_connection::mkQuery(); query->exec("select * from writtenFile"); QVERIFY(! query->next()); query->exec("select * from readFile"); QVERIFY(! query->next()); query->exec("select * from readFileCmd"); QVERIFY(! query->next()); QCOMPARE(countStoredFiles(), 0); // a single command seems to work // Check for two commands, where one read file is unique for each command // while the other is common to both. auto cmd2 = generateCmdInfo(); auto readEvent3 = generateFileReadEvent(); cmd2.fileReadInfos = {fileReadEventToReadInfo(readEvent1),fileReadEventToReadInfo(readEvent3)}; cmd1.idInDb = db_controller::addCommand(cmd1); stdiocpp::ftruncate_unlocked(fileEvents.file()); push_back_readEvent(fileEvents, readEvent1); push_back_readEvent(fileEvents, readEvent2); db_addFileEventsWrapper(cmd1, fileEvents ); cmd2.idInDb = db_controller::addCommand(cmd2); stdiocpp::ftruncate_unlocked(fileEvents.file()); push_back_readEvent(fileEvents, readEvent1); push_back_readEvent(fileEvents, readEvent3); db_addFileEventsWrapper(cmd2, fileEvents ); QCOMPARE(deleteCommandInDb(cmd1.idInDb), 1); // readEvent1 is common to both and should remain, readEvent2 should be deleted, // readEvent3 should still be there const char* qReadFileSize = "select * from readFile where size=?"; query->prepare(qReadFileSize); query->addBindValue(qint64(readEvent1->e.size())); query->exec(); QVERIFY(query->next()); query->prepare(qReadFileSize); query->addBindValue(qint64(readEvent2->e.size())); query->exec(); QVERIFY(! query->next()); query->prepare(qReadFileSize); query->addBindValue(qint64(readEvent3->e.size())); query->exec(); QVERIFY(query->next()); QCOMPARE(countStoredFiles(), 2); QCOMPARE(deleteCommandInDb(cmd2.idInDb), 1); query->exec("select * from writtenFile"); QVERIFY(! query->next()); query->exec("select * from readFile"); QVERIFY(! query->next()); query->exec("select * from readFileCmd"); QVERIFY(! query->next()); query->exec("select * from hashmeta"); QVERIFY(! query->next()); query->exec("select * from pathtable"); QVERIFY(! query->next()); QCOMPARE(countStoredFiles(), 0); } void tSchemeUpdates(){ const QString & dbDir = db_connection::getDatabaseDir(); os::rmdir(dbDir.toUtf8()); // Copy a database with sample data to the test-database-dir // and check, whether the data survives the scheme update(s), which // are automatically performed upon the first database-usage. // Until including v2.2 nothing testworthy happened // -> src-path for database defined in cmake. QString testDbPath; if (QDir(SHOURNALTEST_SQLITE_v_2_2).exists()){ testDbPath = SHOURNALTEST_SQLITE_v_2_2; } else { // also consider current directory to allow for easy testing // on another machine. testDbPath = splitAbsPath(SHOURNALTEST_SQLITE_v_2_2).second; if (! QDir(testDbPath).exists()){ QIErr() << QString("dir of testdatabase not found: %1").arg(testDbPath); QVERIFY(false); } } QVERIFY(testhelper::copyRecursively(testDbPath, dbDir)); QueryColumns & queryCols = QueryColumns::instance(); SqlQuery q; q.addWithAnd(queryCols.wFile_id, 1); auto cmd = queryForCmd(q); QCOMPARE(cmd->computeSize(), 1); QVERIFY(cmd->next()); QCOMPARE(cmd->value().fileWriteInfos.size(),2); auto fw = __fInfoById(cmd->value().fileWriteInfos, 1); QVERIFY(fw); QVERIFY(fw->name == "one"); QVERIFY(fw->path == "/home/tycho"); fw = __fInfoById(cmd->value().fileWriteInfos, 2); QVERIFY(fw); QVERIFY(fw->name == "two"); QVERIFY(fw->path == "/home/tycho"); // --------- q.clear(); q.addWithAnd(queryCols.cmd_id, 2); cmd.reset(); cmd = queryForCmd(q); QVERIFY(cmd->next()); QCOMPARE(cmd->value().fileReadInfos.size(),2); auto fr = __fInfoById(cmd->value().fileReadInfos, 1); QVERIFY(fr); QVERIFY(fr->name == "one"); QVERIFY(fr->path == "/home/tycho"); QVERIFY(!fr->isStoredToDisk); fr = __fInfoById(cmd->value().fileReadInfos, 2); QVERIFY(fr); QVERIFY(fr->name == "two"); QVERIFY(fr->path == "/home/tycho"); QVERIFY(!fr->isStoredToDisk); // --------- q.clear(); q.addWithAnd(queryCols.cmd_id, 3); cmd.reset(); cmd = queryForCmd(q); QVERIFY(cmd->next()); QCOMPARE(cmd->value().fileWriteInfos.size(),2); fw = __fInfoById(cmd->value().fileWriteInfos, 3); QVERIFY(fw); QVERIFY(fw->name == "three"); QVERIFY(fw->path == "/tmp"); fw = __fInfoById(cmd->value().fileWriteInfos, 4); QVERIFY(fw); QVERIFY(fw->name == "four"); QVERIFY(fw->path == "/tmp"); // --------- q.clear(); q.addWithAnd(queryCols.cmd_id, 4); cmd.reset(); cmd = queryForCmd(q); QVERIFY(cmd->next()); QCOMPARE(cmd->value().fileReadInfos.size(),2); fr = __fInfoById(cmd->value().fileReadInfos, 3); QVERIFY(fr); QVERIFY(fr->name == "script1.sh"); QVERIFY(fr->path == "/home/tycho/storeme"); QVERIFY(fr->isStoredToDisk); fr = __fInfoById(cmd->value().fileReadInfos, 4); QVERIFY(fr); QVERIFY(fr->name == "script2.sh"); QVERIFY(fr->path == "/home/tycho/storeme"); QVERIFY(fr->isStoredToDisk); } }; DECLARE_TEST(DbCtrlTest) #include "test_db_controller.moc" ================================================ FILE: test/test_fdcommunication.cpp ================================================ // Some older systems like CentOS7 (older glibc-versions) don't provide it yet. // Below workaround is ugly but works. #if __has_include() #include #else enum kcmp_type { KCMP_FILE}; #endif #include #include #include #include #include "autotest.h" #include "fdcommunication.h" #include "os.h" #include "cleanupresource.h" #include "excos.h" using fdcommunication::SocketCommunication; using Message = SocketCommunication::Message; static QPair makeSockets(){ auto sockets = os::socketpair(PF_UNIX, SOCK_STREAM); SocketCommunication sendSock; sendSock.setSockFd(sockets[0]); SocketCommunication receiveSock; receiveSock.setReceiveBufferSize(1024); receiveSock.setSockFd(sockets[1]); return {sendSock, receiveSock}; } class FCommunicationTest : public QObject { Q_OBJECT private: bool fdsAreEqual(int fd1, int fd2) { auto pid = getpid(); auto ret = syscall(SYS_kcmp, pid, pid, KCMP_FILE, fd1, fd2); if(ret == -1)throw os::ExcOs("SYS_kcmp failed: "); return ret == 0; } private slots: void initTestCase(){ logger::setup(__FILE__); } void tNormal() { auto sockets = makeSockets(); auto sendSock = sockets.first; auto receiveSock = sockets.second; auto closeSocks = finally([&sendSock, &receiveSock] { close(sendSock.sockFd()); close(receiveSock.sockFd()); }); Message msg1{1, "echo hi some_text_with_umläüts and greek " "δ ω π Δ σ α β γ Σ λ ε µ DONE" }; sendSock.sendMsg(msg1); Message msg2{2, "abcdefg"}; sendSock.sendMsg(msg2); auto messages = receiveSock.receiveMessages(); QCOMPARE(2, messages.size()); QCOMPARE(msg1, messages[0]); QCOMPARE(msg2, messages[1]); // send both messages aggregated. sendSock.sendMessages({msg1, msg2}); QCOMPARE(2, messages.size()); QCOMPARE(msg1, messages[0]); QCOMPARE(msg2, messages[1]); } void tFd() { auto sockets = makeSockets(); auto sendSock = sockets.first; auto receiveSock = sockets.second; auto closeSocks = finally([&sendSock, &receiveSock] { close(sendSock.sockFd()); close(receiveSock.sockFd()); }); QTemporaryFile tmpFile1; QVERIFY(tmpFile1.open()); Message msg1{1, "foobar", tmpFile1.handle()}; sendSock.sendMsg(msg1); auto messages = receiveSock.receiveMessages(); QCOMPARE(1, messages.size()); QCOMPARE(msg1.msgId, messages[0].msgId); QCOMPARE(msg1.bytes, messages[0].bytes); QVERIFY(messages[0].fd != -1); QVERIFY(fdsAreEqual(msg1.fd, messages[0].fd)); os::close(messages[0].fd); // check two fds in sequence. // In this case they are also received in sequence. receiveSock.setReceiveFdSize(10); QTemporaryFile tmpFile2; QVERIFY(tmpFile2.open()); Message msg2{2, "ok_youä#ü", tmpFile2.handle()}; sendSock.sendMsg(msg1); sendSock.sendMsg(msg2); messages = receiveSock.receiveMessages(); QCOMPARE(1, messages.size()); QCOMPARE(msg1.msgId, messages[0].msgId); QCOMPARE(msg1.bytes, messages[0].bytes); QVERIFY(messages[0].fd != -1); QVERIFY(fdsAreEqual(msg1.fd, messages[0].fd)); os::close(messages[0].fd); messages = receiveSock.receiveMessages(); QCOMPARE(1, messages.size()); QCOMPARE(msg2.msgId, messages[0].msgId); QCOMPARE(msg2.bytes, messages[0].bytes); QVERIFY(messages[0].fd != -1); QVERIFY(fdsAreEqual(msg2.fd, messages[0].fd)); os::close(messages[0].fd); // test two fds at once sendSock.sendMessages({msg1, msg2}); messages = receiveSock.receiveMessages(); QCOMPARE(2, messages.size()); QCOMPARE(msg1.msgId, messages[0].msgId); QCOMPARE(msg1.bytes, messages[0].bytes); QVERIFY(messages[0].fd != -1); QCOMPARE(msg2.msgId, messages[1].msgId); QCOMPARE(msg2.bytes, messages[1].bytes); QVERIFY(messages[1].fd != -1); QVERIFY(fdsAreEqual(msg1.fd, messages[0].fd)); os::close(messages[0].fd); QVERIFY(fdsAreEqual(msg2.fd, messages[1].fd)); os::close(messages[1].fd); // test two fds with regular message (without fd) in between Message msgReg(3, "reg"); sendSock.sendMessages({msg1, msgReg, msg2}); messages = receiveSock.receiveMessages(); QCOMPARE(3, messages.size()); QCOMPARE(msg1.msgId, messages[0].msgId); QCOMPARE(msg1.bytes, messages[0].bytes); QVERIFY(messages[0].fd != -1); QVERIFY(fdsAreEqual(msg1.fd, messages[0].fd)); os::close(messages[0].fd); QCOMPARE(msgReg, messages[1]); QCOMPARE(msg2.msgId, messages[2].msgId); QCOMPARE(msg2.bytes, messages[2].bytes); QVERIFY(messages[2].fd != -1); QVERIFY(fdsAreEqual(msg2.fd, messages[2].fd)); os::close(messages[2].fd); } }; DECLARE_TEST(FCommunicationTest) #include "test_fdcommunication.moc" ================================================ FILE: test/test_fileeventhandler.cpp ================================================ #include #include #include #include #include #include "autotest.h" #include "helper_for_test.h" #include "util.h" #include "os.h" #include "osutil.h" #include "settings.h" #include "stdiocpp.h" #include "fileeventhandler.h" /// Write the content of buf to fd, let /// the FileEventHandler process that file and /// compare the partial hashes. void writeCompareBuf(const std::string & buf, const std::string & hasStr, const int fd){ ftruncate(fd, 0); write(fd, buf.c_str(), buf.size()); lseek(fd, 0, SEEK_SET); FileEventHandler fEventHandler; fEventHandler.handleCloseWrite(fd); lseek(fd, 0, SEEK_SET); uint64_t correctHash = XXH64(hasStr.c_str(), hasStr.size(), 0 ); stdiocpp::fseek(fEventHandler.fileEvents().file(), 0 , SEEK_SET); FileEvent* e = fEventHandler.fileEvents().read(); QVERIFY(e != nullptr); auto path = osutil::findPathOfFd(fd); //QIErr() << "std::string(e.fullPath), path" << QString(e.fullPath) << QString::fromStdString(path); QCOMPARE(std::string(e->path()), path); QVERIFY(! e->hash().isNull()); QCOMPARE(correctHash, e->hash().value()); } class FileEventHandlerTest : public QObject { Q_OBJECT private slots: void initTestCase(){ logger::setup(__FILE__); } void init(){ //testhelper::setupPaths(); } void cleanup(){ // testhelper::cleanupPaths(); } void tWrite() { /// Primarily a test, if hashChunkSize and /// hashMaxCountOfReads are handled correctly // Don't use QTemporaryFile here, we need a regular // file with st1.st_nlink > 0 char tmpFileName[] = "fileevent_test_XXXXXX"; int fd = mkstemp(tmpFileName); QVERIFY(os::fstat(fd).st_nlink > 0); auto rmTmpFile = finally([&tmpFileName] { remove(tmpFileName); }); auto & sets = Settings::instance(); sets.m_wSettings.includePaths->insert("/"); sets.m_hashSettings.hashEnable = true; sets.m_hashSettings.hashMeta = HashMeta(2, 2); std::string buf = "g"; writeCompareBuf(buf, buf, fd); buf = "gh"; writeCompareBuf(buf, buf, fd); buf = "abc"; writeCompareBuf(buf, buf, fd); buf = "abcd"; writeCompareBuf(buf, buf, fd); // only 2 chars (hashChunkSize) each at index 0 and // 10/hashMaxCountOfReads = 5 should be read and used for hash writeCompareBuf("ab___cd___", "abcd", fd); sets.m_hashSettings.hashMeta = HashMeta(3, 2); writeCompareBuf("abc__def___", "abcdef", fd); sets.m_hashSettings.hashMeta = HashMeta(1, 2); writeCompareBuf("a____d_____", "ad", fd); sets.m_hashSettings.hashMeta = HashMeta(1, 3); writeCompareBuf("a__b__c___", "abc", fd); } void tRead(){ // TODO: implement a test... auto & readSettings = Settings::instance().m_scriptSettings; readSettings.enable = true; readSettings.includePaths->insert("/"); // todo: mk unique path readSettings.maxFileSize = 50000; readSettings.onlyWritable = true; readSettings.includeExtensions = {}; readSettings.maxCountOfFiles = 1; readSettings.flushToDiskTotalSize = 10*1000; } }; DECLARE_TEST(FileEventHandlerTest) #include "test_fileeventhandler.moc" ================================================ FILE: test/test_osutil.cpp ================================================ #include #include #include #include "autotest.h" #include "osutil.h" using namespace osutil; class OsutilTest : public QObject { Q_OBJECT private slots: void initTestCase(){ logger::setup(__FILE__); } void testReadWholeFile() { QTemporaryFile f; QVERIFY(f.open()); QByteArray val("123456"); f.write(val); f.seek(0); // different buffer sizes should ot change the result QCOMPARE(readWholeFile(f.handle(), 6), val); f.seek(0); QCOMPARE(readWholeFile(f.handle(), 7), val); f.seek(0); QCOMPARE(readWholeFile(f.handle(), 3), val); f.seek(0); QCOMPARE(readWholeFile(f.handle(), 1), val); f.seek(0); } }; DECLARE_TEST(OsutilTest) #include "test_osutil.moc" ================================================ FILE: test/test_pathtree.cpp ================================================ #include #include #include #include "autotest.h" #include "pathtree.h" #include "util.h" class PathTreeTest : public QObject { Q_OBJECT void checkAllSubPathsExist(PathTree& tree, const StrLight& parentPath, std::unordered_set paths){ for(auto treeIt=tree.subpathIter(parentPath); treeIt != tree.end(); ++treeIt){ auto it = paths.find(*treeIt); QVERIFY2(it != paths.end(), (*treeIt).c_str()); paths.erase(it); } QVERIFY(paths.empty()); } void checkAllExist(PathTree& tree, std::unordered_set paths){ for(const auto & p : tree){ auto it = paths.find(p); QVERIFY2(it != paths.end(), (p).c_str()); paths.erase(it); } auto p = (paths.empty()) ? "" : *paths.begin(); QVERIFY2(paths.empty(), p.c_str()); } void erasePathTreeFromIt(PathTree& tree, PathTree::iterator it){ while(it != tree.end() ){ it = tree.erase(it); } } private slots: void initTestCase(){ logger::setup(__FILE__); } void testContains(){ PathTree tree; QVERIFY(! tree.contains("/")); tree.insert("/"); QVERIFY(tree.contains("/")); tree.clear(); QVERIFY(! tree.contains("/")); tree.insert("/"); tree.insert("/home/user/foo"); QVERIFY(tree.contains("/")); QVERIFY(tree.contains("/home/user/foo")); } void testParent() { PathTree tree; QVERIFY(! tree.isParentPath("/")); QVERIFY(! tree.isParentPath("/home")); tree.insert("/home/user"); QVERIFY(tree.isParentPath("/home")); QVERIFY(tree.isParentPath("/")); QVERIFY(tree.isParentPath("/home/user", true)); QVERIFY(! tree.isParentPath("/home/user")); QVERIFY(! tree.isParentPath("/home/user/foo", true)); QVERIFY(! tree.isParentPath("/home/user/foo", false)); // special case root PathTree tree2; tree2.insert("/"); QVERIFY(! tree2.isParentPath("/", false)); QVERIFY( tree2.isParentPath("/", true)); QVERIFY( ! tree2.isParentPath("/home")); tree2.insert("/home/user"); QVERIFY( tree2.isParentPath("/", false)); QVERIFY( tree2.isParentPath("/", true)); QVERIFY( tree2.isParentPath("/home")); QVERIFY(! tree2.isParentPath("/home/user", false)); QVERIFY( tree2.isParentPath("/home/user", true)); } void testSub() { PathTree tree; QVERIFY(! tree.isSubPath("/")); QVERIFY(! tree.isSubPath("/home")); tree.insert("/home/user1"); tree.insert("/home/user2"); QVERIFY(! tree.isSubPath("/home")); QVERIFY(! tree.isSubPath("/home/user1", false)); QVERIFY( tree.isSubPath("/home/user1", true)); QVERIFY( tree.isSubPath("/home/user1/foo", false)); QVERIFY( tree.isSubPath("/home/user1/foo", true)); QVERIFY(! tree.isSubPath("/home/nouser/foo", true)); // special case root PathTree tree2; tree2.insert("/"); QVERIFY(! tree2.isSubPath("/", false)); QVERIFY( tree2.isSubPath("/", true)); QVERIFY( tree2.isSubPath("/home")); tree2.insert("/home"); QVERIFY(! tree2.isSubPath("/", false)); QVERIFY( tree2.isSubPath("/", true)); QVERIFY( tree2.isSubPath("/home")); QVERIFY( tree2.isSubPath("/home/foo")); auto tree3 = std::make_shared(); tree3->insert("/tmp/shournal-integration-test-AsKCoY"); QVERIFY( tree3->isSubPath("/tmp/shournal-integration-test-AsKCoY/foo1", false)); } void testFindSub(){ PathTree tree; tree.insert("/"); QVERIFY(tree.begin() != tree.end()); QVERIFY(tree.subpathIter("/") == tree.end()); tree.insert("/home"); QVERIFY(tree.subpathIter("/") != tree.end()); QVERIFY2(*tree.subpathIter("/") == "/home", (*tree.subpathIter("/")).c_str()); tree.insert("/home/user"); tree.insert("/var"); checkAllSubPathsExist(tree, "/", {"/home", "/home/user", "/var"}); tree.clear(); QVERIFY(tree.begin() == tree.end()); tree.insert("/home/foo"); tree.insert("/media/data/123"); tree.insert("/media/data/456"); tree.insert("/media/data/789"); checkAllSubPathsExist(tree, "/media", {"/media/data/123", "/media/data/456", "/media/data/789", }); } void testClear(){ PathTree tree; tree.insert("/home/user"); tree.clear(); QVERIFY(! tree.isSubPath("/home/user/foo")); tree.insert("/home/user"); QVERIFY(tree.isSubPath("/home/user/foo")); } void testIter(){ PathTree tree; QVERIFY(tree.begin() == tree.end()); std::unordered_set paths { "/home/user/foodir", "/home/user/another", "/media/cdrom/aha", "/media/ok/123", "/var/log", "/" }; tree.insert(paths.begin(), paths.end()); checkAllExist(tree, paths); } void testErase(){ PathTree tree; const std::unordered_set paths { "/home/user", "/home/user/sub1", "/home/user/sub2/subsub1", "/media/cdrom", "/var", "/" }; tree.insert(paths.begin(), paths.end()); auto it = tree.iter("/home"); QVERIFY(it != tree.end()); erasePathTreeFromIt(tree, it); checkAllExist(tree, {"/media/cdrom", "/var", "/"}); tree.insert(paths.begin(), paths.end()); it = tree.iter("/home/user"); QVERIFY(it != tree.end()); erasePathTreeFromIt(tree, it); checkAllExist(tree, {"/media/cdrom", "/var", "/"}); tree.insert(paths.begin(), paths.end()); it = tree.iter("/home/user/sub1"); QVERIFY(it != tree.end()); erasePathTreeFromIt(tree, it); checkAllExist(tree, { "/home/user", "/home/user/sub2/subsub1", "/media/cdrom", "/var", "/", }); tree.insert(paths.begin(), paths.end()); it = tree.iter("/home/user/sub2/subsub1"); QVERIFY(it != tree.end()); erasePathTreeFromIt(tree, it); checkAllExist(tree, { "/home/user", "/home/user/sub1", "/media/cdrom", "/var", "/", }); tree.insert(paths.begin(), paths.end()); it = tree.iter("/var"); QVERIFY(it != tree.end()); erasePathTreeFromIt(tree, it); checkAllExist(tree, { "/home/user", "/home/user/sub1", "/home/user/sub2/subsub1", "/media/cdrom", "/", }); tree.insert(paths.begin(), paths.end()); it = tree.iter("/"); QVERIFY(it != tree.end()); erasePathTreeFromIt(tree, it); checkAllExist(tree, {}); } }; DECLARE_TEST(PathTreeTest) #include "test_pathtree.moc" ================================================ FILE: test/test_qformattedstream.cpp ================================================ #include "qoutstream.h" #include "qformattedstream.h" #include "autotest.h" class QFormattedtStreamTest : public QObject { Q_OBJECT private slots: void initTestCase(){ logger::setup(__FILE__); } void testIt() { QString str; QFormattedStream s(&str); s.setLineStart("# "); s.setMaxLineWidth(5); s << "aha next\ntext\nFoo\nAVeryLongWord"; s << "ok\n"; s << "na"; QCOMPARE(str, QString("# aha\n# nex\n# t\n# tex\n# t\n# Foo\n# AVe\n# ryL\n# ong\n" "# Wor\n# d \n# ok\n# na ")); } }; DECLARE_TEST(QFormattedtStreamTest) #include "test_qformattedstream.moc" ================================================ FILE: test/test_qoptargparse.cpp ================================================ #include #include #include #include "autotest.h" #include "qoptargparse/qoptargparse.h" #include "qoptargparse/qoptsqlarg.h" #include "qoptargparse/qoptvarlenarg.h" class QOptArgparseTest : public QObject { Q_OBJECT private slots: void initTestCase(){ logger::setup(__FILE__); } void testIt() { QOptArgParse parser; QOptArg arg1("", "one", ""); parser.addArg(&arg1); QOptSqlArg arg2Sql("", "two", "", {E_CompareOperator::EQ} ); parser.addArg(&arg2Sql); QOptSqlArg arg3Sql("", "three", "", {E_CompareOperator::EQ} ); parser.addArg(&arg3Sql); QOptSqlArg arg4Sql("", "four", "", {E_CompareOperator::BETWEEN}, E_CompareOperator::BETWEEN ); parser.addArg(&arg4Sql); QOptVarLenArg arg5VarLen("", "five", ""); parser.addArg(&arg5VarLen); QOptArg arg6("", "six", "", false); arg6.setFinalizeFlag(true); parser.addArg(&arg6); QVector argv = {"--one", "1", "--two", "-eq", "2", "--three", "3", "--four", "-between", "4_1", "4_2", "--five", "3", "5_1", "5_2", "5_3", "--six", "6_1", "6_2", "6_3", "6_4", nullptr}; parser.parse(argv.size() - 1, (char**)argv.data()); QVERIFY(arg1.wasParsed()); QCOMPARE(arg1.getValue(), QString("1")); QVERIFY(arg2Sql.wasParsed()); QCOMPARE(arg2Sql.getValue(), QString("2")); QCOMPARE(arg2Sql.parsedOperator(), E_CompareOperator::EQ); QVERIFY(arg3Sql.wasParsed()); QCOMPARE(arg3Sql.getValue(), QString("3")); QCOMPARE(arg3Sql.parsedOperator(), E_CompareOperator::EQ); QVERIFY(arg4Sql.wasParsed()); QCOMPARE(arg4Sql.getValues(), QStringList({"4_1", "4_2"})); QCOMPARE(arg4Sql.parsedOperator(), E_CompareOperator::BETWEEN); QVERIFY(arg5VarLen.wasParsed()); QCOMPARE(arg5VarLen.getValues(), QStringList({"5_1", "5_2", "5_3"})); QVERIFY(arg6.wasParsed()); QCOMPARE(parser.rest().len, 4); const char* arg6Actual[] = {"6_1", "6_2", "6_3", "6_4"}; for(int i=0; i < parser.rest().len; i++ ){ QVERIFY(strcmp(parser.rest().argv[i], arg6Actual[i]) == 0); } } }; DECLARE_TEST(QOptArgparseTest) #include "test_qoptargparse.moc" ================================================ FILE: test/test_util.cpp ================================================ #include #include #include #include "autotest.h" class UtilTest : public QObject { Q_OBJECT template void splitAbsPathTest(const T & p, const T& expectedPath, const T& expectedFile) { auto pair = splitAbsPath(p); QVERIFY(pair.first == expectedPath); QVERIFY(pair.second == expectedFile); } private slots: void initTestCase(){ logger::setup(__FILE__); } void testSplitAbsPath() { splitAbsPathTest("/", "/", ""); splitAbsPathTest("/", "/", ""); splitAbsPathTest("/home", "/", "home"); splitAbsPathTest("/home", "/", "home"); splitAbsPathTest("/home/user", "/home", "user"); splitAbsPathTest("/home/user", "/home", "user"); splitAbsPathTest("/home/user/foo", "/home/user", "foo"); splitAbsPathTest("/home/user/foo", "/home/user", "foo"); } void testPathJoinFilename(){ QVERIFY(pathJoinFilename(QString("/"), QString("foo")) == "/foo"); QVERIFY(pathJoinFilename(QString("/home/foo"), QString("bar")) == "/home/foo/bar"); } }; DECLARE_TEST(UtilTest) #include "test_util.moc"