Repository: vrmiguel/bustd Branch: master Commit: 7d7dd9d916a9 Files: 36 Total size: 76.3 KB Directory structure: gitextract_7yr9f5ut/ ├── .github/ │ └── workflows/ │ └── build.yml ├── .gitignore ├── LICENSE ├── README.md ├── rust/ │ ├── Cargo.toml │ ├── build.rs │ ├── cc/ │ │ └── helper.c │ └── src/ │ ├── cli.rs │ ├── daemon.rs │ ├── errno.rs │ ├── error.rs │ ├── kill.rs │ ├── linux_version.rs │ ├── main.rs │ ├── memory/ │ │ ├── mem_info.rs │ │ ├── mem_lock.rs │ │ ├── mod.rs │ │ └── pressure.rs │ ├── monitor.rs │ ├── process.rs │ ├── uname.rs │ └── utils.rs ├── tools/ │ ├── .gitignore │ └── mem-eater.c └── zig/ ├── .gitignore ├── LICENSE ├── README.md ├── build.zig └── src/ ├── config.zig ├── daemonize.zig ├── main.zig ├── memory.zig ├── missing_syscalls.zig ├── monitor.zig ├── pressure.zig └── process.zig ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/workflows/build.yml ================================================ on: [push, pull_request] name: build-and-test jobs: armv7-glibc: name: Ubuntu 18.04 (for ARMv7 - glibc) runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: toolchain: stable target: armv7-unknown-linux-gnueabihf override: true - name: Install binutils-arm-none-eabi run: | sudo apt-get update sudo apt-get install binutils-arm-none-eabi - uses: actions-rs/cargo@v1 with: use-cross: true command: build args: --release --target=armv7-unknown-linux-gnueabihf - name: Strip binary run: arm-none-eabi-strip target/armv7-unknown-linux-gnueabihf/release/bustd - name: Upload binary uses: actions/upload-artifact@v2 with: name: 'bustd-linux-armv7-glibc' path: target/armv7-unknown-linux-gnueabihf/release/bustd armv7-musl: name: Ubuntu 20.01 (for ARMv7 - musl) runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: toolchain: stable target: armv7-unknown-linux-musleabihf override: true - name: Install binutils-arm-none-eabi run: | sudo apt-get update sudo apt-get install binutils-arm-none-eabi - name: Run cargo build uses: actions-rs/cargo@v1 with: use-cross: true command: build args: --release --target=armv7-unknown-linux-musleabihf - name: Strip binary run: arm-none-eabi-strip target/armv7-unknown-linux-musleabihf/release/bustd - name: Upload binary uses: actions/upload-artifact@v2 with: name: 'bustd-linux-armv7-musl' path: target/armv7-unknown-linux-musleabihf/release/bustd ubuntu: name: Ubuntu 20.04 runs-on: ubuntu-latest strategy: matrix: rust: - stable steps: - name: Checkout sources uses: actions/checkout@v2 - name: Install toolchain uses: actions-rs/toolchain@v1 with: toolchain: stable target: x86_64-unknown-linux-musl override: true - name: Install dependencies for musl libc run: | sudo apt-get update sudo apt-get install musl-tools - name: Run cargo build uses: actions-rs/cargo@v1 with: command: build args: --release --target x86_64-unknown-linux-musl - name: Run cargo test uses: actions-rs/cargo@v1 with: command: test args: --release --target x86_64-unknown-linux-musl - name: Strip binary run: strip target/x86_64-unknown-linux-musl/release/bustd - name: Upload binary uses: actions/upload-artifact@v2 with: name: 'bustd-x86-64-musl' path: target/x86_64-unknown-linux-musl/release/bustd ubuntu-glibc: name: Ubuntu 18.04 - glibc runs-on: ubuntu-18.04 strategy: matrix: rust: - stable steps: - name: Checkout sources uses: actions/checkout@v2 - name: Install toolchain uses: actions-rs/toolchain@v1 with: toolchain: stable - name: Run cargo build uses: actions-rs/cargo@v1 with: command: build args: --release - name: Run cargo test uses: actions-rs/cargo@v1 with: command: test args: --release - name: Strip binary run: strip target/release/bustd - name: Upload binary uses: actions/upload-artifact@v2 with: name: 'bustd-x86-64-glibc' path: target/release/bustd ================================================ FILE: .gitignore ================================================ target/ ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2021 Vinícius R. Miguel Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ # `bustd`: Available memory or bust! `bustd` is a lightweight process killer daemon for out-of-memory scenarios for Linux! ## Features ### Small memory usage! `bustd` seems to use less memory than some other lean daemons such as `earlyoom`: ```console $ ps -F -C bustd UID PID PPID C SZ RSS PSR STIME TTY TIME CMD vrmiguel 353609 187407 5 151 8 2 01:20 pts/2 00:00:00 target/x86_64-unknown-linux-musl/release/bustd -V -n $ ps -F -C earlyoom UID PID PPID C SZ RSS PSR STIME TTY TIME CMD vrmiguel 350497 9498 0 597 688 6 01:12 pts/1 00:00:00 ./earlyoom/ ``` ¹: RSS stands for resident set size and represents the portion of RAM occupied by a process. ²: Compared when bustd was in [this commit](https://github.com/vrmiguel/bustd/commit/61beb097b3631afb231a76bb9187b802c9818793) and earlyoom in [this one](https://github.com/rfjakob/earlyoom/commit/509df072be79b3be2a1de6581499e360ab0180be). `bustd` compiled with musl libc and earlyoom with glibc through GCC 11.1. Different configurations would likely change these figures. ### Small CPU usage Much like `earlyoom` and `nohang`, `bustd` uses adaptive sleep times during its memory polling. Unlike these two, however, `bustd` does not read from `/proc/meminfo`, instead opting for the `sysinfo` syscall. This approach has its up- and downsides. The amount of free RAM that `sysinfo` reads does not account for cached memory, while `MemAvailable` in `/proc/meminfo` does. The `sysinfo` syscall is one order of magnitude faster, at least according to [this kernel patch](https://sourceware.org/legacy-ml/libc-alpha/2015-08/msg00512.html) (granted, from 2015). As `bustd` can't solely rely on the free RAM readings of `sysinfo`, we check for memory stress through [Pressure Stall Information](https://www.kernel.org/doc/html/v5.8/accounting/psi.html). ### `bustd` will try to lock all pages mapped into its address space Much like `earlyoom`, `bustd` uses [`mlockall`](https://www.ibm.com/docs/en/aix/7.2?topic=m-mlockall-munlockall-subroutine) to avoid being sent to swap, which allows the daemon to remain responsive even when the system memory is under heavy load and susceptible to [thrashing](https://en.wikipedia.org/wiki/Thrashing_(computer_science)). ### Checks for Pressure Stall Information The Linux kernel, since version 4.20 (and built with `CONFIG_PSI=y`), presents canonical new pressure metrics for memory, CPU, and IO. In the words of [Facebook Incubator](https://facebookmicrosites.github.io/psi/docs/overview): ``` PSI stats are like barometers that provide fair warning of impending resource shortages, enabling you to take more proactive, granular, and nuanced steps when resources start becoming scarce. ``` More specifically, `bustd` checks for how long, in microseconds, processes have stalled in the last 10 seconds. By default, `bustd` will kill a process when processes have stalled for 25 microseconds in the last ten seconds. ## Packaging ### Arch Linux Available on the Arch User Repository ### Gentoo Available on the [GURU project](https://gitweb.gentoo.org/repo/proj/guru.git) ### Pop!_OS Available on the [Pop!_OS PPA](https://launchpad.net/~system76/+archive/ubuntu/pop) (outdated) ## Building Requirements: * [Rust toolchain](https://rustup.rs/) * Any C compiler * Linux 4.20+ built with `CONFIG_PSI=y` ```shell git clone https://github.com/vrmiguel/bustd cd bustd && cargo run --release ``` The `-n, --no-daemon` flag is useful for running `bustd` through an init system such as `systemd`. ## Prebuilt binaries Binaries are generated at every commit through [GitHub Actions](https://github.com/vrmiguel/bustd/actions) ## TODO - [x] Allow for customization of the critical scenario (PSI cutoff) - [x] Command-line argument for disabling daemonization (useful for runnning `bustd` as a systemd service) - [x] Command-line argument to enable killing the entire process group, not just the chosen process itself - [x] Allow the user to setup a list of software that `bustd` should never kill - [ ] Notification sending and general notification customization settings ================================================ FILE: rust/Cargo.toml ================================================ [package] name = "bustd" authors = ["Vinícius R. Miguel "] version = "0.1.1" edition = "2018" readme = "README.md" repository = "https://github.com/vrmiguel/bustd" description = "Lightweight process killer daemon for out-of-memory scenarios" categories = ["command-line-utilities", "memory-management"] license = "MIT" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] glob = { version = "0.3.1", optional = true } libc = "0.2.144" cfg-if = "1.0.0" daemonize = "0.5.0" argh = "0.1.10" memchr = "2.5.0" [build-dependencies] cc = "1.0.68" libc = "0.2.97" [dev-dependencies] # Using a somewhat popular crate, `procfs`, to test our own # implementation of proc-fs reads. # Probably not the best decision possible but OK for now procfs = { version = "0.14.2", default-features = false } [features] glob-ignore = ["glob"] [profile.release] lto = true codegen-units = 1 opt-level = 3 strip = true ================================================ FILE: rust/build.rs ================================================ fn main() { cc::Build::new().file("cc/helper.c").compile("helper"); println!("cargo:rerun-if-changed=cc/helper.c"); } ================================================ FILE: rust/cc/helper.c ================================================ #include #ifdef MCL_ONFAULT const int _MCL_ONFAULT = MCL_ONFAULT; #else const int _MCL_ONFAULT = -1; #endif ================================================ FILE: rust/src/cli.rs ================================================ use argh::FromArgs; #[derive(FromArgs)] /// Lightweight process killer daemon for out-of-memory scenarios pub struct CommandLineArgs { /// toggles on verbose output #[argh(switch, short = 'V')] pub verbose: bool, /// when set, the process will not be daemonized #[argh(switch, short = 'n')] pub no_daemon: bool, /// when set, the victim's entire process group will be killed #[argh(switch, short = 'g')] pub kill_pgroup: bool, /// sets the PSI value on which, if surpassed, a process will be killed #[argh(option, short = 'p', long = "psi", default = "25.0")] pub cutoff_psi: f32, // TODO: responsitivity multiplier? #[cfg(feature = "glob-ignore")] /// all processes whose names match any of the supplied vertical bar-separated glob patterns will never be chosen to be killed #[argh( option, short = 'u', long = "unkillables", from_str_fn(parse_unkillables) )] pub ignored: Option>, } #[cfg(feature = "glob-ignore")] fn parse_unkillables(arg: &str) -> Result, String> { let unkillables: Result, _> = arg.split('|').map(glob::Pattern::new).collect(); unkillables.map_err(|err| err.to_string()) } ================================================ FILE: rust/src/daemon.rs ================================================ use std::fs::OpenOptions; use daemonize::Daemonize; use crate::{error::Result, utils}; pub fn daemonize() -> Result<()> { let running_as_sudo = utils::running_as_sudo(); let username = if running_as_sudo { "root".into() } else { utils::get_username().unwrap_or_else(|| "nobody".into()) }; let open_opts = OpenOptions::new() .truncate(false) .create(true) .write(true) .to_owned(); let (stdout_path, stderr_path, pidfile_path) = if running_as_sudo { ( "/var/log/bustd.out", "/var/log/bustd.err", "/var/run/bustd.pid", ) } else { ("/tmp/bustd.out", "/tmp/bustd.err", "/tmp/bustd.pid") }; let stdout = open_opts.open(stdout_path)?; let stderr = open_opts.open(stderr_path)?; let daemonize = Daemonize::new() .user(&*username) .pid_file(pidfile_path) .chown_pid_file(false) .working_directory("/tmp") .stdout(stdout) .stderr(stderr); daemonize.start()?; println!( "[LOG] User {} has started the daemon successfully.", username ); Ok(()) } ================================================ FILE: rust/src/errno.rs ================================================ use cfg_if::cfg_if; use libc::{self, c_int}; cfg_if! { if #[cfg(target_os = "android")] { unsafe fn _errno() -> *mut c_int { libc::__errno() } } else if #[cfg(target_os = "linux")] { unsafe fn _errno() -> *mut c_int { libc::__errno_location() } } } #[allow(clippy::unnecessary_cast)] pub fn errno() -> i32 { unsafe { (*_errno()) as i32 } } ================================================ FILE: rust/src/error.rs ================================================ #![allow(unused)] use std::{any::Any, str::Utf8Error}; #[derive(Debug)] pub enum Error { // Only possible uname error: "buf is invalid" UnameFailed, ProcessNotFound(&'static str), InvalidPidSupplied, ProcessGroupNotFound, InvalidSignal, Io { reason: String, }, Daemonize { error: daemonize::Error, }, #[allow(unused)] Unicode { error: Utf8Error, }, NoPermission, // mlockall-specific errors CouldNotLockMemory, TooMuchMemoryToLock, InvalidFlags, // Should not happen but better safe than sorry UnknownMlockall, UnknownKill, UnknownGetpguid, #[cfg(feature = "glob-ignore")] GlobPattern { error: glob::PatternError, }, // Errors that are likely impossible to happen InvalidLinuxVersion, MalformedStatm, MalformedPressureFile, ParseInt, ParseFloat, SysConfFailed, SysInfoFailed, } pub type Result = std::result::Result; impl From for Error { fn from(err: std::io::Error) -> Self { Self::Io { reason: err.to_string(), } } } impl From for Error { fn from(_: std::num::ParseIntError) -> Self { Self::ParseInt } } impl From for Error { fn from(_: std::num::ParseFloatError) -> Self { Self::ParseFloat } } impl From for Error { fn from(error: daemonize::Error) -> Self { Self::Daemonize { error } } } impl From for Error { fn from(error: std::str::Utf8Error) -> Self { Self::Unicode { error } } } #[cfg(feature = "glob-ignore")] impl From for Error { fn from(error: glob::PatternError) -> Self { Self::GlobPattern { error } } } ================================================ FILE: rust/src/kill.rs ================================================ use std::fs; use std::time::Duration; use std::time::Instant; use libc::kill; use libc::{EINVAL, EPERM, ESRCH, SIGKILL, SIGTERM}; use crate::errno::errno; use crate::error::{Error, Result}; use crate::process::Process; use crate::{cli, utils}; pub fn choose_victim( proc_buf: &mut [u8], buf: &mut [u8], args: &cli::CommandLineArgs, ) -> Result { let now = Instant::now(); // `args` is currently only used when checking for unkillable patterns #[cfg(not(feature = "glob-ignore"))] let _ = args; let mut processes = fs::read_dir("/proc/")? .filter_map(|e| e.ok()) .filter_map(|entry| entry.file_name().to_str()?.trim().parse::().ok()) .filter(|pid| *pid > 1) .filter_map(|pid| Process::from_pid(pid, proc_buf).ok()); let first_process = processes.next(); if first_process.is_none() { // Likely an impossible scenario but we found no process to kill! return Err(Error::ProcessNotFound("choose_victim")); } let mut victim = first_process.unwrap(); // TODO: find another victim if victim.vm_rss_kib() fails here let mut victim_vm_rss_kib = victim.vm_rss_kib(buf)?; for process in processes { if victim.oom_score > process.oom_score { // Our current victim is less innocent than the process being analysed continue; } #[cfg(feature = "glob-ignore")] { if let Some(patterns) = &args.ignored { if matches!(process.is_unkillable(buf, patterns), Ok(true)) { continue; } } } let cur_vm_rss_kib = process.vm_rss_kib(buf)?; if cur_vm_rss_kib == 0 { // Current process is a kernel thread continue; } if process.oom_score == victim.oom_score && cur_vm_rss_kib <= victim_vm_rss_kib { continue; } let cur_oom_score_adj = match process.oom_score_adj(buf) { Ok(oom_score_adj) => oom_score_adj, // TODO: warn that this error happened Err(_) => continue, }; if cur_oom_score_adj == -1000 { // Follow the behaviour of the standard OOM killer: don't kill processes with oom_score_adj equals to -1000 continue; } // eprintln!("[DBG] New victim with PID={}!", process.pid); victim = process; victim_vm_rss_kib = cur_vm_rss_kib; } println!("[LOG] Found victim in {} secs.", now.elapsed().as_secs()); println!( "[LOG] Victim => pid: {}, comm: {}, oom_score: {}", victim.pid, victim.comm(buf).unwrap_or("unknown").trim(), victim.oom_score ); Ok(victim) } pub fn kill_process(pid: i32, signal: i32) -> Result<()> { let res = unsafe { kill(pid, signal) }; if res == -1 { return Err(match errno() { // An invalid signal was specified EINVAL => Error::InvalidSignal, // Calling process doesn't have permission to send signals to any // of the target processes EPERM => Error::NoPermission, // The target process or process group does not exist. ESRCH => Error::ProcessNotFound("kill"), _ => Error::UnknownKill, }); } Ok(()) } pub fn kill_process_group(process: Process) -> Result<()> { let pid = process.pid; let pgid = utils::get_process_group(pid as i32)?; // TODO: kill and wait let _ = kill_process(-pgid, SIGTERM); Ok(()) } /// Tries to kill a process and wait for it to exit /// Will first send the victim a SIGTERM and escalate to SIGKILL if necessary /// Returns Ok(true) if the victim was successfully terminated pub fn kill_and_wait(process: Process) -> Result { let pid = process.pid; let now = Instant::now(); let _ = kill_process(pid as i32, SIGTERM); let half_a_sec = Duration::from_secs_f32(0.5); let mut sigkill_sent = false; for _ in 0..20 { std::thread::sleep(half_a_sec); if !process.is_alive() { println!("[LOG] Process with PID {} has exited.\n", pid); return Ok(true); } if !sigkill_sent { let _ = kill_process(pid as i32, SIGKILL); sigkill_sent = true; println!( "[LOG] Escalated to SIGKILL after {} nanosecs", now.elapsed().as_nanos() ); } } Ok(false) } ================================================ FILE: rust/src/linux_version.rs ================================================ use std::{cmp::Ordering, fmt::Display}; #[derive(Debug, PartialEq, Eq)] pub struct LinuxVersion { pub major: u8, pub minor: u8, } impl LinuxVersion { /// Given a release string (e.g. as given by `uname -r`), attempt /// to extract the major and minor values of the Linux version pub fn from_str(release: &str) -> Option { // The position of the first dot in the 'release' string let dot_idx = release.find('.')?; let (major, minor): (&str, &str) = release.split_at(dot_idx); let major: u8 = major.parse().ok()?; // Eat the leading dot in front of minor let minor = &minor[1..]; let dot_idx = minor.find('.')?; let minor: u8 = minor[0..dot_idx].parse().ok()?; Some(Self { major, minor }) } } impl Display for LinuxVersion { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let (major, minor) = (self.major, self.minor); write!(f, "Linux {major}.{minor}") } } impl PartialOrd for LinuxVersion { fn partial_cmp(&self, other: &Self) -> Option { match self.major.partial_cmp(&other.major) { Some(Ordering::Equal) => {} ord => return ord, } self.minor.partial_cmp(&other.minor) } } #[cfg(test)] mod tests { use crate::linux_version::LinuxVersion; #[test] fn should_be_able_to_parse_linux_versions() { assert_eq!( LinuxVersion::from_str("5.16.18-1-MANJARO").unwrap(), LinuxVersion { major: 5, minor: 16 } ); assert_eq!( LinuxVersion::from_str("3.8.3-Fedora").unwrap(), LinuxVersion { major: 3, minor: 8 } ); } #[test] fn should_be_able_to_compare_linux_versions() { assert!(LinuxVersion::from_str("3.8.3") >= LinuxVersion::from_str("3.6.9")); // We do not require PATCH accuracy assert!( LinuxVersion::from_str("3.8.3").unwrap() == LinuxVersion::from_str("3.8.9").unwrap() ); assert!( LinuxVersion::from_str("5.8.3").unwrap() > LinuxVersion::from_str("3.13.9").unwrap() ); assert!( LinuxVersion::from_str("5.8.3").unwrap() < LinuxVersion::from_str("5.13.9").unwrap() ); assert!( LinuxVersion::from_str("5.8.3").unwrap() > LinuxVersion::from_str("4.20.0").unwrap() ); assert!( LinuxVersion::from_str("4.21.0").unwrap() > LinuxVersion::from_str("4.20.0").unwrap() ); assert!( LinuxVersion::from_str("4.15.0").unwrap() < LinuxVersion::from_str("4.20.0").unwrap() ); } } ================================================ FILE: rust/src/main.rs ================================================ // use uname::Uname; use std::ops::Not; use linux_version::LinuxVersion; use uname::Uname; use crate::{error::Error, memory::lock_memory_pages, monitor::Monitor}; mod cli; mod daemon; mod errno; mod error; mod kill; mod linux_version; mod memory; mod monitor; mod process; mod uname; mod utils; /// The first Linux version in which PSI information became available const LINUX_4_20: LinuxVersion = LinuxVersion { major: 4, minor: 20, }; fn main() -> error::Result<()> { let args: cli::CommandLineArgs = argh::from_env(); let should_daemonize = args.no_daemon.not(); // Show uname info and return the Linux version running { let ensure_msg = "Ensure you're running at least Linux 4.20"; let uname = Uname::new()?; uname.print_info()?; match uname.parse_version() { Ok(version) => { if version < LINUX_4_20 { eprintln!( "{version} does not meet minimum requirements for bustd!\n{ensure_msg}" ); return Err(Error::InvalidLinuxVersion); } } Err(_) => { eprintln!("Failed to parse Linux version!\n{ensure_msg}"); } } if let Ok(version) = uname.parse_version() { if version < LINUX_4_20 { eprintln!("{version} does not meet minimum requirements for bustd!\n{ensure_msg}"); return Err(Error::InvalidLinuxVersion); } } else { eprintln!("Failed to parse Linux version!\n{ensure_msg}"); } }; // In order to correctly use `mlockall`, we'll try our best to avoid heap allocations and // reuse these buffers right here, even though it makes the code less readable. // Buffer specific to process creation let proc_buf = [0_u8; 50]; // Buffer for anything else let buf = [0_u8; 100]; if should_daemonize { // Daemonize current process println!("\nStarting daemonization process!"); daemon::daemonize()?; } // Attempt to lock the memory pages mapped to the daemon // in order to avoid being sent to swap when the system // memory is stressed if let Err(err) = lock_memory_pages() { eprintln!("Failed to lock memory pages: {:?}. Continuing anyway.", err); } else { // Save this on both bustd.out and bustd.err println!("Memory pages locked!"); eprintln!("Memory pages locked!"); } Monitor::new(proc_buf, buf, args)?.poll() } ================================================ FILE: rust/src/memory/mem_info.rs ================================================ use std::{fmt, mem}; use libc::sysinfo; use crate::{ error::{Error, Result}, utils::bytes_to_megabytes, }; #[derive(Debug, Default)] pub struct MemoryInfo { pub total_ram_mb: u64, pub total_swap_mb: u64, pub available_ram_mb: u64, pub available_swap_mb: u64, pub available_ram_percent: u8, pub available_swap_percent: u8, } /// Simple wrapper over libc's sysinfo fn sys_info() -> Result { // Safety: the all-zero byte pattern is a valid sysinfo struct let mut sys_info: sysinfo = unsafe { mem::zeroed() }; // Safety: sysinfo() is safe and must not fail when passed a valid reference let ret_val = unsafe { libc::sysinfo(&mut sys_info) }; if ret_val != 0 { // The only error that sysinfo() can have happens when // it is supplied an invalid struct sysinfo pointer // // This error should really not happen during this function return Err(Error::SysInfoFailed); } Ok(sys_info) } impl MemoryInfo { pub fn new() -> Result { let sysinfo { mem_unit, freeram, totalram, totalswap, freeswap, .. } = sys_info()?; let ratio = |x, y| ((x as f32 / y as f32) * 100.0) as u8; let available_ram_mb = bytes_to_megabytes(freeram, mem_unit); let total_ram_mb = bytes_to_megabytes(totalram, mem_unit); let total_swap_mb = bytes_to_megabytes(totalswap, mem_unit); let available_swap_mb = bytes_to_megabytes(freeswap, mem_unit); let available_ram_percent = ratio(available_ram_mb, total_ram_mb); let available_swap_percent = if total_swap_mb != 0 { ratio(available_swap_mb, total_swap_mb) } else { 0 }; Ok(MemoryInfo { total_ram_mb, available_ram_mb, total_swap_mb, available_swap_mb, available_ram_percent, available_swap_percent, }) } } impl fmt::Display for MemoryInfo { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { writeln!(f, "Total RAM: {} MB", self.total_ram_mb)?; writeln!( f, "Available RAM: {} MB ({}%)", self.available_ram_mb, self.available_ram_percent )?; writeln!(f, "Total swap: {} MB", self.total_swap_mb)?; writeln!( f, "Available swap: {} MB ({} %)", self.available_swap_mb, self.available_swap_percent ) } } ================================================ FILE: rust/src/memory/mem_lock.rs ================================================ use libc::{c_int, mlockall}; use libc::{EAGAIN, EINVAL, ENOMEM, EPERM}; use libc::{MCL_CURRENT, MCL_FUTURE}; use crate::errno::errno; use crate::error::{Error, Result}; extern "C" { pub static _MCL_ONFAULT: libc::c_int; } pub fn _mlockall_wrapper(flags: c_int) -> Result<()> { let err = unsafe { mlockall(flags) }; if err == 0 { return Ok(()); } // If err != 0, errno was set to describe the error that mlockall had Err(match errno() { // Some or all of the memory identified by the operation could not be locked when the call was made. EAGAIN => Error::CouldNotLockMemory, // The flags argument is zero, or includes unimplemented flags. EINVAL => Error::InvalidFlags, // Locking all of the pages currently mapped into the address space of the process // would exceed an implementation-defined limit on the amount of memory // that the process may lock. ENOMEM => Error::TooMuchMemoryToLock, // The calling process does not have appropriate privileges to perform the requested operation EPERM => Error::NoPermission, // Should not happen _ => Error::UnknownMlockall, }) } pub fn lock_memory_pages() -> Result<()> { // TODO: check for _MCL_ONFAULT == -1 #[allow(non_snake_case)] let MCL_ONFAULT: c_int = unsafe { _MCL_ONFAULT }; match _mlockall_wrapper(MCL_CURRENT | MCL_FUTURE | MCL_ONFAULT) { Err(err) => { eprintln!("First try at mlockall failed: {:?}", err); } Ok(_) => return Ok(()), } _mlockall_wrapper(MCL_CURRENT | MCL_FUTURE) } ================================================ FILE: rust/src/memory/mod.rs ================================================ mod mem_info; mod mem_lock; pub mod pressure; pub use mem_info::MemoryInfo; pub use mem_lock::lock_memory_pages; ================================================ FILE: rust/src/memory/pressure.rs ================================================ use std::fs::File; use std::io::Read; use crate::error::{Error, Result}; use crate::utils::str_from_bytes; macro_rules! malformed { () => { Error::MalformedPressureFile }; } /// Returns the avg10 value in the `some` row of `/proc/pressure/memory`, which /// indicates the absolute stall time (in us) in which at least some tasks were stalled. /// /// The data we're reading looks like: /// ```some avg10=0.00 avg60=0.00 avg300=0.00 total=0``` /// pub fn pressure_some_avg10(buf: &mut [u8]) -> Result { let mut file = File::open("/proc/pressure/memory")?; buf.fill(0); // `buf` won't be large enough to fit all of `/proc/pressure/memory` // but will be large enough to hold at least the first line, which has the data we want file.read(buf)?; let contents = str_from_bytes(buf)?; let line = contents.lines().next().ok_or(malformed!())?; let mut words = line.split_ascii_whitespace(); if let Some(indicator) = words.next() { // This has to be the case but checking to be sure if indicator == "some" { let entry = words.next().ok_or(malformed!())?; // The entry is of the form `avg10=0.00` // We'll break this string in two in order to parse the value on the right-hand side let equals_pos = entry.find('=').ok_or(malformed!())?; let avg10 = entry.get(equals_pos + 1..).ok_or(malformed!())?; let avg10: f32 = avg10.trim().parse()?; return Ok(avg10); } } Err(malformed!()) } ================================================ FILE: rust/src/monitor.rs ================================================ use std::time::Duration; use crate::cli::CommandLineArgs; use crate::error::Result; use crate::kill; use crate::memory; use crate::memory::MemoryInfo; use crate::process::Process; enum MemoryStatus { NearTerminal(f32), Okay, } pub struct Monitor { memory_info: MemoryInfo, proc_buf: [u8; 50], buf: [u8; 100], status: MemoryStatus, args: CommandLineArgs, } impl Monitor { /// Determines how much oomf should sleep /// This function is essentially a copy of how earlyoom calculates its sleep time /// /// Credits: https://github.com/rfjakob/earlyoom/blob/dea92ae67997fcb1a0664489c13d49d09d472d40/main.c#L365 /// MIT Licensed pub fn sleep_time_ms(&self) -> Duration { // Maximum expected memory fill rate as seen // with `stress -m 4 --vm-bytes 4G` const RAM_FILL_RATE: i64 = 6000; // Maximum expected swap fill rate as seen // with membomb on zRAM const SWAP_FILL_RATE: i64 = 800; // Maximum and minimum time to sleep, in ms. const MIN_SLEEP: i64 = 100; const MAX_SLEEP: i64 = 1000; // TODO: make these percentages configurable by args./config. file const RAM_TERMINAL_PERCENT: f64 = 10.; const SWAP_TERMINAL_PERCENT: f64 = 10.; let ram_headroom_kib = (self.memory_info.available_ram_percent as f64 - RAM_TERMINAL_PERCENT) * (self.memory_info.total_ram_mb as f64 * 10.0); let swap_headroom_kib = (self.memory_info.available_swap_percent as f64 - SWAP_TERMINAL_PERCENT) * (self.memory_info.total_swap_mb as f64 * 10.0); let ram_headroom_kib = i64::max(ram_headroom_kib as i64, 0); let swap_headroom_kib = i64::max(swap_headroom_kib as i64, 0); let time_to_sleep = ram_headroom_kib / RAM_FILL_RATE + swap_headroom_kib / SWAP_FILL_RATE; let time_to_sleep = i64::min(time_to_sleep, MAX_SLEEP); let time_to_sleep = i64::max(time_to_sleep, MIN_SLEEP); Duration::from_millis(time_to_sleep as u64) } pub fn new(proc_buf: [u8; 50], mut buf: [u8; 100], args: CommandLineArgs) -> Result { let memory_info = MemoryInfo::new()?; let status = if memory_info.available_ram_percent <= 15 { MemoryStatus::NearTerminal(memory::pressure::pressure_some_avg10(&mut buf)?) } else { MemoryStatus::Okay }; Ok(Self { memory_info, proc_buf, buf, status, args, }) } fn memory_is_low(&self) -> bool { let terminal_psi = self.args.cutoff_psi; matches!(self.status, MemoryStatus::NearTerminal(psi) if psi >= terminal_psi) } fn get_victim(&mut self) -> Result { kill::choose_victim(&mut self.proc_buf, &mut self.buf, &self.args) } fn update_memory_stats(&mut self) -> Result<()> { self.memory_info = memory::MemoryInfo::new()?; self.status = if self.memory_info.available_ram_percent <= 15 { let psi = memory::pressure::pressure_some_avg10(&mut self.buf)?; MemoryStatus::NearTerminal(psi) } else { MemoryStatus::Okay }; Ok(()) } fn free_up_memory(&mut self) -> Result<()> { let victim = self.get_victim()?; // TODO: is this necessary? // // Check for memory stats again to see if the // low-memory situation was solved while // we were searching for our victim self.update_memory_stats()?; if self.memory_is_low() { if self.args.kill_pgroup { kill::kill_process_group(victim)?; } else { kill::kill_and_wait(victim)?; } } Ok(()) } // Use the never type here whenever it reaches stable #[allow(unreachable_code)] pub fn poll(&mut self) -> Result<()> { loop { // Update our memory readings self.update_memory_stats()?; if self.memory_is_low() { self.free_up_memory()?; } // Calculating the adaptive sleep time let sleep_time = self.sleep_time_ms(); if self.args.verbose { eprintln!("[adaptive-sleep] {}ms", sleep_time.as_millis()); } std::thread::sleep(sleep_time); } Ok(()) } } ================================================ FILE: rust/src/process.rs ================================================ use std::io::Read; use std::io::Write; use libc::getpgid; use crate::{ error::{Error, Result}, utils::{self, str_from_bytes}, }; #[derive(Debug, Default)] pub struct Process { pub pid: u32, pub oom_score: i16, } impl Process { pub fn from_pid(pid: u32, buf: &mut [u8]) -> Result { let oom_score = Self::oom_score_from_pid(pid, buf).or(Err(Error::ProcessNotFound("from_pid")))?; Ok(Self { pid, oom_score }) } #[allow(dead_code)] /// Returns the current process represented as a Process struct /// Unused in the actual code but very often used when debugging pub fn this(buf: &mut [u8]) -> Result { let pid = unsafe { libc::getpid() } as u32; Self::from_pid(pid, buf) } /// Return true if the process is alive /// Could still return true if the process has exited but hasn't yet been reaped. /// TODO: would it be better to check for /proc// in here? pub fn is_alive_from_pid(pid: u32) -> bool { // Safety: `getpgid` is memory safe let group_id = unsafe { getpgid(pid as i32) }; group_id > 0 } pub fn is_alive(&self) -> bool { Self::is_alive_from_pid(self.pid) } pub fn comm<'a>(&self, buf: &'a mut [u8]) -> Result<&'a str> { write!(&mut *buf, "/proc/{}/comm\0", self.pid)?; { let mut file = utils::file_from_buffer(buf)?; buf.fill(0); let _ = file.read(buf)?; } str_from_bytes(buf) } pub fn oom_score_from_pid(pid: u32, buf: &mut [u8]) -> Result { write!(&mut *buf, "/proc/{}/oom_score\0", pid)?; let contents = { let mut file = utils::file_from_buffer(buf)?; buf.fill(0); let _ = file.read(buf)?; str_from_bytes(buf)?.trim() }; Ok(contents.parse()?) } /// Reads VmRSS from /proc//statm /// In order to match the VmRSS value in /proc//status, we'll /// multiply the number of pages in `statm` by the page size of our system and then convert /// that value to KiB pub fn vm_rss_kib(&self, buf: &mut [u8]) -> Result { write!(&mut *buf, "/proc/{}/statm\0", self.pid)?; let mut columns = { let mut file = utils::file_from_buffer(buf)?; buf.fill(0); let _ = file.read(buf)?; str_from_bytes(buf)?.split_ascii_whitespace() }; let vm_rss: i64 = columns.nth(1).ok_or(Error::MalformedStatm)?.parse()?; let page_size = utils::page_size()?; // Converting VM RSS to KiB let vm_rss_kib = vm_rss * page_size / 1024; Ok(vm_rss_kib) } #[cfg(feature = "glob-ignore")] /// Checks if the process' name matches any of the given glob patterns pub fn is_unkillable(&self, buf: &mut [u8], patterns: &[glob::Pattern]) -> Result { let comm = self.comm(buf)?.trim(); for pattern in patterns { if pattern.matches(comm) { println!( "Skipping \"{}\" since it matches an unkillable pattern", comm ); return Ok(true); } } Ok(false) } pub fn oom_score_adj(&self, buf: &mut [u8]) -> Result { write!(&mut *buf, "/proc/{}/oom_score_adj\0", self.pid)?; let contents = { let mut file = utils::file_from_buffer(buf)?; buf.fill(0); let _ = file.read(buf)?; str_from_bytes(buf)?.trim() }; Ok(contents.parse()?) } } #[cfg(test)] mod tests { // We'll use the Process struct from procfs // in order to test our own Process struct. // // The reason we don't use `procfs` directly is // because our implementation is considerably leaner. use procfs; // Returns the Process representing the // process of the caller test fn this() -> ([u8; 100], crate::process::Process) { let mut buf = [0_u8; 100]; (buf, crate::process::Process::this(&mut buf).unwrap()) } #[test] fn comm() { let (mut buf, this) = this(); let comm = this.comm(&mut buf).unwrap(); // We'll now represent the current process using // the external `procfs` crate as well let _this = procfs::process::Process::myself().unwrap(); let _stat = _this.stat().unwrap(); let _comm = _stat.comm; assert_eq!(comm.trim(), _comm) } #[test] fn oom_score() { let (_, this) = this(); let _this = procfs::process::Process::myself().unwrap(); assert_eq!(this.oom_score, _this.oom_score().unwrap() as i16); } #[test] fn pid() { let (_, this) = this(); let _this = procfs::process::Process::myself().unwrap(); assert_eq!(this.pid as i32, _this.pid); } } ================================================ FILE: rust/src/uname.rs ================================================ use std::ffi::CStr; use std::mem; use crate::error::{Error, Result}; use crate::linux_version::LinuxVersion; use libc::{uname, utsname}; pub struct Uname { uts_struct: utsname, } impl Uname { pub fn new() -> Result { // Safety: libc::utsname is a bunch of char arrays and therefore // can be safely zeroed. let mut uts_struct: utsname = unsafe { mem::zeroed() }; let ret_val = unsafe { uname(&mut uts_struct) }; // uname returns a negative number upon failure if ret_val < 0 { return Err(Error::UnameFailed); } Ok(Self { uts_struct }) } pub fn print_info(&self) -> Result<()> { // Safety: dereference of these raw pointers are safe since we know they're not NULL, since // the buffers in struct utsname are all correctly allocated in the stack at this moment let sysname = unsafe { CStr::from_ptr(self.uts_struct.sysname.as_ptr()) }; let hostname = unsafe { CStr::from_ptr(self.uts_struct.nodename.as_ptr()) }; let release = unsafe { CStr::from_ptr(self.uts_struct.release.as_ptr()) }; let arch = unsafe { CStr::from_ptr(self.uts_struct.machine.as_ptr()) }; let sysname = sysname.to_str()?; let hostname = hostname.to_str()?; let release = release.to_str()?; let arch = arch.to_str()?; println!("OS: {}", sysname); println!("Hostname: {}", hostname); println!("Version: {}", release); println!("Architecture: {}", arch); Ok(()) } pub fn parse_version(&self) -> Result { let release = unsafe { CStr::from_ptr(self.uts_struct.release.as_ptr()) }; let release = release.to_str()?; LinuxVersion::from_str(release).ok_or(Error::InvalidLinuxVersion) } } ================================================ FILE: rust/src/utils.rs ================================================ use std::ffi::OsStr; use std::fs::File; use std::os::unix::prelude::OsStrExt; use std::path::Path; use std::{ffi::CStr, mem, ptr, str}; use libc::_SC_PAGESIZE; use libc::{getpgid, sysconf, EINVAL, EPERM, ESRCH}; use libc::{getpwuid_r, passwd}; use memchr::memchr; use crate::errno::errno; use crate::error::{Error, Result}; /// Gets the effective user ID of the calling process fn effective_user_id() -> u32 { // Safety: the POSIX Programmer's Manual states that // geteuid will always be successful. unsafe { libc::geteuid() } } /// Gets the process group of the process /// with the given PID. pub fn get_process_group(pid: i32) -> Result { let pgid = unsafe { getpgid(pid) }; if pgid == -1 { return Err(match errno() { EPERM => Error::NoPermission, ESRCH => Error::ProcessGroupNotFound, EINVAL => Error::InvalidPidSupplied, _ => Error::UnknownGetpguid, }); } Ok(pgid) } /// Checks if the program is running with sudo permissions. pub fn running_as_sudo() -> bool { effective_user_id() == 0 } /// Get the size of the system's memory page in bytes. pub fn page_size() -> Result { // _SC_PAGESIZE is defined in POSIX.1 // Safety: no memory unsafety can arise from `sysconf` let page_size = unsafe { sysconf(_SC_PAGESIZE) }; if page_size == -1 { return Err(Error::SysConfFailed); } #[allow(clippy::useless_conversion)] // The type of page_size differs between architectures // so we use .into() to convert to i64 if necessary Ok(page_size.into()) } /// Attempt to get the user's username from the system's password bank pub fn get_username() -> Option { let mut buf = [0; 2048]; let mut result = ptr::null_mut(); let mut passwd: passwd = unsafe { mem::zeroed() }; let uid = effective_user_id(); let getpwuid_r_code = unsafe { getpwuid_r(uid, &mut passwd, buf.as_mut_ptr(), buf.len(), &mut result) }; if getpwuid_r_code == 0 && !result.is_null() { // If getpwuid_r succeeded, let's get the username from it let username = unsafe { CStr::from_ptr(passwd.pw_name) }; let username = String::from_utf8_lossy(username.to_bytes()); return Some(username.into()); } None } fn bytes_until_first_nil(buf: &[u8]) -> &[u8] { let first_nul_idx = memchr(0, buf).unwrap_or(buf.len()); &buf[0..first_nul_idx] } /// Construct a string slice ranging from the first position to the position of the first nul byte pub fn str_from_bytes(buf: &[u8]) -> Result<&str> { let bytes = bytes_until_first_nil(buf); Ok(str::from_utf8(bytes)?) } fn path_from_bytes(buf: &[u8]) -> &Path { let bytes = bytes_until_first_nil(buf); Path::new(OsStr::from_bytes(bytes)) } /// Given a slice of bytes, try to interpret it as a file path and open the corresponding file. pub fn file_from_buffer(buf: &[u8]) -> Result { let path = path_from_bytes(buf); let file = File::open(path)?; Ok(file) } pub fn bytes_to_megabytes(bytes: impl Into, mem_unit: impl Into) -> u64 { const B_TO_MB: u64 = 1000 * 1000; bytes.into() / B_TO_MB * mem_unit.into() } #[cfg(test)] mod tests { use super::str_from_bytes; #[test] fn should_construct_string_slice_from_bytes() { assert_eq!(str_from_bytes(b"ABC\0").unwrap(), "ABC"); assert_eq!(str_from_bytes(b"ABC\0abc").unwrap(), "ABC"); } } ================================================ FILE: tools/.gitignore ================================================ mem-eater ================================================ FILE: tools/mem-eater.c ================================================ // `bustd`'s memory eater #include #include #include #include #include #include struct free_mem_s { unsigned available_mem_mib; unsigned available_swap_mib; }; typedef struct free_mem_s free_mem_t; void display(free_mem_t * mem, float psi) { printf("\rFree RAM: %d MiB. Free swap: %d MiB. PSI: %0.2f", mem->available_mem_mib, mem->available_swap_mib, psi); fflush(stdout); } float memory_pressure_some_avg_10(void) { FILE * memory_pressure = fopen("/proc/pressure/memory", "r"); if(!memory_pressure) { perror("/proc/pressure/memory. Exiting.\n"); fclose(memory_pressure); _exit(1); } float psi; if (EOF == fscanf(memory_pressure, "some avg10=%f", &psi)) { perror("Failed to read memory pressure values. Exiting.\n"); fclose(memory_pressure); _exit(1); } fclose(memory_pressure); return psi; } free_mem_t poll_free_mem(void) { FILE * meminfo = fopen("/proc/meminfo", "r"); if(!meminfo) { fprintf(stderr, "/proc/meminfo not found. Exiting.\n"); fclose(meminfo); _exit(1); } char line[256]; bool avail_mem_read = false; bool avail_swap_read = false; free_mem_t free_mem; while((!avail_mem_read || !avail_swap_read) && fgets(line, sizeof(line), meminfo)) { int val; if(sscanf(line, "MemAvailable: %d kB", &val) == 1) { avail_mem_read = true; free_mem.available_mem_mib = (unsigned) val / 1024; } if(sscanf(line, "SwapFree: %d kB", &val) == 1) { avail_swap_read = true; free_mem.available_swap_mib = (unsigned) val / 1024; } } for (int i = 0; i < 100; i++) { putchar(' '); } if (!avail_swap_read || !avail_mem_read) { fprintf(stderr, "failed to read available system memory or swap amounts. Exiting.\n"); fclose(meminfo); _exit(1); } fclose(meminfo); return free_mem; } int main(void) { time_t start, now; float time_left = 4.0; time(&start); while(time_left > 0.0) { time(&now); time_left = 4.0 - difftime(now, start); printf("\rmem-eater will start consuming system memory in: %.2f secs. Press Ctrl+C if you don't want that to happen.", time_left); fflush(stdout); usleep(20); } while(1) { free_mem_t free_mem = poll_free_mem(); float psi = memory_pressure_some_avg_10(); display(&free_mem, psi); void *m = malloc(1024*1024); memset(m,0,1024*1024); } return 0; } ================================================ FILE: zig/.gitignore ================================================ zig-cache/ zig-out/ ================================================ FILE: zig/LICENSE ================================================ MIT License Copyright (c) 2022 Vinícius Miguel Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: zig/README.md ================================================ :warning: This will be marged with the original [bustd](https://github.com/vrmiguel/bustd) repository. # `buztd`: Available memory or bust! `buztd` is a lightweight process killer daemon for out-of-memory scenarios for Linux! This particular project is a Zig version of the [original `bustd` project](https://github.com/vrmiguel/bustd). ## Features ### Extremely thin memory usage The Zig version of `bustd` makes no heap allocations and relies solely on a single 128-byte buffer in the stack for all its allocation needs. ### Small CPU usage Much like `earlyoom` and `nohang`, `buztd` uses adaptive sleep times during its memory polling. Unlike these two, however, `buztd` does not read from `/proc/meminfo`, instead opting for the `sysinfo` syscall. This approach has its up- and downsides. The amount of free RAM that `sysinfo` reads does not account for cached memory, while `MemAvailable` in `/proc/meminfo` does. However, the `sysinfo` syscall is one order of magnitude faster than parsing `/proc/meminfo`, at least according to [this kernel patch](https://sourceware.org/legacy-ml/libc-alpha/2015-08/msg00512.html) (granted, from 2015). As `buztd` can't solely rely on the free RAM readings of `sysinfo`, we check for memory stress through [Pressure Stall Information](https://www.kernel.org/doc/html/v5.8/accounting/psi.html). More on that below. ### `buztd` will try to lock all pages mapped into its address space Much like `earlyoom`, `buztd` uses [`mlockall`](https://www.ibm.com/docs/en/aix/7.2?topic=m-mlockall-munlockall-subroutine) to avoid being sent to swap, which allows the daemon to remain responsive even when the system memory is under heavy load and susceptible to [thrashing](https://en.wikipedia.org/wiki/Thrashing_(computer_science)). ### Checks for Pressure Stall Information The Linux kernel, since version 4.20 (and built with `CONFIG_PSI=y`), presents canonical new pressure metrics for memory, CPU, and IO. In the words of [Facebook Incubator](https://facebookmicrosites.github.io/psi/docs/overview): ``` PSI stats are like barometers that provide fair warning of impending resource shortages, enabling you to take more proactive, granular, and nuanced steps when resources start becoming scarce. ``` More specifically, `buztd` checks for how long, in microseconds, processes have stalled in the last 10 seconds. By default, `buztd` will kill a process when processes have stalled for 25 microseconds in the last ten seconds. Example: ``` some avg10=0.00 avg60=0.00 avg300=0.00 total=11220657 full avg10=0.00 avg60=0.00 avg300=0.00 total=10947429 ``` These ratios are percentages of recent trends over ten, sixty, and three hundred second windows. The `some` row indicates the percentage of time n that given time frame in which _any_ process has stalled due to memory thrashing. `buztd` allows you to configure the value of `some avg10` in which, if surpassed, some process will be killed. The ideal value for this cutoff varies a lot between systems. Try messing around with `tools/mem-eater.c` to guesstimate a value that works well for you. ## Building Requirements: * [Zig 0.10](https://ziglang.org/) * Linux 4.20+ built with `CONFIG_PSI=y` ```shell git clone https://github.com/vrmiguel/buztd cd buztd # Choose which compilation mode you'd like: zig build -Drelease-fast # Turns on optimization and disables safety checks zig build -Drelease-safe # Turns on optimization and keeps safety checks zig build -Drelease-small # Turns on size optimizations and disables safety checks ``` ## Configuration As of the time of writing, this version of `buztd` offers no command-line argument parsing, but allows easy configuration through the `src/config.zig` file. ```zig /// Sets whether or not buztd should daemonize /// itself. Don't use this if running buztd as a systemd /// service or something of the sort. pub const should_daemonize: bool = false; /// Free RAM percentage figures below this threshold are considered to be near terminal, meaning /// that buztd will start to check for Pressure Stall Information whenever the /// free RAM figures go below this. /// However, this free RAM amount is what the sysinfo syscall gives us, which does not take in consideration /// reclaimable or cached pages. The true free RAM amount available to the OS is bigger than what it indicates. pub const free_ram_threshold: u8 = 15; /// The Linux kernel presents canonical pressure metrics for memory, found in `/proc/pressure/memory`. /// Example: /// some avg10=0.00 avg60=0.00 avg300=0.00 total=11220657 /// full avg10=0.00 avg60=0.00 avg300=0.00 total=10947429 /// These ratios are percentages of recent trends over ten, sixty, and /// three hundred second windows. The `some` row indicates the percentage of time // in that given time frame in which _any_ process has stalled due to memory thrashing. /// /// This value configured here is the value of `some avg10` in which, if surpassed, some /// process will be killed. /// /// The ideal value for this cutoff varies a lot between systems. /// Try messing around with `tools/mem-eater.c` to guesstimate a value that works well for you. pub const cutoff_psi: f32 = 0.05; /// Sets processes that buztd must never kill. /// The values expected here are the `comm` values of the process you don't want to have terminated. /// A comm-value is the filename of the executable truncated to 16 characters.. pub const unkillables = std.ComptimeStringMap(void, .{ .{ "firefox", void }, .{ "rustc", void }, .{ "electron", void }, }); /// If any error occurs, restarts the monitoring instead of exiting with an unsuccesful status code pub const retry: bool = true; ``` ================================================ FILE: zig/build.zig ================================================ const std = @import("std"); pub fn build(b: *std.build.Builder) void { // Standard target options allows the person running `zig build` to choose // what target to build for. Here we do not override the defaults, which // means any target is allowed, and the default is native. Other options // for restricting supported target set are available. const target = b.standardTargetOptions(.{}); // Standard release options allow the person running `zig build` to select // between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall. const mode = b.standardReleaseOptions(); const exe = b.addExecutable("buztd", "src/main.zig"); exe.setTarget(target); exe.setBuildMode(mode); exe.linkLibC(); exe.install(); const run_cmd = exe.run(); run_cmd.step.dependOn(b.getInstallStep()); if (b.args) |args| { run_cmd.addArgs(args); } const run_step = b.step("run", "Run the app"); run_step.dependOn(&run_cmd.step); const exe_tests = b.addTest("src/main.zig"); exe_tests.setTarget(target); exe_tests.setBuildMode(mode); const test_step = b.step("test", "Run unit tests"); test_step.dependOn(&exe_tests.step); } ================================================ FILE: zig/src/config.zig ================================================ //! The configuration file for buztd const std = @import("std"); /// Sets whether or not buztd should daemonize /// itself. Don't use this if running buztd as a systemd /// service or something of the sort. pub const should_daemonize: bool = false; /// Free RAM percentage figures below this threshold are considered to be near terminal, meaning /// that buztd will start to check for Pressure Stall Information whenever the /// free RAM figures go below this. /// However, this free RAM amount is what the sysinfo syscall gives us, which does not take in consideration /// reclaimable or cached pages. The true free RAM amount available to the OS is bigger than what it indicates. pub const free_ram_threshold: u8 = 15; /// The Linux kernel presents canonical pressure metrics for memory, found in `/proc/pressure/memory`. /// Example: /// some avg10=0.00 avg60=0.00 avg300=0.00 total=11220657 /// full avg10=0.00 avg60=0.00 avg300=0.00 total=10947429 /// These ratios are percentages of recent trends over ten, sixty, and /// three hundred second windows. The `some` row indicates the percentage of time // in that given time frame in which _any_ process has stalled due to memory thrashing. /// /// This value configured here is the value of `some avg10` in which, if surpassed, some /// process will be killed. /// /// The ideal value for this cutoff varies a lot between systems. /// Try messing around with `tools/mem-eater.c` to guesstimate a value that works well for you. pub const cutoff_psi: f32 = 0.05; /// Sets processes that buztd must never kill. /// The values expected here are the `comm` values of the process you don't want to have terminated. /// A comm-value is the filename of the executable truncated to 16 characters.. /// /// Example: /// pub const unkillables = std.ComptimeStringMap(void, .{ /// .{ "firefox", void }, /// .{ "rustc", void }, /// .{ "electron", void }, /// }); pub const unkillables = std.ComptimeStringMap(void, .{ // Ideally, don't kill the oomkiller .{ "buztd", void }, }); /// If any error occurs, restarts the monitoring instead of exiting with an unsuccesful status code pub const retry: bool = true; ================================================ FILE: zig/src/daemonize.zig ================================================ const std = @import("std"); const os = std.os; const unistd = @cImport({ @cInclude("unistd.h"); }); const signal = @cImport({ @cInclude("signal.h"); }); const stat = @cImport({ @cInclude("sys/stat.h"); }); /// Any error that might come up during process daemonization const DaemonizeError = error{FailedToSetSessionId} || os.ForkError; const SignalHandler = struct { fn ignore(sig: i32, info: *const os.siginfo_t, ctx_ptr: ?*const anyopaque) callconv(.C) void { // Ignore the signal received _ = sig; _ = ctx_ptr; _ = info; _ = ctx_ptr; } }; /// Forks the current process and makes /// the parent process quit fn fork_and_keep_child() os.ForkError!void { const is_parent_proc = (try os.fork()) != 0; // Exit off of the parent process if (is_parent_proc) { os.exit(0); } } // TODO: // * Add logging // * Chdir /// Daemonizes the calling process pub fn daemonize() DaemonizeError!void { try fork_and_keep_child(); if (unistd.setsid() < 0) { return error.FailedToSetSessionId; } // Setup signal handling var act = os.Sigaction{ .handler = .{ .sigaction = SignalHandler.ignore }, .mask = os.empty_sigset, .flags = (os.SA.SIGINFO | os.SA.RESTART | os.SA.RESETHAND), }; os.sigaction(signal.SIGCHLD, &act, null); os.sigaction(signal.SIGHUP, &act, null); // Fork yet again and keep only the child process try fork_and_keep_child(); // Set new file permissions _ = stat.umask(0); var fd: u8 = 0; // The maximum number of files a process can have open // at any time const max_files_opened = unistd.sysconf(unistd._SC_OPEN_MAX); while (fd < max_files_opened) : (fd += 1) { _ = unistd.close(fd); } } test "fork_and_keep_child works" { const getpid = os.linux.getpid; const expect = std.testing.expect; const linux = std.os.linux; const fmt = std.fmt; const first_pid = getpid(); try fork_and_keep_child(); const new_pid = getpid(); // We should now be running on a new process try expect(first_pid != new_pid); var stat_buf: linux.Stat = undefined; var buf = [_:0]u8{0} ** 128; // Current process is alive (obviously) _ = try fmt.bufPrint(&buf, "/proc/{}/stat", .{new_pid}); try expect(linux.stat(&buf, &stat_buf) == 0); // Old process should now be dead _ = try fmt.bufPrint(&buf, "/proc/{}/stat", .{first_pid}); // Give the OS some time to reap the old process std.time.sleep(250_000); try expect( // Stat should now fail linux.stat(&buf, &stat_buf) != 0); } ================================================ FILE: zig/src/main.zig ================================================ const std = @import("std"); test "imports" { _ = @import("pressure.zig"); _ = @import("daemonize.zig"); _ = @import("process.zig"); _ = @import("memory.zig"); _ = @import("monitor.zig"); _ = @import("config.zig"); } const pressure = @import("pressure.zig"); const daemon = @import("daemonize.zig"); const process = @import("process.zig"); const memory = @import("memory.zig"); const monitor = @import("monitor.zig"); const config = @import("config.zig"); const syscalls = @import("missing_syscalls.zig"); const MCL = syscalls.MCL; pub fn startMonitoring() anyerror!void { if (config.should_daemonize) { try daemon.daemonize(); } var buffer: [128]u8 = undefined; if (syscalls.mlockall(MCL.CURRENT | MCL.FUTURE | MCL.ONFAULT)) { std.log.warn("Memory pages locked.", .{}); } else |err| { std.log.warn("Failed to lock memory pages: {}. Continuing.", .{err}); } var m = try monitor.Monitor.new(&buffer); try m.poll(); } pub fn main() anyerror!void { startMonitoring() catch |err| { // If config.retry is set, get back up and running if (config.retry) { std.log.err("{s}. Continuing.", .{err}); try main(); } else { return err; } }; } ================================================ FILE: zig/src/memory.zig ================================================ const syscalls = @import("missing_syscalls.zig"); pub const MemoryInfo = struct { const Self = @This(); total_ram_mb: u64, total_swap_mb: u64, available_ram_mb: u64, available_swap_mb: u64, available_ram_percent: u8, available_swap_percent: u8, fn bytes_to_megabytes(bytes: u64, mem_unit: u64) u64 { const B_TO_MB: u64 = 1000 * 1000; return bytes / B_TO_MB * mem_unit; } fn ratio(x: u64, y: u64) u8 { const xf = @intToFloat(f32, x); const yf = @intToFloat(f32, y); const _ratio = (xf / yf) * 100.0; return @floatToInt(u8, _ratio); } pub fn new() !Self { var si: syscalls.SysInfo = undefined; try syscalls.sysinfo(&si); const mem_unit = @intCast(u64, si.mem_unit); const available_ram_mb = bytes_to_megabytes(si.freeram, mem_unit); const total_ram_mb = bytes_to_megabytes(si.totalram, mem_unit); const total_swap_mb = bytes_to_megabytes(si.totalswap, mem_unit); const available_swap_mb = bytes_to_megabytes(si.freeswap, mem_unit); const available_ram_percent = ratio(available_ram_mb, total_ram_mb); const available_swap_percent = blk: { if (total_swap_mb != 0) { break :blk ratio(available_swap_mb, total_swap_mb); } else { break :blk 0; } }; return Self{ .available_ram_mb = available_ram_mb, .total_ram_mb = total_ram_mb, .total_swap_mb = total_swap_mb, .available_swap_mb = available_swap_mb, .available_ram_percent = available_ram_percent, .available_swap_percent = available_swap_percent, }; } }; ================================================ FILE: zig/src/missing_syscalls.zig ================================================ // Until mlockall and sysinfo are added to zig's stdlib const std = @import("std"); const os = std.os; const l = std.os.linux; // Flag magic numbers pub const MCL = struct { pub const CURRENT = 1; pub const FUTURE = 2; pub const ONFAULT = 4; }; // TODO: test if this works outside of x86_64 /// Contains certain statistics on memory and swap usage, as well as the load average pub const SysInfo = extern struct { uptime: c_long, loads: [3]c_ulong, totalram: c_ulong, freeram: c_ulong, sharedram: c_ulong, bufferram: c_ulong, totalswap: c_ulong, freeswap: c_ulong, procs: c_ushort, totalhigh: c_ulong, freehigh: c_ulong, mem_unit: c_int, // pad _f: [20 - 2 * @sizeOf(c_long) - @sizeOf(c_int)]u8, }; pub const MLockError = error{ CouldNotLock, SystemResources, PermissionDenied } || os.UnexpectedError; fn syscall_mlockall(flags: i32) usize { return l.syscall1(.mlockall, @bitCast(usize, @as(isize, flags))); } pub fn mlockall(flags: i32) MLockError!void { const rc = l.getErrno(syscall_mlockall(flags)); switch (rc) { .SUCCESS => return, .AGAIN => return error.CouldNotLock, .PERM => return error.PermissionDenied, .NOMEM => return error.SystemResources, .INVAL => unreachable, else => |err| return os.unexpectedErrno(err), } } fn syscall_sysinfo(info: *SysInfo) usize { return l.syscall1(.sysinfo, @ptrToInt(info)); } pub fn sysinfo(info: *SysInfo) os.UnexpectedError!void { const rc = l.getErrno(syscall_sysinfo(info)); switch (rc) { .SUCCESS => return, .FAULT => unreachable, else => |err| return os.unexpectedErrno(err), } } ================================================ FILE: zig/src/monitor.zig ================================================ const std = @import("std"); const time = std.time; const math = std.math; const memory = @import("memory.zig"); const pressure = @import("pressure.zig"); const process = @import("process.zig"); const config = @import("config.zig"); const MemoryStatusTag = enum { ok, near_terminal, }; const MemoryStatus = union(MemoryStatusTag) { /// Memory is "okay": basically no risk of memory thrashing ok: void, /// Nearing the terminal PSI cutoff: memory thrashing is occurring or close to it. Holds the current PSI value. near_terminal: f32, }; pub const Monitor = struct { mem_info: memory.MemoryInfo, /// Memory status as of last checked status: MemoryStatus, /// A pointer to a buffer of at least 128 bytes buffer: []u8, const Self = @This(); pub fn new(buffer: []u8) !Self { var self = Self{ .mem_info = undefined, .status = undefined, .buffer = buffer, }; try self.updateMemoryStats(); return self; } pub fn updateMemoryStats(self: *Self) !void { self.mem_info = try memory.MemoryInfo.new(); self.status = blk: { if (self.mem_info.available_ram_percent <= config.free_ram_threshold) { const psi = try pressure.pressureSomeAvg10(self.buffer); std.log.warn("read avg10: {}", .{psi}); break :blk MemoryStatus{ .near_terminal = psi }; } else { break :blk MemoryStatus.ok; } }; } fn freeUpMemory(self: *Self) !void { const victim_process = try process.findVictimProcess(self.buffer); // Check for memory stats again to see if the // low-memory situation was solved while // we were searching for our victim try self.updateMemoryStats(); if (self.isMemoryLow()) { try victim_process.terminateSelf(); } } pub fn poll(self: *Self) !void { while (true) { if (self.isMemoryLow()) { try self.freeUpMemory(); } try self.updateMemoryStats(); const sleep_time = self.sleepTimeNs(); std.log.warn("sleeping for {}ms, {}% of RAM is free", .{ sleep_time, self.mem_info.available_ram_percent }); // Convert ms to ns time.sleep(sleep_time * 1000000); } } /// Determines for how long buztd should sleep /// This function is essentially a copy of how earlyoom calculates its sleep time /// /// Credits: https://github.com/rfjakob/earlyoom/blob/dea92ae67997fcb1a0664489c13d49d09d472d40/main.c#L365 /// MIT Licensed fn sleepTimeNs(self: *const Self) u64 { // Maximum expected memory fill rate as seen // with `stress -m 4 --vm-bytes 4G` const ram_fill_rate: i64 = 6000; // Maximum expected swap fill rate as seen // with membomb on zRAM const swap_fill_rate: i64 = 800; // Maximum and minimum sleep times (in ms) const min_sleep: i64 = 100; const max_sleep: i64 = 1000; // TODO: make these percentages configurable by args./config. file const ram_terminal_percent: f64 = 10.0; const swap_terminal_percent: f64 = 10.0; const f_ram_headroom_kib = (@intToFloat(f64, self.mem_info.available_ram_percent) - ram_terminal_percent) * (@intToFloat(f64, self.mem_info.total_ram_mb) * 10.0); const f_swap_headroom_kib = (@intToFloat(f64, self.mem_info.available_swap_percent) - swap_terminal_percent) * (@intToFloat(f64, self.mem_info.total_swap_mb) * 10.0); const i_ram_headroom_kib = math.max(0, @floatToInt(i64, f_ram_headroom_kib)); const i_swap_headroom_kib = math.max(0, @floatToInt(i64, f_swap_headroom_kib)); var time_to_sleep = @divFloor(i_ram_headroom_kib, ram_fill_rate) + @divFloor(i_swap_headroom_kib, swap_fill_rate); time_to_sleep = math.min(time_to_sleep, max_sleep); time_to_sleep = math.max(time_to_sleep, min_sleep); return @intCast(u64, time_to_sleep); } fn isMemoryLow(self: *const Self) bool { return switch (self.status) { MemoryStatusTag.ok => false, MemoryStatusTag.near_terminal => |psi| psi >= config.cutoff_psi, }; } }; ================================================ FILE: zig/src/pressure.zig ================================================ const std = @import("std"); const os = std.os; const fmt = std.fmt; const fs = std.fs; const mem = std.mem; const assert = std.debug.assert; pub fn pressureSomeAvg10(buffer: []u8) !f32 { assert(buffer.len >= 128); const memory_pressure_file = try fs.cwd().openFile("/proc/pressure/memory", .{}); defer memory_pressure_file.close(); var memory_pressure_reader = memory_pressure_file.reader(); // Read "some" const some = try memory_pressure_reader.readUntilDelimiter(buffer, ' '); assert(mem.eql(u8, some, "some")); // Read "avg10=" (`readUntilDelimiter` will eat the '=') const avg10 = try memory_pressure_reader.readUntilDelimiter(buffer, '='); assert(mem.eql(u8, avg10, "avg10")); // Next up is the value we want const avg10_value = try memory_pressure_reader.readUntilDelimiter(buffer, ' '); std.log.info("avg10: {s}", .{avg10_value}); return try fmt.parseFloat(f32, avg10_value); } ================================================ FILE: zig/src/process.zig ================================================ const std = @import("std"); const csig = @cImport({ @cInclude("signal.h"); }); const unistd = @cImport({ @cInclude("unistd.h"); }); const config = @import("config.zig"); const fs = std.fs; const fmt = std.fmt; const mem = std.mem; const os = std.os; const libc = std.c; const time = std.time; fn signalToString(signal: u8) []const u8 { return switch (signal) { csig.SIGTERM => "SIGTERM", csig.SIGKILL => "SIGKILL", else => "unknown", }; } pub const Process = struct { pid: u32, oom_score: i16, buffer: []u8, const Self = @This(); const ProcessError = error{ MalformedOomScore, MalformedOomScoreAdj, MalformedVmRss }; fn fromPid(pid: u32, buffer: []u8) !Self { const oom_score = try oomScoreFromPid(pid, buffer); return Self{ .pid = pid, .oom_score = oom_score, .buffer = buffer }; } fn oomScoreFromPid(pid: u32, buffer: []u8) !i16 { const path = try fmt.bufPrint(buffer, "/proc/{}/oom_score", .{pid}); // The file descriptor for the oom_score file of this process const oom_score_fd = try os.open(path, os.O.RDONLY, 0); defer os.close(oom_score_fd); const bytes_read = try os.read(oom_score_fd, buffer); const oom_score = parse(i16, buffer[0 .. bytes_read - 1]) orelse return error.MalformedOomScore; return oom_score; } pub fn oomScoreAdj(self: *const Self) !i16 { const path = try fmt.bufPrint(self.buffer, "/proc/{}/oom_score_adj", .{self.pid}); // The file descriptor for the oom_score file of this process const oom_score_adj_fd = try os.open(path, os.O.RDONLY, 0); defer os.close(oom_score_adj_fd); const bytes_read = try os.read(oom_score_adj_fd, self.buffer); const oom_score_adj = parse(i16, self.buffer[0 .. bytes_read - 1]) orelse return error.MalformedOomScoreAdj; return oom_score_adj; } pub fn comm(self: *const Self) ![]u8 { const path = try fmt.bufPrint(self.buffer, "/proc/{}/comm", .{self.pid}); // The file descriptor for the oom_score file of this process const comm_fd = try os.open(path, os.O.RDONLY, 0); defer os.close(comm_fd); const bytes_read = try os.read(comm_fd, self.buffer); return self.buffer[0 .. bytes_read - 1]; } pub fn isAlive(self: *const Self) bool { const group_id = unistd.getpgid(@intCast(c_int, self.pid)); return group_id > 0; } pub fn vmRss(self: *const Self) !usize { var filename = try fmt.bufPrint(self.buffer, "/proc/{}/statm", .{self.pid}); var statm_file = try fs.cwd().openFile(filename, .{}); defer statm_file.close(); var statm_reader = statm_file.reader(); // Skip first field (total program size) try statm_reader.skipUntilDelimiterOrEof(' '); var rss_str = try statm_reader.readUntilDelimiter(self.buffer, ' '); var ret = parse(usize, rss_str) orelse return error.MalformedVmRss; return (ret * std.mem.page_size) / 1024; } pub fn signalSelf(self: *const Self, signal: u8) void { // Don't warn `kill` failure if the process is no longer alive if (0 != libc.kill(@intCast(i32, self.pid), signal) and self.isAlive()) { std.log.warn("Failed to send {s} to process {}", .{ signalToString(signal), self.pid }); } else { std.log.warn("Successfully sent {s} to process {}", .{ signalToString(signal), self.pid }); } } pub fn terminateSelf(self: Self) !void { const quarter_sec_in_ns: u64 = 250000000; self.signalSelf(csig.SIGTERM); var attempt: u8 = 0; while (attempt < 20) : (attempt += 1) { time.sleep(quarter_sec_in_ns); if (!self.isAlive()) { std.log.warn("Process {} has exited.", .{self.pid}); return; } // Escalate to sigkill self.signalSelf(csig.SIGKILL); } } }; /// Wrapper over fmt.parseInt which returns null /// in failure instead of an error fn parse(comptime T: type, buf: []const u8) ?T { return fmt.parseInt(T, buf, 10) catch null; } /// Used to try to give LLVM hints on branch prediction. /// /// I have no idea how effective this is in practice. fn coldNoOp() void { @setCold(true); } /// Searches for a process to kill in order to /// free up memory pub fn findVictimProcess(buffer: []u8) !Process { var victim: Process = undefined; var victim_vm_rss: usize = undefined; var victim_is_undefined = true; var timer = try time.Timer.start(); var proc_dir = try fs.cwd().openIterableDir("/proc", .{ .access_sub_paths = false }); var proc_it = proc_dir.iterate(); while (try proc_it.next()) |proc_entry| { // We're only interested in directories of /proc if (proc_entry.kind != .Directory) { continue; } else { // `/proc` usually has much more directories than it has files coldNoOp(); } // But we're not interested in files that don't relate to a PID const pid = parse(u32, proc_entry.name) orelse continue; // Don't consider killing the init system if (pid <= 1) { coldNoOp(); continue; } const process = try Process.fromPid(pid, buffer); if (victim_is_undefined) { // We're still reading the first process so a victim hasn't been chosen coldNoOp(); victim = process; victim_vm_rss = try victim.vmRss(); victim_is_undefined = false; std.log.info("First victim set", .{}); } if (victim.oom_score > process.oom_score) { // Our current victim is less innocent than the process being analysed continue; } const victim_comm = try victim.comm(); if (config.unkillables.get(victim_comm) != null) { // The current process was set as unkillable continue; } const current_vm_rss = try process.vmRss(); if (current_vm_rss == 0) { // Current process is a kernel thread continue; } // TODO: recheck this if (process.oom_score == victim.oom_score and current_vm_rss <= victim_vm_rss) { continue; } const current_oom_score_adj = process.oomScoreAdj() catch { std.log.warn("Failed to read adj. OOM score for PID {}. Continuing.", .{process.pid}); continue; }; if (current_oom_score_adj == -1000) { // Follow the behaviour of the standard OOM killer: don't kill processes with oom_score_adj equals to -1000 continue; } victim = process; victim_vm_rss = current_vm_rss; } const ns_elapsed = timer.read(); std.debug.print("Victim found in {} ns.: {s} with PID {} and OOM score {}\n", .{ ns_elapsed, try victim.comm(), victim.pid, victim.oom_score }); return victim; }