mirror of
https://github.com/marcoallegretti/WEFT_OS.git
synced 2026-11-22 08:13:59 +00:00
fix(runtime): limit a component's memory in total and end IPC on failed sends
The memory limit applied to each linear memory separately, so a component made of several core modules could multiply it. A limiter now accounts all memories together, and bounds table elements, memories, tables and core instances. IPC fixes: - A failed or timed-out send (10 seconds) ends the connection, since part of the line may already have been written. - A line over the limit ends the connection, also when its newline arrives in the same read. - A line that is not UTF-8 is dropped and logged. TasksMax is raised to 512, since the runtime runs a few threads per CPU. Counter and Notes still run under the limits (check_counter.py and check_notes.py), and check_grants.py passes.
This commit is contained in:
parent
044634692f
commit
cbc984eef6
5 changed files with 168 additions and 22 deletions
|
|
@ -164,7 +164,7 @@ const SYSTEMD_SCOPE_ARGS: &[&str] = &[
|
||||||
"-p",
|
"-p",
|
||||||
"MemoryMax=512M",
|
"MemoryMax=512M",
|
||||||
"-p",
|
"-p",
|
||||||
"TasksMax=64",
|
"TasksMax=512",
|
||||||
];
|
];
|
||||||
|
|
||||||
fn systemd_cgroup_available() -> bool {
|
fn systemd_cgroup_available() -> bool {
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,9 @@ use std::os::unix::net::UnixStream;
|
||||||
|
|
||||||
/// The longest message in either direction, in bytes.
|
/// The longest message in either direction, in bytes.
|
||||||
pub const MAX_MESSAGE: usize = 64 * 1024;
|
pub const MAX_MESSAGE: usize = 64 * 1024;
|
||||||
|
/// How long sending one message may block before the connection is given
|
||||||
|
/// up.
|
||||||
|
const SEND_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
|
||||||
|
|
||||||
/// Checks a message the component wants to send.
|
/// Checks a message the component wants to send.
|
||||||
pub fn check_payload(payload: &str) -> Result<(), String> {
|
pub fn check_payload(payload: &str) -> Result<(), String> {
|
||||||
|
|
@ -50,15 +53,23 @@ impl IpcState {
|
||||||
if let Some(reason) = self.closed {
|
if let Some(reason) = self.closed {
|
||||||
return Err(format!("IPC connection {reason}"));
|
return Err(format!("IPC connection {reason}"));
|
||||||
}
|
}
|
||||||
let _ = self.socket.set_nonblocking(false);
|
|
||||||
let mut line = payload.to_owned();
|
let mut line = payload.to_owned();
|
||||||
line.push('\n');
|
line.push('\n');
|
||||||
let result = self
|
let result = self
|
||||||
.socket
|
.socket
|
||||||
.write_all(line.as_bytes())
|
.set_nonblocking(false)
|
||||||
.map_err(|e| e.to_string());
|
.and_then(|()| self.socket.set_write_timeout(Some(SEND_TIMEOUT)))
|
||||||
let _ = self.socket.set_nonblocking(true);
|
.and_then(|()| self.socket.write_all(line.as_bytes()));
|
||||||
result
|
let restored = self.socket.set_nonblocking(true);
|
||||||
|
match result.and(restored) {
|
||||||
|
Ok(()) => Ok(()),
|
||||||
|
// Part of the line may have been written; anything sent after it
|
||||||
|
// would join it, so the connection ends.
|
||||||
|
Err(e) => {
|
||||||
|
self.close("failed");
|
||||||
|
Err(e.to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The next complete message, if one has arrived.
|
/// The next complete message, if one has arrived.
|
||||||
|
|
@ -81,9 +92,8 @@ impl IpcState {
|
||||||
if self.recv_buf.contains(&b'\n') {
|
if self.recv_buf.contains(&b'\n') {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if self.recv_buf.len() > MAX_MESSAGE {
|
if self.recv_buf.len() > MAX_MESSAGE + 2 {
|
||||||
self.close("ended by a message over the size limit");
|
self.overlong();
|
||||||
self.recv_buf.clear();
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -103,8 +113,13 @@ impl IpcState {
|
||||||
self.closed = Some(reason);
|
self.closed = Some(reason);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn overlong(&mut self) {
|
||||||
|
self.close("ended by a message over the size limit");
|
||||||
|
self.recv_buf.clear();
|
||||||
|
}
|
||||||
|
|
||||||
/// Removes the first complete line from the buffer. A line that is not
|
/// Removes the first complete line from the buffer. A line that is not
|
||||||
/// UTF-8, or longer than a message, is dropped.
|
/// UTF-8 is dropped; one longer than a message ends the connection.
|
||||||
fn take_line(&mut self) -> Option<String> {
|
fn take_line(&mut self) -> Option<String> {
|
||||||
loop {
|
loop {
|
||||||
let pos = self.recv_buf.iter().position(|&b| b == b'\n')?;
|
let pos = self.recv_buf.iter().position(|&b| b == b'\n')?;
|
||||||
|
|
@ -112,10 +127,12 @@ impl IpcState {
|
||||||
let text = raw.strip_suffix(b"\n").unwrap_or(&raw);
|
let text = raw.strip_suffix(b"\n").unwrap_or(&raw);
|
||||||
let text = text.strip_suffix(b"\r").unwrap_or(text);
|
let text = text.strip_suffix(b"\r").unwrap_or(text);
|
||||||
if text.len() > MAX_MESSAGE {
|
if text.len() > MAX_MESSAGE {
|
||||||
continue;
|
self.overlong();
|
||||||
|
return None;
|
||||||
}
|
}
|
||||||
if let Ok(message) = String::from_utf8(text.to_vec()) {
|
match String::from_utf8(text.to_vec()) {
|
||||||
return Some(message);
|
Ok(message) => return Some(message),
|
||||||
|
Err(_) => tracing::warn!("IPC message that is not UTF-8 dropped"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -164,6 +181,34 @@ mod tests {
|
||||||
assert!(ipc.send("after").is_err());
|
assert!(ipc.send("after").is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_message_at_the_limit_arrives_and_one_over_it_ends_the_connection() {
|
||||||
|
let (mut ipc, mut peer) = pair();
|
||||||
|
let mut at_limit = vec![b'x'; MAX_MESSAGE];
|
||||||
|
at_limit.push(b'\n');
|
||||||
|
peer.write_all(&at_limit).unwrap();
|
||||||
|
let mut received = None;
|
||||||
|
while received.is_none() {
|
||||||
|
received = ipc.recv();
|
||||||
|
}
|
||||||
|
assert_eq!(received.map(|m| m.len()), Some(MAX_MESSAGE));
|
||||||
|
// One byte over, with its newline in the same read.
|
||||||
|
let mut over = vec![b'x'; MAX_MESSAGE + 1];
|
||||||
|
over.extend_from_slice(b"\nnext\n");
|
||||||
|
peer.write_all(&over).unwrap();
|
||||||
|
while ipc.closed.is_none() {
|
||||||
|
assert_eq!(ipc.recv(), None);
|
||||||
|
}
|
||||||
|
assert_eq!(ipc.recv(), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_line_that_is_not_utf8_is_dropped() {
|
||||||
|
let (mut ipc, mut peer) = pair();
|
||||||
|
peer.write_all(b"\xff\xfe\nafter\n").unwrap();
|
||||||
|
assert_eq!(ipc.recv().as_deref(), Some("after"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn a_closed_connection_is_reported_to_senders() {
|
fn a_closed_connection_is_reported_to_senders() {
|
||||||
let (mut ipc, peer) = pair();
|
let (mut ipc, peer) = pair();
|
||||||
|
|
|
||||||
105
crates/weft-runtime/src/limits.rs
Normal file
105
crates/weft-runtime/src/limits.rs
Normal file
|
|
@ -0,0 +1,105 @@
|
||||||
|
//! The memory and table limits of a component's store.
|
||||||
|
//!
|
||||||
|
//! The limit on linear memory covers all of a component's memories
|
||||||
|
//! together: a component made of several core modules cannot multiply it
|
||||||
|
//! by declaring more memories. A memory or table that would grow past its
|
||||||
|
//! limit does not grow, and the component sees an allocation failure.
|
||||||
|
|
||||||
|
/// The most table elements across all of a component's tables.
|
||||||
|
const MAX_TABLE_ELEMENTS: usize = 1_000_000;
|
||||||
|
/// The most core instances, memories and tables in one store.
|
||||||
|
const MAX_INSTANCES: usize = 1_000;
|
||||||
|
const MAX_MEMORIES: usize = 100;
|
||||||
|
const MAX_TABLES: usize = 1_000;
|
||||||
|
|
||||||
|
pub struct Limits {
|
||||||
|
max_memory: usize,
|
||||||
|
memory: usize,
|
||||||
|
table_elements: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Limits {
|
||||||
|
/// Limits a store's linear memory to `max_memory` bytes in total.
|
||||||
|
pub fn new(max_memory: usize) -> Self {
|
||||||
|
Self {
|
||||||
|
max_memory,
|
||||||
|
memory: 0,
|
||||||
|
table_elements: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Accounts a growth from `current` to `desired` against `used` of `max`.
|
||||||
|
fn grow(used: &mut usize, max: usize, current: usize, desired: usize) -> bool {
|
||||||
|
let added = desired.saturating_sub(current);
|
||||||
|
match used.checked_add(added) {
|
||||||
|
Some(total) if total <= max => {
|
||||||
|
*used = total;
|
||||||
|
true
|
||||||
|
}
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl wasmtime::ResourceLimiter for Limits {
|
||||||
|
fn memory_growing(
|
||||||
|
&mut self,
|
||||||
|
current: usize,
|
||||||
|
desired: usize,
|
||||||
|
_maximum: Option<usize>,
|
||||||
|
) -> anyhow::Result<bool> {
|
||||||
|
Ok(grow(&mut self.memory, self.max_memory, current, desired))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn table_growing(
|
||||||
|
&mut self,
|
||||||
|
current: usize,
|
||||||
|
desired: usize,
|
||||||
|
_maximum: Option<usize>,
|
||||||
|
) -> anyhow::Result<bool> {
|
||||||
|
Ok(grow(
|
||||||
|
&mut self.table_elements,
|
||||||
|
MAX_TABLE_ELEMENTS,
|
||||||
|
current,
|
||||||
|
desired,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn instances(&self) -> usize {
|
||||||
|
MAX_INSTANCES
|
||||||
|
}
|
||||||
|
|
||||||
|
fn memories(&self) -> usize {
|
||||||
|
MAX_MEMORIES
|
||||||
|
}
|
||||||
|
|
||||||
|
fn tables(&self) -> usize {
|
||||||
|
MAX_TABLES
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use wasmtime::ResourceLimiter;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_limit_covers_all_memories_together() {
|
||||||
|
let mut limits = Limits::new(100);
|
||||||
|
assert!(limits.memory_growing(0, 60, None).unwrap());
|
||||||
|
// A second memory gets only what the first left.
|
||||||
|
assert!(!limits.memory_growing(0, 60, None).unwrap());
|
||||||
|
assert!(limits.memory_growing(0, 40, None).unwrap());
|
||||||
|
// Growing the first one further no longer fits.
|
||||||
|
assert!(!limits.memory_growing(60, 61, None).unwrap());
|
||||||
|
assert_eq!(limits.memory, 100);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_refused_growth_is_not_counted() {
|
||||||
|
let mut limits = Limits::new(100);
|
||||||
|
assert!(!limits.memory_growing(0, 101, None).unwrap());
|
||||||
|
assert!(limits.memory_growing(0, 100, None).unwrap());
|
||||||
|
assert!(!limits.memory_growing(0, usize::MAX, None).unwrap());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -5,6 +5,8 @@ use anyhow::Context;
|
||||||
mod grants;
|
mod grants;
|
||||||
#[cfg(feature = "wasmtime-runtime")]
|
#[cfg(feature = "wasmtime-runtime")]
|
||||||
mod ipc;
|
mod ipc;
|
||||||
|
#[cfg(feature = "wasmtime-runtime")]
|
||||||
|
mod limits;
|
||||||
|
|
||||||
/// The default limit on a component's linear memory, in MiB.
|
/// The default limit on a component's linear memory, in MiB.
|
||||||
const DEFAULT_MAX_MEMORY_MIB: usize = 256;
|
const DEFAULT_MAX_MEMORY_MIB: usize = 256;
|
||||||
|
|
@ -147,7 +149,7 @@ fn run_module(
|
||||||
struct State {
|
struct State {
|
||||||
ctx: WasiCtx,
|
ctx: WasiCtx,
|
||||||
table: ResourceTable,
|
table: ResourceTable,
|
||||||
limits: wasmtime::StoreLimits,
|
limits: limits::Limits,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl IoView for State {
|
impl IoView for State {
|
||||||
|
|
@ -319,13 +321,7 @@ fn run_module(
|
||||||
State {
|
State {
|
||||||
ctx,
|
ctx,
|
||||||
table: ResourceTable::new(),
|
table: ResourceTable::new(),
|
||||||
// A memory or table that would grow past the limit does not
|
limits: limits::Limits::new(max_memory),
|
||||||
// grow: the component sees an allocation failure.
|
|
||||||
limits: wasmtime::StoreLimitsBuilder::new()
|
|
||||||
.memory_size(max_memory)
|
|
||||||
.table_elements(100_000)
|
|
||||||
.instances(1_000)
|
|
||||||
.build(),
|
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
store.limiter(|state| &mut state.limits);
|
store.limiter(|state| &mut state.limits);
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ Installing a package does not yet ask the user to approve its capabilities; ever
|
||||||
|
|
||||||
## Process Isolation
|
## Process Isolation
|
||||||
|
|
||||||
Each app session runs as a separate OS process (`weft-runtime`). When systemd is available, the process is wrapped in a systemd scope (`weft-apps.slice`) with `CPUQuota=200%`, `MemoryMax=512M` and `TasksMax=64`. Independently of systemd, the runtime limits the component's linear memory to 256 MiB (`--max-memory-mib` changes it): a memory that would grow past the limit does not grow, and the component sees an allocation failure. App messages over `weft:app/ipc` are single lines of at most 64 KiB, the limit weft-appd applies; a component's message that is longer or contains a line break is refused, and a longer line from weft-appd ends the connection, after which sending reports that it is closed. CPU time within the quota is not limited per component.
|
Each app session runs as a separate OS process (`weft-runtime`). When systemd is available, the process is wrapped in a systemd scope (`weft-apps.slice`) with `CPUQuota=200%`, `MemoryMax=512M` and `TasksMax=512` (threads included; the runtime runs a few per CPU). Independently of systemd, the runtime limits the component's linear memory to 256 MiB in total, across all its memories (`--max-memory-mib` changes it), and bounds its tables, memories and core instances: a memory or table that would grow past its limit does not grow, and the component sees an allocation failure. App messages over `weft:app/ipc` are single lines of at most 64 KiB, the limit weft-appd applies; a component's message that is longer or contains a line break is refused, a longer line from weft-appd, or a failed or timed-out send (10 seconds), ends the connection, after which sending reports that it is closed. CPU time within the quota is not limited per component.
|
||||||
|
|
||||||
## Filesystem Isolation
|
## Filesystem Isolation
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue