Skip to content

Commit 11cc18e

Browse files
committed
feat(vmm): support standalone shared netd deployment
1 parent cec65de commit 11cc18e

4 files changed

Lines changed: 66 additions & 20 deletions

File tree

docs/libvirt-network-filter.md

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,38 @@ validated by libvirt.
6767

6868
## Deployment modes
6969

70-
Production packages may socket-activate one shared `netd`, but that is only a
71-
deployment convenience. Development mode is two ordinary commands:
70+
Production should run one shared service. `netd` reads only the `[netd]`
71+
section, so its root-owned configuration can be small and independent of every
72+
VMM instance:
73+
74+
```toml
75+
# /etc/dstack/netd.toml
76+
[netd]
77+
socket = "/run/dstack/netd.sock"
78+
allowed_uids = [991, 992]
79+
libvirt_uri = "qemu:///system"
80+
```
81+
82+
```ini
83+
# /etc/systemd/system/dstack-netd.service
84+
[Unit]
85+
Description=dstack host networking service
86+
After=libvirtd.service
87+
88+
[Service]
89+
ExecStart=/usr/bin/dstack-vmm --config /etc/dstack/netd.toml netd
90+
Restart=on-failure
91+
92+
[Install]
93+
WantedBy=multi-user.target
94+
```
95+
96+
All VMM instance configurations point to the same socket and use distinct
97+
`cvm.instance_id` values. A dedicated netd uses a different socket. A
98+
host-wide lock serializes mutations made by shared and dedicated netd
99+
processes.
100+
101+
Development mode is two ordinary commands:
72102

73103
```bash
74104
sudo dstack-vmm --config ./vmm.toml netd \

dstack/vmm/src/app.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -499,7 +499,7 @@ impl App {
499499
}
500500
self.set_started(id, false)?;
501501
self.stop_vm_process(id).await?;
502-
let networks = self.work_dir(id).runtime_networks();
502+
let networks = self.work_dir(id)?.runtime_networks();
503503
self.remove_filtered_networks(id, &networks).await?;
504504
Ok(())
505505
}
@@ -690,7 +690,7 @@ impl App {
690690
}
691691
}
692692

693-
let runtime_networks = self.work_dir(id).runtime_networks();
693+
let runtime_networks = self.work_dir(id)?.runtime_networks();
694694
if let Err(error) = self.remove_filtered_networks(id, &runtime_networks).await {
695695
warn!(id, %error, "failed to remove filtered networking during VM removal");
696696
}

dstack/vmm/src/main.rs

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use std::{path::Path, time::Duration};
77
use anyhow::{anyhow, Context, Result};
88
use app::App;
99
use clap::{Args as ClapArgs, Parser, Subcommand};
10-
use config::Config;
10+
use config::{Config, NetdConfig};
1111
use dstack_api_auth::{Authenticator, HttpAuthConfig, HttpAuthFairing};
1212
use guest_api_service::GuestApiHandler;
1313
use host_api_service::HostApiHandler;
@@ -194,23 +194,28 @@ async fn main() -> Result<()> {
194194
}
195195

196196
let figment = config::load_config_figment(args.config.as_deref());
197-
let mut config = Config::extract_or_default(&figment)?.abs_path()?;
198-
config.cvm.instance_id = netd::instance_id(&config.cvm.instance_id, config.run_path.as_path());
199-
if let Some(socket) = args.netd_socket.as_deref() {
200-
config.netd.socket = socket.into();
201-
}
202-
203197
if let Some(Command::Netd(netd_args)) = &args.command {
198+
let mut netd_config: NetdConfig = figment
199+
.extract_inner("netd")
200+
.context("failed to load [netd] configuration")?;
201+
if let Some(socket) = args.netd_socket.as_deref() {
202+
netd_config.socket = socket.into();
203+
}
204204
if let Some(socket) = netd_args.socket.as_deref() {
205-
config.netd.socket = socket.into();
205+
netd_config.socket = socket.into();
206206
}
207-
config
208-
.netd
207+
netd_config
209208
.allowed_uids
210209
.extend(netd_args.allow_uids.iter().copied());
211-
config.netd.allowed_uids.sort_unstable();
212-
config.netd.allowed_uids.dedup();
213-
return netd::serve(config.netd).await;
210+
netd_config.allowed_uids.sort_unstable();
211+
netd_config.allowed_uids.dedup();
212+
return netd::serve(netd_config).await;
213+
}
214+
215+
let mut config = Config::extract_or_default(&figment)?.abs_path()?;
216+
config.cvm.instance_id = netd::instance_id(&config.cvm.instance_id, config.run_path.as_path());
217+
if let Some(socket) = args.netd_socket.as_deref() {
218+
config.netd.socket = socket.into();
214219
}
215220

216221
// Preserve the existing startup validation. The broader static checks are

dstack/vmm/src/one_shot.rs

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@
33
// SPDX-License-Identifier: Apache-2.0
44

55
use crate::app::{
6-
make_sys_config, simulator_config_for_manifest, sync_tee_simulator_config, Image, VmConfig,
7-
VmWorkDir,
6+
make_sys_config, resolved_networks, simulator_config_for_manifest, sync_tee_simulator_config,
7+
Image, VmConfig, VmWorkDir,
88
};
9-
use crate::config::Config;
9+
use crate::config::{Config, NetworkFilterMode, NetworkingMode};
1010
use crate::main_service;
1111
use anyhow::{Context, Result};
1212
use fs_err as fs;
@@ -279,6 +279,17 @@ Compose file content (first 200 chars):
279279
gateway_enabled: app_compose.gateway_enabled(),
280280
};
281281

282+
if !dry_run
283+
&& config.cvm.network_filter.mode == NetworkFilterMode::Libvirt
284+
&& resolved_networks(&manifest, &config.cvm)
285+
.iter()
286+
.any(|network| network.mode == NetworkingMode::Bridge)
287+
{
288+
anyhow::bail!(
289+
"one-shot execution does not manage libvirt-filtered TAP lifecycle; run the VMM server directly or use --dry-run"
290+
);
291+
}
292+
282293
let process_configs = vm_builder_config
283294
.config_qemu(&workdir_path, &config.cvm, &gpus)
284295
.context("Failed to build QEMU configuration")?;

0 commit comments

Comments
 (0)