From c9e37016a674e1d25d7489f8a1ee90e6d7225e74 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 28 Sep 2025 07:15:48 +0000 Subject: [PATCH 1/3] cvm: Add built-in swap config --- Cargo.lock | 1 + dstack-types/Cargo.toml | 1 + dstack-types/src/lib.rs | 3 ++ dstack-util/src/system_setup.rs | 53 +++++++++++++++++++++++++++++++++ vmm/src/main_service.rs | 32 ++++++++++---------- 5 files changed, 74 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f5a5ba07..e475d5c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2414,6 +2414,7 @@ dependencies = [ "serde", "serde-human-bytes", "sha3", + "size-parser", ] [[package]] diff --git a/dstack-types/Cargo.toml b/dstack-types/Cargo.toml index ca3f22cd..d0b0ae64 100644 --- a/dstack-types/Cargo.toml +++ b/dstack-types/Cargo.toml @@ -13,3 +13,4 @@ license.workspace = true serde = { workspace = true, features = ["derive"] } serde-human-bytes.workspace = true sha3.workspace = true +size-parser = { workspace = true, features = ["serde"] } diff --git a/dstack-types/src/lib.rs b/dstack-types/src/lib.rs index 5ecf70db..30284ae4 100644 --- a/dstack-types/src/lib.rs +++ b/dstack-types/src/lib.rs @@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize}; use serde_human_bytes as hex_bytes; +use size_parser::human_size; #[derive(Deserialize, Serialize, Debug, Clone)] pub struct AppCompose { @@ -39,6 +40,8 @@ pub struct AppCompose { pub no_instance_id: bool, #[serde(default = "default_true")] pub secure_time: bool, + #[serde(default, with = "human_size")] + pub swap_size: u64, } fn default_true() -> bool { diff --git a/dstack-util/src/system_setup.rs b/dstack-util/src/system_setup.rs index fa6eea9c..104e4a81 100644 --- a/dstack-util/src/system_setup.rs +++ b/dstack-util/src/system_setup.rs @@ -6,6 +6,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, ops::Deref, path::{Path, PathBuf}, + time::Duration, }; use anyhow::{anyhow, bail, Context, Result}; @@ -621,6 +622,55 @@ impl<'a> Stage0<'a> { } } + async fn setup_swap(&self, swap_size: u64) -> Result<()> { + let swapvol_path = "dstack/data/swapvol"; + let swapvol_device_path = format!("/dev/zvol/{swapvol_path}"); + + if Path::new(&swapvol_device_path).exists() { + cmd! { + zfs set volmode=none $swapvol_path; + zfs destroy $swapvol_path; + } + .context("Failed to destroy swap zvol")?; + } + + if swap_size == 0 { + return Ok(()); + } + + info!("Creating swap zvol at {swapvol_device_path} (size {swap_size} bytes)"); + + let size_str = swap_size.to_string(); + cmd! { + zfs create -V $size_str + -o compression=zle + -o logbias=throughput + -o sync=always + -o primarycache=metadata + -o com.sun:auto-snapshot=false + $swapvol_path + } + .with_context(|| format!("Failed to create swap zvol {swapvol_path}"))?; + + let mut count = 0u32; + while !Path::new(&swapvol_device_path).exists() && count < 10 { + std::thread::sleep(Duration::from_secs(1)); + count += 1; + } + if !Path::new(&swapvol_device_path).exists() { + bail!("Device {swapvol_device_path} did not appear after 10 seconds"); + } + + cmd! { + mkswap $swapvol_device_path; + swapon $swapvol_device_path; + swapon --show; + } + .context("Failed to enable swap on zvol")?; + + Ok(()) + } + async fn mount_data_disk(&self, initialized: bool, disk_crypt_key: &str) -> Result<()> { let name = "dstack_data_disk"; let fs_dev = "/dev/mapper/".to_string() + name; @@ -805,6 +855,9 @@ impl<'a> Stage0<'a> { self.vmm.notify_q("boot.progress", "unsealing env").await; self.mount_data_disk(is_initialized, &hex::encode(&app_keys.disk_crypt_key)) .await?; + + self.setup_swap(self.shared.app_compose.swap_size).await?; + self.vmm .notify_q( "instance.info", diff --git a/vmm/src/main_service.rs b/vmm/src/main_service.rs index 9003c526..d57fd0eb 100644 --- a/vmm/src/main_service.rs +++ b/vmm/src/main_service.rs @@ -173,22 +173,22 @@ pub fn create_manifest_from_vm_config( None => GpuConfig::default(), }; - Ok(Manifest::builder() - .id(id) - .name(request.name.clone()) - .app_id(app_id) - .image(request.image.clone()) - .vcpu(request.vcpu) - .memory(request.memory) - .disk_size(request.disk_size) - .port_map(port_map) - .created_at_ms(now) - .hugepages(request.hugepages) - .pin_numa(request.pin_numa) - .gpus(gpus) - .kms_urls(request.kms_urls.clone()) - .gateway_urls(request.gateway_urls.clone()) - .build()) + Ok(Manifest { + id, + name: request.name.clone(), + app_id, + vcpu: request.vcpu, + memory: request.memory, + disk_size: request.disk_size, + image: request.image.clone(), + port_map, + created_at_ms: now, + hugepages: request.hugepages, + pin_numa: request.pin_numa, + gpus: Some(gpus), + kms_urls: request.kms_urls.clone(), + gateway_urls: request.gateway_urls.clone(), + }) } impl RpcHandler { From 141899f0b5ee3e20d0fb1a1b443a82fc07d6235e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 28 Sep 2025 11:46:06 +0000 Subject: [PATCH 2/3] swap: Add UI and cli support --- vmm/src/console.html | 102 +++++++++++++++++++++++++++++++++++++++++-- vmm/src/vmm-cli.py | 16 +++++++ 2 files changed, 115 insertions(+), 3 deletions(-) diff --git a/vmm/src/console.html b/vmm/src/console.html index 3ab894fa..42361dda 100644 --- a/vmm/src/console.html +++ b/vmm/src/console.html @@ -661,6 +661,19 @@

Deploy a new instance

+
+ +
+ + +
+ Leave as 0 to disable swap. +
+
VM Configuration {{ formatMemory(vm.configuration?.memory) }}
+
+ Swap: + {{ formatMemory(bytesToMB(vm.configuration.swap_size)) }} +
Disk Size: {{ vm.configuration?.disk_size }} GB @@ -1063,6 +1080,21 @@

Update VM Config

+
+ +
+ + +
+ Enable "Update compose" to change swap size. +
+
Derive VM memory: 2048, // This will be computed from memoryValue and memoryUnit memoryValue: 2, memoryUnit: 'GB', + swap_size: 0, + swapValue: 0, + swapUnit: 'GB', disk_size: 20, selectedGpus: [], attachAllGpus: false, @@ -1492,6 +1527,9 @@

Derive VM

memory: 0, // This will be computed from memoryValue and memoryUnit memoryValue: 0, memoryUnit: 'MB', + swap_size: 0, + swapValue: 0, + swapUnit: 'GB', disk_size: 0, image: '', ports: [], @@ -1646,6 +1684,11 @@

Derive VM

app_compose.pre_launch_script = vmForm.value.preLaunchScript; } + const swapBytes = Math.max(0, Math.round(vmForm.value.swap_size || 0)); + if (swapBytes > 0) { + app_compose.swap_size = swapBytes; + } + // If APP_LAUNCH_TOKEN is set, add it's sha256 hash to the app compose file const launchToken = vmForm.value.encryptedEnvs.find(env => env.key === 'APP_LAUNCH_TOKEN'); if (launchToken) { @@ -1753,15 +1796,39 @@

Derive VM

showCreateDialog.value = true; vmForm.value.encryptedEnvs = []; vmForm.value.app_id = null; + vmForm.value.swapValue = 0; + vmForm.value.swapUnit = 'GB'; + vmForm.value.swap_size = 0; loadGpus(); }; // Memory conversion functions const convertMemoryToMB = (value, unit) => { + const numericValue = Number(value); + if (!Number.isFinite(numericValue) || numericValue < 0) { + return 0; + } if (unit === 'GB') { - return value * 1024; + return numericValue * 1024; } - return value; + return numericValue; + }; + + const BYTES_PER_MB = 1024 * 1024; + + const convertSwapToBytes = (value, unit) => { + const mb = convertMemoryToMB(value, unit); + if (!Number.isFinite(mb) || mb <= 0) { + return 0; + } + return Math.max(0, Math.round(mb * BYTES_PER_MB)); + }; + + const bytesToMB = (bytes) => { + if (!bytes) { + return 0; + } + return bytes / BYTES_PER_MB; }; const formatMemory = (memoryMB) => { @@ -1784,6 +1851,14 @@

Derive VM

upgradeDialog.value.memory = convertMemoryToMB(upgradeDialog.value.memoryValue, upgradeDialog.value.memoryUnit); }); + watch([() => vmForm.value.swapValue, () => vmForm.value.swapUnit], () => { + vmForm.value.swap_size = convertSwapToBytes(vmForm.value.swapValue, vmForm.value.swapUnit); + }); + + watch([() => upgradeDialog.value.swapValue, () => upgradeDialog.value.swapUnit], () => { + upgradeDialog.value.swap_size = convertSwapToBytes(upgradeDialog.value.swapValue, upgradeDialog.value.swapUnit); + }); + const createVm = async () => { try { // Convert memory based on selected unit @@ -1801,6 +1876,12 @@

Derive VM

user_config: vmForm.value.user_config, gpus: configGpu(vmForm.value), }; + const swapBytes = Math.max(0, Math.round(form.swap_size || 0)); + if (swapBytes > 0) { + form.swap_size = swapBytes; + } else { + delete form.swap_size; + } const _response = await rpcCall('CreateVm', form); loadVMList(); showCreateDialog.value = false; @@ -1976,6 +2057,10 @@

Derive VM

const selectedGpuSlots = currentGpuConfig.gpus?.map(gpu => gpu.slot) || []; const appCompose = JSON.parse(updatedVM.configuration?.compose_file || "{}"); + const swapBytes = Number(appCompose.swap_size || 0); + const swapMb = bytesToMB(swapBytes); + const swapDisplay = autoMemoryDisplay(swapMb); + upgradeDialog.value = { show: true, vm: updatedVM, @@ -1988,6 +2073,9 @@

Derive VM

vcpu: updatedVM.configuration?.vcpu || 1, memory: updatedVM.configuration?.memory || 1024, ...autoMemoryDisplay(updatedVM.configuration?.memory), + swap_size: swapBytes, + swapValue: Number(swapDisplay.memoryValue), + swapUnit: swapDisplay.memoryUnit, disk_size: updatedVM.configuration?.disk_size || 10, image: updatedVM.configuration?.image || '', ports: updatedVM.configuration?.ports?.map(port => ({ ...port })) || [], @@ -2022,6 +2110,13 @@

Derive VM

} } app_compose.pre_launch_script = upgradeDialog.value.preLaunchScript?.trim(); + + const swapBytes = Math.max(0, Math.round(upgradeDialog.value.swap_size || 0)); + if (swapBytes > 0) { + app_compose.swap_size = swapBytes; + } else { + delete app_compose.swap_size; + } return JSON.stringify(app_compose); } @@ -2414,6 +2509,7 @@

Derive VM

composeHashPreview, upgradeComposeHashPreview, formatMemory, + bytesToMB, // Pagination and search variables searchQuery, currentPage, @@ -2435,4 +2531,4 @@

Derive VM

- \ No newline at end of file + diff --git a/vmm/src/vmm-cli.py b/vmm/src/vmm-cli.py index e3cf66e7..bbdac681 100755 --- a/vmm/src/vmm-cli.py +++ b/vmm/src/vmm-cli.py @@ -498,6 +498,12 @@ def create_app_compose(self, args) -> None: if args.prelaunch_script: app_compose["pre_launch_script"] = open( args.prelaunch_script, 'rb').read().decode('utf-8') + if args.swap is not None: + swap_bytes = max(0, int(round(args.swap)) * 1024 * 1024) + if swap_bytes > 0: + app_compose["swap_size"] = swap_bytes + else: + app_compose.pop("swap_size", None) compose_file = json.dumps( app_compose, indent=4, ensure_ascii=False).encode('utf-8') @@ -537,6 +543,10 @@ def create_vm(self, args) -> None: "pin_numa": args.pin_numa, "stopped": args.stopped, } + if args.swap is not None: + swap_bytes = max(0, int(round(args.swap)) * 1024 * 1024) + if swap_bytes > 0: + params["swap_size"] = swap_bytes if args.ppcie: params["gpus"] = { @@ -908,6 +918,9 @@ def main(): '--no-instance-id', action='store_true', help='Disable instance ID') compose_parser.add_argument( '--secure-time', action='store_true', help='Enable secure time') + compose_parser.add_argument( + '--swap', type=parse_memory_size, default=None, + help='Swap size (e.g. 4G). Set to 0 to disable') compose_parser.add_argument( '--output', required=True, help='Path to output app-compose.json file') @@ -923,6 +936,9 @@ def main(): '--memory', type=parse_memory_size, default=1024, help='Memory size (e.g. 1G, 100M)') deploy_parser.add_argument( '--disk', type=parse_disk_size, default=20, help='Disk size (e.g. 1G, 100M)') + deploy_parser.add_argument( + '--swap', type=parse_memory_size, default=None, + help='Swap size (e.g. 4G). Set to 0 to disable') deploy_parser.add_argument( '--env-file', help='File with environment variables to encrypt', default=None) deploy_parser.add_argument( From 26d4c2db11c09fd53a3da424298c65c96dfa75e6 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 29 Sep 2025 02:14:45 +0000 Subject: [PATCH 3/3] swap: Handle ext4 case --- dstack-util/src/system_setup.rs | 37 ++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/dstack-util/src/system_setup.rs b/dstack-util/src/system_setup.rs index fc11bc7b..00207f89 100644 --- a/dstack-util/src/system_setup.rs +++ b/dstack-util/src/system_setup.rs @@ -686,8 +686,38 @@ impl<'a> Stage0<'a> { } } - async fn setup_swap(&self, swap_size: u64) -> Result<()> { - let swapvol_path = "dstack/data/swapvol"; + async fn setup_swap(&self, swap_size: u64, opts: &DstackOptions) -> Result<()> { + match opts.storage_fs { + FsType::Zfs => self.setup_swap_zvol(swap_size).await, + FsType::Ext4 => self.setup_swapfile(swap_size).await, + } + } + + async fn setup_swapfile(&self, swap_size: u64) -> Result<()> { + let swapfile = self.args.mount_point.join("swapfile"); + if swapfile.exists() { + fs::remove_file(&swapfile).context("Failed to remove swapfile")?; + info!("Removed existing swapfile"); + } + if swap_size == 0 { + return Ok(()); + } + let swapfile = swapfile.display().to_string(); + info!("Creating swapfile at {swapfile} (size {swap_size} bytes)"); + let size_str = swap_size.to_string(); + cmd! { + fallocate -l $size_str $swapfile; + chmod 600 $swapfile; + mkswap $swapfile; + swapon $swapfile; + swapon --show; + } + .context("Failed to enable swap on swapfile")?; + Ok(()) + } + + async fn setup_swap_zvol(&self, swap_size: u64) -> Result<()> { + let swapvol_path = "dstack/swap"; let swapvol_device_path = format!("/dev/zvol/{swapvol_path}"); if Path::new(&swapvol_device_path).exists() { @@ -1014,7 +1044,8 @@ impl<'a> Stage0<'a> { &opts, ) .await?; - self.setup_swap(self.shared.app_compose.swap_size).await?; + self.setup_swap(self.shared.app_compose.swap_size, &opts) + .await?; self.vmm .notify_q( "instance.info",