diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 63766de8f..d05e32977 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -1873,7 +1873,6 @@ dependencies = [ "dstack-build-info", "dstack-gateway-rpc", "dstack-guest-agent-rpc", - "dstack-kms-rpc", "dstack-types", "flate2", "fs-err", diff --git a/dstack/gateway/Cargo.toml b/dstack/gateway/Cargo.toml index b4c05113c..5875ad397 100644 --- a/dstack/gateway/Cargo.toml +++ b/dstack/gateway/Cargo.toml @@ -32,7 +32,7 @@ pin-project.workspace = true serde_json.workspace = true rand.workspace = true dstack-build-info.workspace = true -ra-rpc = { workspace = true, features = ["client", "rocket"] } +ra-rpc = { workspace = true, features = ["rocket"] } dstack-gateway-rpc.workspace = true certbot.workspace = true bytes.workspace = true @@ -41,7 +41,6 @@ smallvec.workspace = true futures.workspace = true cmd_lib.workspace = true load_config.workspace = true -dstack-kms-rpc.workspace = true ra-tls.workspace = true dstack-guest-agent-rpc.workspace = true http-client = { workspace = true, features = ["prpc"] } @@ -72,10 +71,6 @@ ktls.workspace = true libc.workspace = true socket2.workspace = true -[[bin]] -name = "gen_debug_key" -path = "src/gen_debug_key.rs" - [dev-dependencies] insta.workspace = true tempfile.workspace = true diff --git a/dstack/gateway/docs/cluster-deployment.md b/dstack/gateway/docs/cluster-deployment.md index 2a80a5514..1a46ab901 100644 --- a/dstack/gateway/docs/cluster-deployment.md +++ b/dstack/gateway/docs/cluster-deployment.md @@ -168,6 +168,8 @@ This allows a simple deployment order: ### 2.5 Configuration File Examples +> **Note:** A non-empty `rpc_domain` makes the gateway request its RPC TLS key and certificate from the local dstack Guest Agent. Ensure `/var/run/dstack/dstack.sock` is available, or set `DSTACK_AGENT_ADDRESS` to another Guest Agent endpoint. Set `rpc_domain = ""` when supplying pre-generated certificates. + gateway-1.toml: ```toml @@ -184,7 +186,6 @@ ca_certs = "/var/lib/gateway/certs/gateway-ca.cert" mandatory = false [core] -kms_url = "https://kms.demo.dstack.org" rpc_domain = "rpc.gateway-1.demo.dstack.org" [core.admin] @@ -240,7 +241,6 @@ ca_certs = "/var/lib/gateway/certs/gateway-ca.cert" mandatory = false [core] -kms_url = "https://kms.demo.dstack.org" rpc_domain = "rpc.gateway-2.demo.dstack.org" [core.sync] diff --git a/dstack/gateway/gateway.toml b/dstack/gateway/gateway.toml index 12e594ffc..fdbe04d42 100644 --- a/dstack/gateway/gateway.toml +++ b/dstack/gateway/gateway.toml @@ -11,7 +11,6 @@ log_level = "info" address = "127.0.0.1:8010" [core] -kms_url = "" # auto set soft ulimit to hard ulimit set_ulimit = true rpc_domain = "" @@ -62,7 +61,6 @@ insecure_enable_debug_rpc = false # gateway's own loopback on a port of their choosing, bypassing port_policy. insecure_localhost_backend = false insecure_skip_attestation = false -key_file = "debug_key.json" address = "127.0.0.1:8012" [core.wg] diff --git a/dstack/gateway/src/config.rs b/dstack/gateway/src/config.rs index 34383e6e6..af4d986a3 100644 --- a/dstack/gateway/src/config.rs +++ b/dstack/gateway/src/config.rs @@ -510,7 +510,6 @@ pub struct Config { pub recycle: RecycleConfig, pub set_ulimit: bool, pub rpc_domain: String, - pub kms_url: String, pub admin: AdminConfig, /// Debug server configuration (separate port for debug RPCs) pub debug: DebugConfig, @@ -538,9 +537,6 @@ pub struct DebugConfig { /// registered instance and so bypasses `port_policy` entirely. #[serde(default)] pub insecure_localhost_backend: bool, - /// Path to pre-generated debug key data file (JSON format containing key, quote, event_log, and vm_config) - #[serde(default)] - pub key_file: String, } #[derive(Debug, Clone, Deserialize)] diff --git a/dstack/gateway/src/gen_debug_key.rs b/dstack/gateway/src/gen_debug_key.rs deleted file mode 100644 index c710548a6..000000000 --- a/dstack/gateway/src/gen_debug_key.rs +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-FileCopyrightText: © 2025 Phala Network -// -// SPDX-License-Identifier: Apache-2.0 - -// Run with: cargo run --bin gen_debug_key -- -// Example: cargo run --bin gen_debug_key -- https://daee134c3b9f66aa2401c3b5ea64f1d34038f45d-3000.tdxlab.dstack.org:12004 - -use anyhow::{Context, Result}; -use base64::{engine::general_purpose::STANDARD, Engine as _}; -use dstack_guest_agent_rpc::{dstack_guest_client::DstackGuestClient, RawQuoteArgs}; -use http_client::prpc::PrpcClient; -use ra_tls::attestation::QuoteContentType; -use ra_tls::rcgen::KeyPair; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct DebugKeyData { - /// Private key in PEM format - key_pem: String, - /// TDX quote in base64 format - quote_base64: String, - /// Event log in JSON string format - event_log: String, - /// VM config in JSON string format - vm_config: String, -} - -#[tokio::main] -async fn main() -> Result<()> { - let args: Vec = std::env::args().collect(); - if args.len() != 2 { - eprintln!("Usage: {} ", args[0]); - eprintln!("Example: {} https://daee134c3b9f66aa2401c3b5ea64f1d34038f45d-3000.tdxlab.dstack.org:12004", args[0]); - std::process::exit(1); - } - let simulator_url = &args[1]; - - // Generate key pair - let key = KeyPair::generate().context("Failed to generate key")?; - let pubkey = key.public_key_der(); - let key_pem = key.serialize_pem(); - - // Calculate report_data - let report_data = QuoteContentType::RaTlsCert.to_report_data(&pubkey); - - // Get quote from simulator - println!("Getting quote from simulator: {simulator_url}"); - let simulator_client = PrpcClient::new(simulator_url.to_string()); - let simulator_client = DstackGuestClient::new(simulator_client); - let quote_response = simulator_client - .get_quote(RawQuoteArgs { - report_data: report_data.to_vec(), - }) - .await - .context("Failed to get quote from simulator")?; - - // Create debug key data structure - let debug_data = DebugKeyData { - key_pem, - quote_base64: STANDARD.encode("e_response.quote), - event_log: quote_response.event_log, - vm_config: quote_response.vm_config, - }; - - // Write to single JSON file - let json_content = - serde_json::to_string_pretty(&debug_data).context("Failed to serialize debug key data")?; - let output_file = "debug_key.json"; - fs_err::write(output_file, json_content).context("Failed to write debug key file")?; - - println!("✓ Successfully generated debug key data:"); - println!(" - {output_file}"); - println!("\nYou can now configure this path in your gateway config:"); - println!("[core.debug]"); - println!("insecure_skip_attestation = true"); - println!( - "key_file = \"{}\"", - fs_err::canonicalize(output_file) - .unwrap_or_default() - .display() - ); - - Ok(()) -} diff --git a/dstack/gateway/src/main.rs b/dstack/gateway/src/main.rs index 01fd6a111..e7b2eb1a2 100644 --- a/dstack/gateway/src/main.rs +++ b/dstack/gateway/src/main.rs @@ -3,23 +3,16 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::{anyhow, Context, Result}; -use base64::{engine::general_purpose::STANDARD, Engine as _}; use clap::Parser; use config::{Config, TlsConfig}; use dstack_guest_agent_rpc::{dstack_guest_client::DstackGuestClient, GetTlsKeyArgs}; -use dstack_kms_rpc::SignCertRequest; use http_client::prpc::PrpcClient; -use ra_rpc::{client::RaClient, prpc_routes as prpc, rocket_helper::QuoteVerifier}; -use ra_tls::rcgen::KeyPair; -use ra_tls::{ - attestation::AttestationVerifier, - cert::{CertConfigV2, CertSigningRequestV2, Csr}, -}; +use ra_rpc::{prpc_routes as prpc, rocket_helper::QuoteVerifier}; +use ra_tls::attestation::AttestationVerifier; use rocket::{ fairing::AdHoc, figment::{providers::Serialized, Figment}, }; -use serde::{Deserialize, Serialize}; use std::sync::Arc; use tracing::{info, warn}; @@ -41,18 +34,6 @@ mod pp; mod proxy; mod web_routes; -#[derive(Debug, Clone, Serialize, Deserialize)] -struct DebugKeyData { - /// Private key in PEM format - key_pem: String, - /// TDX quote in base64 format - quote_base64: String, - /// Event log in JSON string format - event_log: String, - /// VM config in JSON string format - vm_config: String, -} - #[global_allocator] static ALLOCATOR: jemallocator::Jemalloc = jemallocator::Jemalloc; @@ -99,13 +80,10 @@ async fn maybe_gen_certs(config: &Config, tls_config: &TlsConfig) -> Result<()> } } } - match config.debug.insecure_skip_attestation { - true => gen_debug_certs(config, tls_config, alt_names).await, - false => gen_prod_certs(tls_config, alt_names).await, - } + gen_certs(tls_config, alt_names).await } -async fn gen_prod_certs(tls_config: &TlsConfig, alt_names: Vec) -> Result<()> { +async fn gen_certs(tls_config: &TlsConfig, alt_names: Vec) -> Result<()> { info!("Using dstack guest agent for certificate generation"); let agent_client = dstack_agent().context("Failed to create dstack client")?; @@ -134,93 +112,6 @@ async fn gen_prod_certs(tls_config: &TlsConfig, alt_names: Vec) -> Resul Ok(()) } -async fn gen_debug_certs( - config: &Config, - tls_config: &TlsConfig, - alt_names: Vec, -) -> Result<()> { - let kms_url = config.kms_url.clone(); - if kms_url.is_empty() { - info!("KMS URL is empty, skipping cert generation"); - return Ok(()); - } - - // Check if debug key file is configured - if config.debug.key_file.is_empty() { - info!("Debug key file not configured, skipping cert generation"); - return Ok(()); - } - - // Load pre-generated key pair and quote data from JSON file - info!("Loading debug key data from: {}", config.debug.key_file); - let ctx = "Failed to read debug key, run `cargo run --bin gen_debug_key -- ` to generate it"; - let json_content = fs_err::read_to_string(&config.debug.key_file).context(ctx)?; - let debug_data: DebugKeyData = - serde_json::from_str(&json_content).context("Failed to parse debug key JSON")?; - - let key_pem = debug_data.key_pem; - let quote_bin = STANDARD - .decode(&debug_data.quote_base64) - .context("Failed to decode quote from base64")?; - let event_log_json = debug_data.event_log; - let vm_config_json = debug_data.vm_config; - - // Parse key pair - let key = KeyPair::from_pem(&key_pem).context("Failed to parse debug key")?; - let pubkey = key.public_key_der(); - - // Build CSR with attestation from debug quote - let attestation = - ra_tls::attestation::Attestation::from_tdx_quote(quote_bin, event_log_json.as_bytes()) - .context("Failed to create attestation from debug quote")? - .into_versioned(); - - let csr = CertSigningRequestV2 { - confirm: "please sign cert:".to_string(), - pubkey, - config: CertConfigV2 { - org_name: None, - subject: "dstack-gateway".to_string(), - subject_alt_names: alt_names, - usage_server_auth: true, - usage_client_auth: true, - ext_quote: true, - ext_app_info: true, - not_before: None, - not_after: None, - }, - attestation, - }; - let signature = csr.signed_by(&key).context("Failed to sign CSR")?; - - // Send CSR to KMS for signing - let kms_url = format!("{kms_url}/prpc"); - info!("Sending CSR to KMS for signing: {kms_url}"); - let kms_client = RaClient::new(kms_url, true).context("Failed to create kms client")?; - let kms_client = dstack_kms_rpc::kms_client::KmsClient::new(kms_client); - let sign_response = kms_client - .sign_cert(SignCertRequest { - api_version: 2, - csr: csr.to_vec(), - signature, - vm_config: vm_config_json.to_string(), - }) - .await - .context("Failed to sign certificate via KMS")?; - - let ca_cert = sign_response - .certificate_chain - .last() - .context("Empty certificate chain")? - .to_string(); - let certs = sign_response.certificate_chain.join("\n"); - - write_cert(&tls_config.mutual.ca_certs, &ca_cert)?; - write_cert(&tls_config.certs, &certs)?; - write_cert(&tls_config.key, &key.serialize_pem())?; - Ok(()) -} - fn write_cert(path: &str, cert: &str) -> Result<()> { info!("Writing cert to file: {path}"); safe_write::safe_write_with_mode(path, cert, 0o600)?; diff --git a/dstack/gateway/test-run/cluster.sh b/dstack/gateway/test-run/cluster.sh index 23521bd1d..27f58b4c6 100755 --- a/dstack/gateway/test-run/cluster.sh +++ b/dstack/gateway/test-run/cluster.sh @@ -160,8 +160,7 @@ ca_certs = "${abs_run_dir}/certs/gateway-ca.cert" mandatory = false [core] -kms_url = "" -rpc_domain = "gateway.test.local" +rpc_domain = "" [core.debug] insecure_enable_debug_rpc = true diff --git a/dstack/gateway/test-run/e2e/Dockerfile.simulator b/dstack/gateway/test-run/e2e/Dockerfile.simulator new file mode 100644 index 000000000..643e83c42 --- /dev/null +++ b/dstack/gateway/test-run/e2e/Dockerfile.simulator @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +FROM rust:1.92-bookworm AS builder +WORKDIR /src +COPY . . +RUN cargo build --manifest-path dstack/Cargo.toml --locked --release \ + -p dstack-guest-agent-simulator + +FROM debian:bookworm-slim +RUN apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates && \ + rm -rf /var/lib/apt/lists/* +WORKDIR /opt/dstack-simulator +COPY --from=builder /src/dstack/target/release/dstack-simulator /usr/local/bin/dstack-simulator +COPY sdk/simulator/app-compose.json sdk/simulator/appkeys.json \ + sdk/simulator/sys-config.json sdk/simulator/attestation.bin ./ +COPY dstack/gateway/test-run/e2e/configs/simulator.toml ./simulator.toml +CMD ["dstack-simulator", "--config", "/opt/dstack-simulator/simulator.toml"] diff --git a/dstack/gateway/test-run/e2e/Dockerfile.simulator.dockerignore b/dstack/gateway/test-run/e2e/Dockerfile.simulator.dockerignore new file mode 100644 index 000000000..f53ffe346 --- /dev/null +++ b/dstack/gateway/test-run/e2e/Dockerfile.simulator.dockerignore @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +.git +**/target +**/node_modules +**/__pycache__ +**/.env +**/.env.* diff --git a/dstack/gateway/test-run/e2e/configs/gateway-1.toml b/dstack/gateway/test-run/e2e/configs/gateway-1.toml index fc5f49243..b90efd4db 100644 --- a/dstack/gateway/test-run/e2e/configs/gateway-1.toml +++ b/dstack/gateway/test-run/e2e/configs/gateway-1.toml @@ -12,7 +12,6 @@ ca_certs = "/var/lib/gateway/certs/gateway-ca.cert" mandatory = false [core] -kms_url = "" rpc_domain = "gateway-1" [core.admin] diff --git a/dstack/gateway/test-run/e2e/configs/gateway-2.toml b/dstack/gateway/test-run/e2e/configs/gateway-2.toml index 40a9491a1..c7bdd729f 100644 --- a/dstack/gateway/test-run/e2e/configs/gateway-2.toml +++ b/dstack/gateway/test-run/e2e/configs/gateway-2.toml @@ -12,7 +12,6 @@ ca_certs = "/var/lib/gateway/certs/gateway-ca.cert" mandatory = false [core] -kms_url = "" rpc_domain = "gateway-2" [core.admin] diff --git a/dstack/gateway/test-run/e2e/configs/gateway-3.toml b/dstack/gateway/test-run/e2e/configs/gateway-3.toml index f71e43109..0cb845126 100644 --- a/dstack/gateway/test-run/e2e/configs/gateway-3.toml +++ b/dstack/gateway/test-run/e2e/configs/gateway-3.toml @@ -12,7 +12,6 @@ ca_certs = "/var/lib/gateway/certs/gateway-ca.cert" mandatory = false [core] -kms_url = "" rpc_domain = "gateway-3" [core.admin] diff --git a/dstack/gateway/test-run/e2e/configs/simulator.toml b/dstack/gateway/test-run/e2e/configs/simulator.toml new file mode 100644 index 000000000..e7020dbca --- /dev/null +++ b/dstack/gateway/test-run/e2e/configs/simulator.toml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 + +[default] +workers = 8 +max_blocking = 64 +ident = "dstack Gateway E2E Simulator" +temp_dir = "/tmp" +keep_alive = 10 +log_level = "info" + +[default.core] +keys_file = "/opt/dstack-simulator/appkeys.json" +compose_file = "/opt/dstack-simulator/app-compose.json" +sys_config_file = "/opt/dstack-simulator/sys-config.json" +data_disks = ["/"] + +[default.core.simulator] +attestation_file = "/opt/dstack-simulator/attestation.bin" +patch_report_data = true + +[internal] +address = "unix:/var/run/dstack/dstack.sock" +reuse = true diff --git a/dstack/gateway/test-run/e2e/docker-compose.yml b/dstack/gateway/test-run/e2e/docker-compose.yml index 2374b0730..59048d84f 100644 --- a/dstack/gateway/test-run/e2e/docker-compose.yml +++ b/dstack/gateway/test-run/e2e/docker-compose.yml @@ -4,7 +4,7 @@ # E2E test environment for dstack-gateway certbot functionality # Uses mock services: Pebble (ACME) + mock-cf-dns-api (Cloudflare DNS) -# Uses real TDX endpoint for attestation +# Uses a test-local dstack Guest Agent simulator for certificate and app identity flows. networks: certbot-test: @@ -15,8 +15,22 @@ networks: volumes: pebble-certs: + dstack-socket: services: + dstack-simulator: + build: + context: ../../../.. + dockerfile: dstack/gateway/test-run/e2e/Dockerfile.simulator + image: dstack-simulator:gateway-e2e + volumes: + - dstack-socket:/var/run/dstack + healthcheck: + test: ["CMD-SHELL", "test -S /var/run/dstack/dstack.sock"] + interval: 1s + timeout: 1s + retries: 30 + # ==================== Mock Services ==================== # Mock Cloudflare DNS API @@ -73,12 +87,15 @@ services: - "19016:9016" # Admin volumes: - ./configs/gateway-1.toml:/etc/gateway/gateway.toml:ro + - dstack-socket:/var/run/dstack tmpfs: - /var/lib/gateway environment: - RUST_LOG=info,dstack_gateway=debug,certbot=debug - - DSTACK_AGENT_ADDRESS=https://712eab2f507b963e11144ae67218177e93ac2a24-3000.tdxlab.dstack.org:12004/ + - DSTACK_AGENT_ADDRESS=unix:/var/run/dstack/dstack.sock depends_on: + dstack-simulator: + condition: service_healthy mock-cf-dns-api: condition: service_healthy pebble: @@ -109,11 +126,12 @@ services: - "19026:9016" # Admin volumes: - ./configs/gateway-2.toml:/etc/gateway/gateway.toml:ro + - dstack-socket:/var/run/dstack tmpfs: - /var/lib/gateway environment: - RUST_LOG=info,dstack_gateway=debug,certbot=debug - - DSTACK_AGENT_ADDRESS=https://712eab2f507b963e11144ae67218177e93ac2a24-3000.tdxlab.dstack.org:12004/ + - DSTACK_AGENT_ADDRESS=unix:/var/run/dstack/dstack.sock depends_on: gateway-1: condition: service_healthy @@ -140,11 +158,12 @@ services: - "19036:9016" # Admin volumes: - ./configs/gateway-3.toml:/etc/gateway/gateway.toml:ro + - dstack-socket:/var/run/dstack tmpfs: - /var/lib/gateway environment: - RUST_LOG=info,dstack_gateway=debug,certbot=debug - - DSTACK_AGENT_ADDRESS=https://712eab2f507b963e11144ae67218177e93ac2a24-3000.tdxlab.dstack.org:12004/ + - DSTACK_AGENT_ADDRESS=unix:/var/run/dstack/dstack.sock depends_on: gateway-2: condition: service_healthy diff --git a/dstack/gateway/test-run/proxy/gwconfig.py b/dstack/gateway/test-run/proxy/gwconfig.py index 44a230e4c..46e488fd1 100755 --- a/dstack/gateway/test-run/proxy/gwconfig.py +++ b/dstack/gateway/test-run/proxy/gwconfig.py @@ -54,7 +54,6 @@ def main(): [tls.mutual] ca_certs = "{cert}" [core] -kms_url = "" rpc_domain = "" set_ulimit = false [core.debug] diff --git a/dstack/gateway/test-run/test_certbot.sh b/dstack/gateway/test-run/test_certbot.sh index 26a6ba972..29d3a58b4 100755 --- a/dstack/gateway/test-run/test_certbot.sh +++ b/dstack/gateway/test-run/test_certbot.sh @@ -156,8 +156,7 @@ ca_certs = "${abs_run_dir}/certs/gateway-ca.cert" mandatory = false [core] -kms_url = "" -rpc_domain = "gateway.tdxlab.dstack.org" +rpc_domain = "" [core.debug] insecure_enable_debug_rpc = true diff --git a/dstack/gateway/test-run/test_suite.sh b/dstack/gateway/test-run/test_suite.sh index 0de50202b..b3258779b 100755 --- a/dstack/gateway/test-run/test_suite.sh +++ b/dstack/gateway/test-run/test_suite.sh @@ -88,9 +88,7 @@ ca_certs = "${abs_run_dir}/certs/gateway-ca.cert" mandatory = false [core] -# Empty kms_url to skip auto-cert generation (we use pre-generated certs) -kms_url = "" -rpc_domain = "gateway.tdxlab.dstack.org" +rpc_domain = "" [core.debug] insecure_enable_debug_rpc = true diff --git a/tools/dev-stack.sh b/tools/dev-stack.sh index 51bce204f..442fb745b 100755 --- a/tools/dev-stack.sh +++ b/tools/dev-stack.sh @@ -214,11 +214,7 @@ ca_certs = "$CERTS_DIR/gateway-ca.cert" mandatory = false [core] -kms_url = "https://localhost:$KMS_RPC_LISTEN_PORT" -rpc_domain = "$GATEWAY_DOMAIN" - -[core.debug] -insecure_skip_attestation = true +rpc_domain = "" [core.sync] enabled = false