Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion dstack/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 1 addition & 6 deletions dstack/gateway/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ pin-project.workspace = true
serde_json.workspace = true
rand.workspace = true
dstack-build-info.workspace = true
ra-rpc = { workspace = true, features = ["client", "rocket"] }
ra-rpc = { workspace = true, features = ["rocket"] }
dstack-gateway-rpc.workspace = true
certbot.workspace = true
bytes.workspace = true
Expand All @@ -41,7 +41,6 @@ smallvec.workspace = true
futures.workspace = true
cmd_lib.workspace = true
load_config.workspace = true
dstack-kms-rpc.workspace = true
ra-tls.workspace = true
dstack-guest-agent-rpc.workspace = true
http-client = { workspace = true, features = ["prpc"] }
Expand Down Expand Up @@ -72,10 +71,6 @@ ktls.workspace = true
libc.workspace = true
socket2.workspace = true

[[bin]]
name = "gen_debug_key"
path = "src/gen_debug_key.rs"

[dev-dependencies]
insta.workspace = true
tempfile.workspace = true
Expand Down
4 changes: 2 additions & 2 deletions dstack/gateway/docs/cluster-deployment.md
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,8 @@ This allows a simple deployment order:

### 2.5 Configuration File Examples

> **Note:** A non-empty `rpc_domain` makes the gateway request its RPC TLS key and certificate from the local dstack Guest Agent. Ensure `/var/run/dstack/dstack.sock` is available, or set `DSTACK_AGENT_ADDRESS` to another Guest Agent endpoint. Set `rpc_domain = ""` when supplying pre-generated certificates.

gateway-1.toml:

```toml
Expand All @@ -184,7 +186,6 @@ ca_certs = "/var/lib/gateway/certs/gateway-ca.cert"
mandatory = false

[core]
kms_url = "https://kms.demo.dstack.org"
rpc_domain = "rpc.gateway-1.demo.dstack.org"

Comment thread
kvinwang marked this conversation as resolved.
[core.admin]
Expand Down Expand Up @@ -240,7 +241,6 @@ ca_certs = "/var/lib/gateway/certs/gateway-ca.cert"
mandatory = false

[core]
kms_url = "https://kms.demo.dstack.org"
rpc_domain = "rpc.gateway-2.demo.dstack.org"

[core.sync]
Expand Down
2 changes: 0 additions & 2 deletions dstack/gateway/gateway.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ log_level = "info"
address = "127.0.0.1:8010"

[core]
kms_url = ""
# auto set soft ulimit to hard ulimit
set_ulimit = true
rpc_domain = ""
Expand Down Expand Up @@ -62,7 +61,6 @@ insecure_enable_debug_rpc = false
# gateway's own loopback on a port of their choosing, bypassing port_policy.
insecure_localhost_backend = false
insecure_skip_attestation = false
key_file = "debug_key.json"
address = "127.0.0.1:8012"

[core.wg]
Expand Down
4 changes: 0 additions & 4 deletions dstack/gateway/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -510,7 +510,6 @@ pub struct Config {
pub recycle: RecycleConfig,
pub set_ulimit: bool,
pub rpc_domain: String,
pub kms_url: String,
pub admin: AdminConfig,
/// Debug server configuration (separate port for debug RPCs)
pub debug: DebugConfig,
Expand Down Expand Up @@ -538,9 +537,6 @@ pub struct DebugConfig {
/// registered instance and so bypasses `port_policy` entirely.
#[serde(default)]
pub insecure_localhost_backend: bool,
/// Path to pre-generated debug key data file (JSON format containing key, quote, event_log, and vm_config)
#[serde(default)]
pub key_file: String,
}

#[derive(Debug, Clone, Deserialize)]
Expand Down
84 changes: 0 additions & 84 deletions dstack/gateway/src/gen_debug_key.rs

This file was deleted.

117 changes: 4 additions & 113 deletions dstack/gateway/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,16 @@
// SPDX-License-Identifier: Apache-2.0

use anyhow::{anyhow, Context, Result};
use base64::{engine::general_purpose::STANDARD, Engine as _};
use clap::Parser;
use config::{Config, TlsConfig};
use dstack_guest_agent_rpc::{dstack_guest_client::DstackGuestClient, GetTlsKeyArgs};
use dstack_kms_rpc::SignCertRequest;
use http_client::prpc::PrpcClient;
use ra_rpc::{client::RaClient, prpc_routes as prpc, rocket_helper::QuoteVerifier};
use ra_tls::rcgen::KeyPair;
use ra_tls::{
attestation::AttestationVerifier,
cert::{CertConfigV2, CertSigningRequestV2, Csr},
};
use ra_rpc::{prpc_routes as prpc, rocket_helper::QuoteVerifier};
use ra_tls::attestation::AttestationVerifier;
use rocket::{
fairing::AdHoc,
figment::{providers::Serialized, Figment},
};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use tracing::{info, warn};

Expand All @@ -41,18 +34,6 @@ mod pp;
mod proxy;
mod web_routes;

#[derive(Debug, Clone, Serialize, Deserialize)]
struct DebugKeyData {
/// Private key in PEM format
key_pem: String,
/// TDX quote in base64 format
quote_base64: String,
/// Event log in JSON string format
event_log: String,
/// VM config in JSON string format
vm_config: String,
}

#[global_allocator]
static ALLOCATOR: jemallocator::Jemalloc = jemallocator::Jemalloc;

Expand Down Expand Up @@ -99,13 +80,10 @@ async fn maybe_gen_certs(config: &Config, tls_config: &TlsConfig) -> Result<()>
}
}
}
match config.debug.insecure_skip_attestation {
true => gen_debug_certs(config, tls_config, alt_names).await,
false => gen_prod_certs(tls_config, alt_names).await,
}
gen_certs(tls_config, alt_names).await
}

async fn gen_prod_certs(tls_config: &TlsConfig, alt_names: Vec<String>) -> Result<()> {
async fn gen_certs(tls_config: &TlsConfig, alt_names: Vec<String>) -> Result<()> {
info!("Using dstack guest agent for certificate generation");
let agent_client = dstack_agent().context("Failed to create dstack client")?;

Comment thread
kvinwang marked this conversation as resolved.
Expand Down Expand Up @@ -134,93 +112,6 @@ async fn gen_prod_certs(tls_config: &TlsConfig, alt_names: Vec<String>) -> Resul
Ok(())
}

async fn gen_debug_certs(
config: &Config,
tls_config: &TlsConfig,
alt_names: Vec<String>,
) -> Result<()> {
let kms_url = config.kms_url.clone();
if kms_url.is_empty() {
info!("KMS URL is empty, skipping cert generation");
return Ok(());
}

// Check if debug key file is configured
if config.debug.key_file.is_empty() {
info!("Debug key file not configured, skipping cert generation");
return Ok(());
}

// Load pre-generated key pair and quote data from JSON file
info!("Loading debug key data from: {}", config.debug.key_file);
let ctx = "Failed to read debug key, run `cargo run --bin gen_debug_key -- <simulator_url>` to generate it";
let json_content = fs_err::read_to_string(&config.debug.key_file).context(ctx)?;
let debug_data: DebugKeyData =
serde_json::from_str(&json_content).context("Failed to parse debug key JSON")?;

let key_pem = debug_data.key_pem;
let quote_bin = STANDARD
.decode(&debug_data.quote_base64)
.context("Failed to decode quote from base64")?;
let event_log_json = debug_data.event_log;
let vm_config_json = debug_data.vm_config;

// Parse key pair
let key = KeyPair::from_pem(&key_pem).context("Failed to parse debug key")?;
let pubkey = key.public_key_der();

// Build CSR with attestation from debug quote
let attestation =
ra_tls::attestation::Attestation::from_tdx_quote(quote_bin, event_log_json.as_bytes())
.context("Failed to create attestation from debug quote")?
.into_versioned();

let csr = CertSigningRequestV2 {
confirm: "please sign cert:".to_string(),
pubkey,
config: CertConfigV2 {
org_name: None,
subject: "dstack-gateway".to_string(),
subject_alt_names: alt_names,
usage_server_auth: true,
usage_client_auth: true,
ext_quote: true,
ext_app_info: true,
not_before: None,
not_after: None,
},
attestation,
};
let signature = csr.signed_by(&key).context("Failed to sign CSR")?;

// Send CSR to KMS for signing
let kms_url = format!("{kms_url}/prpc");
info!("Sending CSR to KMS for signing: {kms_url}");
let kms_client = RaClient::new(kms_url, true).context("Failed to create kms client")?;
let kms_client = dstack_kms_rpc::kms_client::KmsClient::new(kms_client);
let sign_response = kms_client
.sign_cert(SignCertRequest {
api_version: 2,
csr: csr.to_vec(),
signature,
vm_config: vm_config_json.to_string(),
})
.await
.context("Failed to sign certificate via KMS")?;

let ca_cert = sign_response
.certificate_chain
.last()
.context("Empty certificate chain")?
.to_string();
let certs = sign_response.certificate_chain.join("\n");

write_cert(&tls_config.mutual.ca_certs, &ca_cert)?;
write_cert(&tls_config.certs, &certs)?;
write_cert(&tls_config.key, &key.serialize_pem())?;
Ok(())
}

fn write_cert(path: &str, cert: &str) -> Result<()> {
info!("Writing cert to file: {path}");
safe_write::safe_write_with_mode(path, cert, 0o600)?;
Expand Down
3 changes: 1 addition & 2 deletions dstack/gateway/test-run/cluster.sh
Original file line number Diff line number Diff line change
Expand Up @@ -160,8 +160,7 @@ ca_certs = "${abs_run_dir}/certs/gateway-ca.cert"
mandatory = false

[core]
kms_url = ""
rpc_domain = "gateway.test.local"
rpc_domain = ""

Comment thread
kvinwang marked this conversation as resolved.
[core.debug]
insecure_enable_debug_rpc = true
Expand Down
19 changes: 19 additions & 0 deletions dstack/gateway/test-run/e2e/Dockerfile.simulator
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network>
# SPDX-License-Identifier: Apache-2.0

FROM rust:1.92-bookworm AS builder
WORKDIR /src
COPY . .
RUN cargo build --manifest-path dstack/Cargo.toml --locked --release \
-p dstack-guest-agent-simulator

FROM debian:bookworm-slim
RUN apt-get update && \
apt-get install -y --no-install-recommends ca-certificates && \
rm -rf /var/lib/apt/lists/*
WORKDIR /opt/dstack-simulator
COPY --from=builder /src/dstack/target/release/dstack-simulator /usr/local/bin/dstack-simulator
COPY sdk/simulator/app-compose.json sdk/simulator/appkeys.json \
sdk/simulator/sys-config.json sdk/simulator/attestation.bin ./
COPY dstack/gateway/test-run/e2e/configs/simulator.toml ./simulator.toml
CMD ["dstack-simulator", "--config", "/opt/dstack-simulator/simulator.toml"]
9 changes: 9 additions & 0 deletions dstack/gateway/test-run/e2e/Dockerfile.simulator.dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network>
# SPDX-License-Identifier: Apache-2.0

.git
**/target
**/node_modules
**/__pycache__
**/.env
**/.env.*
1 change: 0 additions & 1 deletion dstack/gateway/test-run/e2e/configs/gateway-1.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ ca_certs = "/var/lib/gateway/certs/gateway-ca.cert"
mandatory = false

[core]
kms_url = ""
rpc_domain = "gateway-1"

[core.admin]
Expand Down
1 change: 0 additions & 1 deletion dstack/gateway/test-run/e2e/configs/gateway-2.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ ca_certs = "/var/lib/gateway/certs/gateway-ca.cert"
mandatory = false

[core]
kms_url = ""
rpc_domain = "gateway-2"

[core.admin]
Expand Down
1 change: 0 additions & 1 deletion dstack/gateway/test-run/e2e/configs/gateway-3.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ ca_certs = "/var/lib/gateway/certs/gateway-ca.cert"
mandatory = false

[core]
kms_url = ""
rpc_domain = "gateway-3"

[core.admin]
Expand Down
Loading
Loading