feat: support passing environment variables to the enclave

sangwa · sangwa · commit 31a8d4239b08 · 2025-07-16T19:23:50.000+03:00
This is useful when running in Kubernetes to pass configuration from ConfigMaps and Secrets.
The new `env` key in the manifest specifies the list of names of environment variables that
should be passed to the enclave.
diff --git a/enclaver/src/bin/odyn/launcher.rs b/enclaver/src/bin/odyn/launcher.rs
@@ -3,6 +3,7 @@ use log::debug;
 use nix::sys::signal::Signal;
 use nix::sys::wait::{WaitPidFlag, WaitStatus};
 use nix::unistd::Pid;
+use std::collections::HashMap;
 use std::ffi::OsString;
 use std::os::unix::process::CommandExt;
 use std::process::Command;
@@ -28,12 +29,17 @@ impl std::fmt::Display for ExitStatus {
 }
 
 // runs the child and reaps all of its children as well
-pub fn run_child(argv: &[OsString], creds: &Credentials) -> Result<ExitStatus> {
+pub fn run_child(
+    argv: &[OsString],
+    creds: &Credentials,
+    env: &HashMap<String, String>,
+) -> Result<ExitStatus> {
     // Don't use tokio::process::Command because it wants to reap the process.
     // However we need to run waitpid() ourselves to reap the zombies and it'll
     // end up picking up the spawned child as well.
     let child = Command::new(&argv[0])
         .args(&argv[1..])
+        .envs(env)
         .uid(creds.uid)
         .gid(creds.gid)
         .process_group(0)
@@ -46,8 +52,12 @@ pub fn run_child(argv: &[OsString], creds: &Credentials) -> Result<ExitStatus> {
 }
 
 // runs the child and reaps all of its children as well
-pub fn start_child(argv: Vec<OsString>, creds: Credentials) -> JoinHandle<Result<ExitStatus>> {
-    tokio::task::spawn_blocking(move || run_child(&argv, &creds))
+pub fn start_child(
+    argv: Vec<OsString>,
+    creds: Credentials,
+    env: HashMap<String, String>,
+) -> JoinHandle<Result<ExitStatus>> {
+    tokio::task::spawn_blocking(move || run_child(&argv, &creds, &env))
 }
 
 // Reap processes until a process with sentinel pid exits.
diff --git a/enclaver/src/bin/odyn/main.rs b/enclaver/src/bin/odyn/main.rs
@@ -9,13 +9,15 @@ pub mod ingress;
 pub mod kms_proxy;
 pub mod launcher;
 
-use anyhow::Result;
+use anyhow::{anyhow, Context, Result};
 use clap::Parser;
-use log::{error, info};
+use log::{error, info, warn};
+use std::collections::HashMap;
 use std::ffi::OsString;
 use std::sync::Arc;
+use tokio::time;
 
-use enclaver::constants::{APP_LOG_PORT, STATUS_PORT};
+use enclaver::constants::{APP_LOG_PORT, ENV_SYNC_PORT, STATUS_PORT};
 use enclaver::nsm::Nsm;
 
 use api::ApiService;
@@ -25,6 +27,8 @@ use egress::EgressService;
 use ingress::IngressService;
 use kms_proxy::KmsProxyService;
 
+const ENV_SYNC_TIMEOUT: time::Duration = time::Duration::from_secs(10);
+
 #[derive(Parser)]
 struct CliArgs {
     #[clap(long = "no-bootstrap", action)]
@@ -57,11 +61,12 @@ async fn launch(args: &CliArgs) -> Result<launcher::ExitStatus> {
     let ingress = IngressService::start(&config)?;
     let kms_proxy = KmsProxyService::start(config.clone(), nsm.clone()).await?;
     let api = ApiService::start(&config, nsm.clone()).await?;
+    let env = sync_environment(&config).await?;
 
     let creds = launcher::Credentials { uid: 0, gid: 0 };
 
     info!("Starting {:?}", args.entrypoint);
-    let exit_status = launcher::start_child(args.entrypoint.clone(), creds).await??;
+    let exit_status = launcher::start_child(args.entrypoint.clone(), creds, env).await??;
     info!("Entrypoint {}", exit_status);
 
     api.stop().await;
@@ -72,6 +77,69 @@ async fn launch(args: &CliArgs) -> Result<launcher::ExitStatus> {
     Ok(exit_status)
 }
 
+async fn sync_environment(config: &Configuration) -> Result<HashMap<String, String>> {
+    use futures::stream::StreamExt;
+    use tokio::io::AsyncReadExt;
+
+    let mut env: HashMap<String, String> = HashMap::new();
+
+    if let Some(ref keys) = config.manifest.env {
+        if !keys.is_empty() {
+            info!("Starting the environment sync");
+
+            let mut incoming = enclaver::vsock::serve(ENV_SYNC_PORT)?;
+
+            match time::timeout(ENV_SYNC_TIMEOUT, incoming.next()).await {
+                Ok(Some(mut sock)) => {
+                    let mut env_buf: Vec<u8> = Vec::new();
+                    let mut read_buf = vec![0u8; 1024];
+                    const ARG_MAX: usize = 128 * 1024;
+
+                    loop {
+                        match time::timeout(ENV_SYNC_TIMEOUT, sock.read(&mut read_buf)).await {
+                            Ok(Ok(0)) => break,
+                            Ok(Ok(n)) => {
+                                if env_buf.len() + n > ARG_MAX {
+                                    return Err(anyhow!("Maximum environment size exceeded"));
+                                }
+                                env_buf.extend_from_slice(&read_buf[..n]);
+                            }
+                            Ok(Err(err)) => {
+                                return Err(anyhow!("Error reading environment: {:#}", err))
+                            }
+                            Err(_) => return Err(anyhow!("Timed out while reading environment")),
+                        }
+                    }
+
+                    let mut synced_env: HashMap<String, String> = serde_json::from_slice(&env_buf)
+                        .context("Failed to parse the synced environment")?;
+
+                    for k in keys.iter() {
+                        if let Some(v) = synced_env.remove(k) {
+                            info!("Syncing environment variable: {}", k);
+                            env.insert(k.clone(), v);
+                        }
+                    }
+                }
+                Ok(None) => {
+                    return Err(anyhow!(
+                        "Failed to accept environment sync vsock connection"
+                    ));
+                }
+                Err(_) => {
+                    return Err(anyhow!("Timed out while waiting for environment sync"));
+                }
+            }
+
+            info!("Environment sync complete");
+        } else {
+            warn!("The list of environment keys in the manifest is empty, skipping the sync");
+        }
+    }
+
+    Ok(env)
+}
+
 async fn run(args: &CliArgs) -> Result<()> {
     // Start the status and logs listeners ASAP so that if we fail to
     // initialize, we can communicate the status and stream the logs
diff --git a/enclaver/src/constants.rs b/enclaver/src/constants.rs
@@ -13,6 +13,7 @@ pub const RELEASE_BUNDLE_DIR: &str = "/enclave";
 pub const STATUS_PORT: u32 = 17000;
 pub const APP_LOG_PORT: u32 = 17001;
 pub const HTTP_EGRESS_VSOCK_PORT: u32 = 17002;
+pub const ENV_SYNC_PORT: u32 = 17003;
 
 // Default TCP Port that the egress proxy listens on inside the enclave, if not
 // specified in the manifest.
diff --git a/enclaver/src/manifest.rs b/enclaver/src/manifest.rs
@@ -23,6 +23,7 @@ pub struct Manifest {
     pub defaults: Option<Defaults>,
     pub kms_proxy: Option<KmsProxy>,
     pub api: Option<Api>,
+    pub env: Option<Vec<String>>,
 }
 
 #[derive(Debug, Eq, PartialEq, Serialize, Deserialize)]
diff --git a/enclaver/src/run.rs b/enclaver/src/run.rs
@@ -1,16 +1,19 @@
 use crate::constants::{
-    APP_LOG_PORT, EIF_FILE_NAME, HTTP_EGRESS_VSOCK_PORT, MANIFEST_FILE_NAME, RELEASE_BUNDLE_DIR,
-    STATUS_PORT,
+    APP_LOG_PORT, EIF_FILE_NAME, ENV_SYNC_PORT, HTTP_EGRESS_VSOCK_PORT, MANIFEST_FILE_NAME,
+    RELEASE_BUNDLE_DIR, STATUS_PORT,
 };
 use crate::manifest::{load_manifest, Defaults, Manifest};
 use crate::utils;
 use anyhow::{anyhow, Result};
 use futures_util::stream::StreamExt;
 use log::{debug, error, info};
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::env;
 use std::path::PathBuf;
 use std::time::Duration;
 use tokio::fs::File;
+use tokio::io::AsyncWriteExt;
 use tokio_util::codec::{FramedRead, LinesCodec};
 use tokio_util::sync::CancellationToken;
 use tokio_vsock::VsockStream;
@@ -19,6 +22,7 @@ use crate::nitro_cli::{EnclaveInfo, NitroCLI, RunEnclaveArgs};
 use crate::proxy::egress_http::HostHttpProxy;
 use crate::proxy::ingress::HostProxy;
 
+const ENV_VSOCK_RETRY_INTERVAL: Duration = Duration::from_millis(250);
 const LOG_VSOCK_RETRY_INTERVAL: Duration = Duration::from_millis(250);
 const STATUS_VSOCK_RETRY_INTERVAL: Duration = Duration::from_millis(250);
 const STATUS_VSOCK_RETRY_LIMIT: i32 = 100;
@@ -147,6 +151,8 @@ impl Enclave {
             self.attach_debug_console(&enclave_info.id).await?;
         }
 
+        self.start_env_sync(enclave_info.cid)?;
+
         self.start_odyn_log_stream(enclave_info.cid)?;
 
         self.start_ingress_proxies(enclave_info.cid).await?;
@@ -216,6 +222,38 @@ impl Enclave {
         Ok(())
     }
 
+    fn start_env_sync(&mut self, cid: u32) -> Result<()> {
+        if let Some(ref keys) = self.manifest.env {
+            if !keys.is_empty() {
+                let vars = keys
+                    .iter()
+                    .filter_map(|k| env::var(k).ok().map(|v| (k.clone(), v)))
+                    .collect::<HashMap<_, _>>();
+                let env = serde_json::to_string(&vars)?;
+
+                self.tasks
+                    .push(utils::spawn!("sync environment", async move {
+                        info!("waiting for enclave to boot to sync environment");
+                        let mut conn = loop {
+                            match VsockStream::connect(cid, ENV_SYNC_PORT).await {
+                                Ok(conn) => break conn,
+                                // TODO: improve the polling frequency / backoff / timeout
+                                Err(_) => tokio::time::sleep(ENV_VSOCK_RETRY_INTERVAL).await,
+                            }
+                        };
+
+                        info!("connected to enclave, starting environment sync");
+                        if let Err(err) = conn.write_all(env.as_bytes()).await {
+                            error!("error sending environment to enclave: {err}");
+                        }
+                        info!("environment sync complete");
+                    })?);
+            }
+        }
+
+        Ok(())
+    }
+
     fn start_odyn_log_stream(&mut self, cid: u32) -> Result<()> {
         self.tasks
             .push(utils::spawn!("odyn log stream", async move {

Original file line number	Diff line number	Diff line change
`@@ -23,6 +23,7 @@ pub struct Manifest {`
`23`	`23`	`pub defaults: Option<Defaults>,`
`24`	`24`	`pub kms_proxy: Option<KmsProxy>,`
`25`	`25`	`pub api: Option<Api>,`
	`26`	`+ pub env: Option<Vec<String>>,`
`26`	`27`	`}`
`27`	`28`
`28`	`29`	`#[derive(Debug, Eq, PartialEq, Serialize, Deserialize)]`