cuda: infrastructure for whole-step CUDA graph capture

- Thread-local launch stream (xserv_cuda::stream): every kernel
  wrapper, cublasSetStream, and NCCL collective now launches on
  current_stream_raw() — the legacy null stream by default (behavior
  unchanged), or the capture stream installed via push_stream during
  graph capture. Capture is impossible on the legacy stream.
- Allocator retain mode: blocks freed inside a retain window are
  quarantined (RetainedBlocks) instead of pooled, so an instantiated
  graph keeps exclusive ownership of every intermediate buffer it
  references across replays.
- Capture mode GLOBAL -> THREAD_LOCAL: concurrent TP rank threads
  must not poison each other's captures with their own cudaMallocs.
- embedding_device_ids / rope_inplace_device_pos: variants reading
  token ids / positions from persistent device buffers, replacing the
  per-call host upload that a captured region cannot contain.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-06-12 20:12:37 +08:00
parent 2a92f268a9
commit 4088f49b7d
20 changed files with 191 additions and 69 deletions

View File

@@ -144,12 +144,12 @@ fn apply_causal_mask(scores: &Tensor, offset: usize) {
DType::F32 => launch_causal_mask_f32(
scores.data_ptr() as *mut c_void,
batch as i32, rows as i32, cols as i32, offset as i32,
std::ptr::null_mut(),
xserv_cuda::current_stream_raw(),
),
DType::BF16 => launch_causal_mask_bf16(
scores.data_ptr() as *mut c_void,
batch as i32, rows as i32, cols as i32, offset as i32,
std::ptr::null_mut(),
xserv_cuda::current_stream_raw(),
),
_ => panic!("unsupported dtype for causal mask"),
}
@@ -233,7 +233,7 @@ pub fn decode_attention(q: &Tensor, k: &Tensor, v: &Tensor) -> Tensor {
head_dim as i32,
scale,
1, // causal (always 1 for decode)
std::ptr::null_mut(),
xserv_cuda::current_stream_raw(),
);
}
@@ -295,7 +295,7 @@ pub fn flash_attention(q: &Tensor, k: &Tensor, v: &Tensor, causal: bool) -> Tens
head_dim as i32,
scale,
if causal { 1 } else { 0 },
std::ptr::null_mut(),
xserv_cuda::current_stream_raw(),
);
}
@@ -354,7 +354,7 @@ pub fn flash_attention_sinks(
scale,
1, // always causal
window_size as i32,
std::ptr::null_mut(),
xserv_cuda::current_stream_raw(),
);
}
@@ -409,7 +409,7 @@ pub fn paged_decode_attention(
head_dim as i32,
max_blocks_per_seq as i32,
scale,
std::ptr::null_mut(),
xserv_cuda::current_stream_raw(),
);
}
@@ -464,7 +464,7 @@ pub fn paged_decode_attention_sinks(
max_blocks_per_seq as i32,
scale,
window_size as i32,
std::ptr::null_mut(),
xserv_cuda::current_stream_raw(),
);
}