[doc] add doc

2024-03-11 10:56:51 +08:00 · 2024-03-11 10:56:51 +08:00 · 633e95b301
parent 9dec66fad6
commit 633e95b301
1 changed files with 1 additions and 0 deletions
--- a/colossalai/inference/README.md
+++ b/colossalai/inference/README.md
@ -94,6 +94,7 @@ inference_config = InferenceConfig(
                max_batch_size=4,
                max_input_len=1024,
                max_output_len=512,
+                use_cuda_graph=False, # Turn on if you want to use CUDA Graph to accelerate inference
            )

 # Step 3: create an engine with model and config