Seed

fastdeploy/config.py

@@ -116,6 +116,7 @@ def __init__(
         self.enable_redundant_experts = False
         self.redundant_experts_num = 0
         self.quantization = None
+        self.seed = 0
         for key, value in args.items():
             if hasattr(self, key):
                 setattr(self, key, value)

fastdeploy/engine/args_utils.py

@@ -313,6 +313,11 @@ class EngineArgs:
     Must be explicitly enabled via the `--enable-logprob` startup parameter to output logprob values.
     """
 
+    seed: Optional[int] = None
+    """
+    Random seed to use for initialization. If not set, a random seed is used.
+    """
+
     def __post_init__(self):
         """
         Post-initialization processing to set default tokenizer if not provided.
@@ -464,7 +469,12 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
             default=EngineArgs.enable_logprob,
             help="Enable output of token-level log probabilities.",
         )
-
+        model_group.add_argument(
+            "--seed",
+            type=int,
+            default=None,
+            help="Random seed for initialization. If not specified, a random seed will be used.",
+        )
         # Parallel processing parameters group
         parallel_group = parser.add_argument_group("Parallel Configuration")
         parallel_group.add_argument(

fastdeploy/engine/sampling_params.py

@@ -15,10 +15,20 @@
 """
 
 from __future__ import annotations
-
 import random
+from functools import cached_property
 from dataclasses import dataclass, fields
 from typing import Any, List, Optional, Union
+from enum import Enum, IntEnum
+
+
+_SAMPLING_EPS = 1e-5
+_MAX_TEMP = 1e-2
+
+class SamplingType(IntEnum):
+    GREEDY = 0
+    RANDOM = 1
+    RANDOM_SEED = 2
 
 
 @dataclass
@@ -109,50 +119,49 @@ def from_dict(cls, req_dict: dict[str, Any]) -> SamplingParams:
         )
 
     @classmethod
-    def from_optional(
-        cls,
-        n,
-        best_of,
-        presence_penalty,
-        frequency_penalty,
-        repetition_penalty,
-        temperature,
-        top_p,
-        top_k,
-        min_p,
-        seed=None,
-        stop=None,
-        stop_token_ids=None,
-        max_tokens=None,
-        reasoning_max_tokens=None,
-        min_tokens=1,
-        logprobs=None,
-        bad_words=None,
-    ) -> SamplingParams:
+    def from_optional(cls,
+                      n,
+                      best_of,
+                      presence_penalty,
+                      frequency_penalty,
+                      repetition_penalty,
+                      temperature,
+                      top_p,
+                      top_k,
+                      min_p,
+                      seed=None,
+                      stop=None,
+                      stop_token_ids=None,
+                      max_tokens=None,
+                      reasoning_max_tokens=None,
+                      min_tokens=1,
+                      logprobs=None,
+                      bad_words=None) -> "SamplingParams":
         """Create instance from command line arguments"""
-        return cls(
-            n=1 if n is None else n,
-            best_of=best_of,
-            presence_penalty=(presence_penalty if presence_penalty is not None else 0.0),
-            frequency_penalty=(frequency_penalty if frequency_penalty is not None else 0.0),
-            repetition_penalty=(repetition_penalty if repetition_penalty is not None else 1.0),
-            temperature=temperature if temperature is not None else 1.0,
-            top_p=top_p,
-            top_k=top_k if top_k is not None else 0,
-            min_p=min_p if min_p is not None else 0.0,
-            seed=seed,
-            stop=stop,
-            stop_token_ids=stop_token_ids,
-            max_tokens=max_tokens if max_tokens is not None else 8192,
-            reasoning_max_tokens=reasoning_max_tokens,
-            min_tokens=min_tokens,
-            logprobs=logprobs,
-            bad_words=bad_words,
-        )
+        return cls(n=1 if n is None else n,
+                   best_of=best_of,
+                   presence_penalty=presence_penalty
+                   if presence_penalty is not None else 0.0,
+                   frequency_penalty=frequency_penalty
+                   if frequency_penalty is not None else 0.0,
+                   repetition_penalty=repetition_penalty
+                   if repetition_penalty is not None else 1.0,
+                   temperature=temperature if temperature is not None else 1.0,
+                   top_p=top_p,
+                   top_k=top_k if top_k is not None else 0,
+                   min_p=min_p if min_p is not None else 0.0,
+                   seed=seed,
+                   stop=stop,
+                   stop_token_ids=stop_token_ids,
+                   max_tokens=max_tokens if max_tokens is not None else 8192,
+                   reasoning_max_tokens=reasoning_max_tokens,
+                   min_tokens=min_tokens,
+                   logprobs=logprobs,
+                   bad_words=bad_words)
 
     def __post_init__(self):
-        if self.seed is None:
-            self.seed = random.randint(0, 922337203685477580)
+        if self.seed == -1:
+            self.seed = None
         if self.max_tokens is not None and self.reasoning_max_tokens is None:
             self.reasoning_max_tokens = max(int(self.max_tokens * 0.8), 1)
         self._verify_args()
@@ -176,7 +185,8 @@ def _verify_args(self) -> None:
         if self.top_k < -1:
             raise ValueError(f"top_k must be 0 (disable), or at least 1, " f"got {self.top_k}.")
         if not isinstance(self.top_k, int):
-            raise TypeError(f"top_k must be an integer, got {type(self.top_k).__name__}")
+            raise TypeError(
+                f"top_k must be an integer, got {type(self.top_k).__name__}")
         if not 0.0 <= self.min_p <= 1.0:
             raise ValueError("min_p must be in [0,1],got f{self.min_p}")
 
@@ -195,10 +205,16 @@ def _verify_args(self) -> None:
         if self.logprobs is not None and self.logprobs < 0:
             raise ValueError(f"logprobs must be non-negative, got {self.logprobs}.")
         if self.logprobs is not None and self.logprobs > 20:
-            raise ValueError("Invalid value for 'top_logprobs': must be less than or equal to 20.")
-
-        if not 0 <= self.seed <= 922337203685477580:
-            raise ValueError("seed must be in [0, 922337203685477580], got " f"{self.seed}.")
+            raise ValueError(
+                "Invalid value for 'top_logprobs': must be less than or equal to 20.")
+
+    @cached_property
+    def sampling_type(self)->SamplingType:
+        if self.temperature < _SAMPLING_EPS:
+            return SamplingType.GREEDY
+        if self.seed is not None:
+            return SamplingType.RANDOM_SEED
+        return SamplingType.RANDOM
 
     def update_from_tokenizer(self, tokenizer):
         """

fastdeploy/model_executor/layers/sample/meta_data.py

@@ -27,13 +27,11 @@ class SamplingMetadata:
     """
 
     temperature: paddle.Tensor
-
     pre_token_ids: paddle.Tensor
     eos_token_ids: paddle.Tensor
     frequency_penalties: paddle.Tensor
     presence_penalties: paddle.Tensor
     repetition_penalties: paddle.Tensor
-
     min_dec_lens: paddle.Tensor
 
     bad_words_token_ids: paddle.Tensor

fastdeploy/utils.py

@@ -29,6 +29,8 @@
 from pathlib import Path
 from typing import Literal, TypeVar, Union
 
+import numpy as np
+import paddle
 import requests
 import yaml
 from aistudio_sdk.snapshot_download import snapshot_download as aistudio_download
@@ -291,6 +293,13 @@ def extract_tar(tar_path, output_dir):
         raise RuntimeError(f"Extraction failed: {e!s}")
 
 
+def set_random_seed(seed: int) -> None:
+    if seed is not None:
+        random.seed(seed)
+        np.random.seed(seed)
+        paddle.seed(seed)
+
+
 def download_model(url, output_dir, temp_tar):
     """
     下载模型，并将其解压到指定目录。

fastdeploy/worker/gpu_model_runner.py

@@ -127,9 +127,7 @@ def __init__(
         # Initialize share inputs
         self._init_share_inputs(self.parallel_config.max_num_seqs)
         self.infer_seed_increment = paddle.full(
-            shape=[self.parallel_config.max_num_seqs, 1],
-            fill_value=4,
-            dtype="int64",
+            shape=[self.parallel_config.max_num_seqs, 1], fill_value=4, dtype="int64"
         )
         self.restore_chunked_prefill_request = dict()
 
@@ -269,6 +267,7 @@ def insert_tasks_v1(self, req_dicts: List[Request]):
             self.share_inputs["first_token_ids"][idx : idx + 1] = self.share_inputs["input_ids"][idx : idx + 1, :1]
             self.share_inputs["ori_seq_lens_encoder"][idx : idx + 1] = length
 
+            print("seed", request.get("seed"))
             if request.get("seed") is not None:
                 self.share_inputs["infer_seed"][idx : idx + 1] = request.get("seed")
 
@@ -509,7 +508,7 @@ def _init_share_inputs(self, max_num_seqs: int):
         """
         Initialize all share buffers for model inputs.
         """
-        self.MAX_INFER_SEED = 9223372036854775806
+
         self.share_inputs = {}
 
         self.share_inputs["pre_ids"] = paddle.full(
@@ -952,9 +951,10 @@ def _dummy_run(
                     ids_remove_padding=self.share_inputs["ids_remove_padding"],
                     forward_meta=self.forward_meta,
                 )
+                logits = self.model.compute_logits(model_output)
 
                 hidden_states = rebuild_padding(
-                    model_output,
+                    logits,
                     self.share_inputs["cum_offsets"],
                     self.share_inputs["seq_lens_this_time"],
                     self.share_inputs["seq_lens_decoder"],

fastdeploy/worker/gpu_worker.py

@@ -26,7 +26,7 @@
 from fastdeploy.config import FDConfig
 from fastdeploy.engine.request import Request
 from fastdeploy.platforms import current_platform
-from fastdeploy.utils import get_logger
+from fastdeploy.utils import get_logger, set_random_seed
 from fastdeploy.worker.gpu_model_runner import GPUModelRunner
 from fastdeploy.worker.output import ModelRunnerOutput
 from fastdeploy.worker.worker_base import WorkerBase
@@ -69,6 +69,7 @@ def init_device(self):
         else:
             raise RuntimeError(f"Not support device type: {self.device_config.device}")
 
+        set_random_seed(self.fd_config.model_config.seed)
         # Construct model runner
         self.model_runner: GPUModelRunner = GPUModelRunner(
             fd_config=self.fd_config,
@@ -123,6 +124,7 @@ def determine_available_memory(self) -> int:
 
         # 2. Profile run
         self.model_runner.profile_run()
+        set_random_seed(self.fd_config.model_config.seed)
 
         # 3. Statistical memory information
         paddle_reserved_mem_after_run = paddle.device.cuda.max_memory_reserved(local_rank)