diff --git a/fastdeploy/config.py b/fastdeploy/config.py index c8428d1f974..0afb57c4b7c 100644 --- a/fastdeploy/config.py +++ b/fastdeploy/config.py @@ -116,6 +116,7 @@ def __init__( self.enable_redundant_experts = False self.redundant_experts_num = 0 self.quantization = None + self.seed = 0 for key, value in args.items(): if hasattr(self, key): setattr(self, key, value) diff --git a/fastdeploy/engine/args_utils.py b/fastdeploy/engine/args_utils.py index 84e4b5533ab..cff949e2747 100644 --- a/fastdeploy/engine/args_utils.py +++ b/fastdeploy/engine/args_utils.py @@ -313,6 +313,11 @@ class EngineArgs: Must be explicitly enabled via the `--enable-logprob` startup parameter to output logprob values. """ + seed: Optional[int] = None + """ + Random seed to use for initialization. If not set, a random seed is used. + """ + def __post_init__(self): """ Post-initialization processing to set default tokenizer if not provided. @@ -464,7 +469,12 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: default=EngineArgs.enable_logprob, help="Enable output of token-level log probabilities.", ) - + model_group.add_argument( + "--seed", + type=int, + default=None, + help="Random seed for initialization. If not specified, a random seed will be used.", + ) # Parallel processing parameters group parallel_group = parser.add_argument_group("Parallel Configuration") parallel_group.add_argument( diff --git a/fastdeploy/engine/sampling_params.py b/fastdeploy/engine/sampling_params.py index 91babf7a86b..521a64c842e 100644 --- a/fastdeploy/engine/sampling_params.py +++ b/fastdeploy/engine/sampling_params.py @@ -15,10 +15,20 @@ """ from __future__ import annotations - import random +from functools import cached_property from dataclasses import dataclass, fields from typing import Any, List, Optional, Union +from enum import Enum, IntEnum + + +_SAMPLING_EPS = 1e-5 +_MAX_TEMP = 1e-2 + +class SamplingType(IntEnum): + GREEDY = 0 + RANDOM = 1 + RANDOM_SEED = 2 @dataclass @@ -109,50 +119,49 @@ def from_dict(cls, req_dict: dict[str, Any]) -> SamplingParams: ) @classmethod - def from_optional( - cls, - n, - best_of, - presence_penalty, - frequency_penalty, - repetition_penalty, - temperature, - top_p, - top_k, - min_p, - seed=None, - stop=None, - stop_token_ids=None, - max_tokens=None, - reasoning_max_tokens=None, - min_tokens=1, - logprobs=None, - bad_words=None, - ) -> SamplingParams: + def from_optional(cls, + n, + best_of, + presence_penalty, + frequency_penalty, + repetition_penalty, + temperature, + top_p, + top_k, + min_p, + seed=None, + stop=None, + stop_token_ids=None, + max_tokens=None, + reasoning_max_tokens=None, + min_tokens=1, + logprobs=None, + bad_words=None) -> "SamplingParams": """Create instance from command line arguments""" - return cls( - n=1 if n is None else n, - best_of=best_of, - presence_penalty=(presence_penalty if presence_penalty is not None else 0.0), - frequency_penalty=(frequency_penalty if frequency_penalty is not None else 0.0), - repetition_penalty=(repetition_penalty if repetition_penalty is not None else 1.0), - temperature=temperature if temperature is not None else 1.0, - top_p=top_p, - top_k=top_k if top_k is not None else 0, - min_p=min_p if min_p is not None else 0.0, - seed=seed, - stop=stop, - stop_token_ids=stop_token_ids, - max_tokens=max_tokens if max_tokens is not None else 8192, - reasoning_max_tokens=reasoning_max_tokens, - min_tokens=min_tokens, - logprobs=logprobs, - bad_words=bad_words, - ) + return cls(n=1 if n is None else n, + best_of=best_of, + presence_penalty=presence_penalty + if presence_penalty is not None else 0.0, + frequency_penalty=frequency_penalty + if frequency_penalty is not None else 0.0, + repetition_penalty=repetition_penalty + if repetition_penalty is not None else 1.0, + temperature=temperature if temperature is not None else 1.0, + top_p=top_p, + top_k=top_k if top_k is not None else 0, + min_p=min_p if min_p is not None else 0.0, + seed=seed, + stop=stop, + stop_token_ids=stop_token_ids, + max_tokens=max_tokens if max_tokens is not None else 8192, + reasoning_max_tokens=reasoning_max_tokens, + min_tokens=min_tokens, + logprobs=logprobs, + bad_words=bad_words) def __post_init__(self): - if self.seed is None: - self.seed = random.randint(0, 922337203685477580) + if self.seed == -1: + self.seed = None if self.max_tokens is not None and self.reasoning_max_tokens is None: self.reasoning_max_tokens = max(int(self.max_tokens * 0.8), 1) self._verify_args() @@ -176,7 +185,8 @@ def _verify_args(self) -> None: if self.top_k < -1: raise ValueError(f"top_k must be 0 (disable), or at least 1, " f"got {self.top_k}.") if not isinstance(self.top_k, int): - raise TypeError(f"top_k must be an integer, got {type(self.top_k).__name__}") + raise TypeError( + f"top_k must be an integer, got {type(self.top_k).__name__}") if not 0.0 <= self.min_p <= 1.0: raise ValueError("min_p must be in [0,1],got f{self.min_p}") @@ -195,10 +205,16 @@ def _verify_args(self) -> None: if self.logprobs is not None and self.logprobs < 0: raise ValueError(f"logprobs must be non-negative, got {self.logprobs}.") if self.logprobs is not None and self.logprobs > 20: - raise ValueError("Invalid value for 'top_logprobs': must be less than or equal to 20.") - - if not 0 <= self.seed <= 922337203685477580: - raise ValueError("seed must be in [0, 922337203685477580], got " f"{self.seed}.") + raise ValueError( + "Invalid value for 'top_logprobs': must be less than or equal to 20.") + + @cached_property + def sampling_type(self)->SamplingType: + if self.temperature < _SAMPLING_EPS: + return SamplingType.GREEDY + if self.seed is not None: + return SamplingType.RANDOM_SEED + return SamplingType.RANDOM def update_from_tokenizer(self, tokenizer): """ diff --git a/fastdeploy/model_executor/layers/sample/meta_data.py b/fastdeploy/model_executor/layers/sample/meta_data.py index 69b2e3e1987..f8781de77c4 100644 --- a/fastdeploy/model_executor/layers/sample/meta_data.py +++ b/fastdeploy/model_executor/layers/sample/meta_data.py @@ -27,13 +27,11 @@ class SamplingMetadata: """ temperature: paddle.Tensor - pre_token_ids: paddle.Tensor eos_token_ids: paddle.Tensor frequency_penalties: paddle.Tensor presence_penalties: paddle.Tensor repetition_penalties: paddle.Tensor - min_dec_lens: paddle.Tensor bad_words_token_ids: paddle.Tensor diff --git a/fastdeploy/utils.py b/fastdeploy/utils.py index 0ceba665a20..29328fa4e03 100644 --- a/fastdeploy/utils.py +++ b/fastdeploy/utils.py @@ -29,6 +29,8 @@ from pathlib import Path from typing import Literal, TypeVar, Union +import numpy as np +import paddle import requests import yaml from aistudio_sdk.snapshot_download import snapshot_download as aistudio_download @@ -291,6 +293,13 @@ def extract_tar(tar_path, output_dir): raise RuntimeError(f"Extraction failed: {e!s}") +def set_random_seed(seed: int) -> None: + if seed is not None: + random.seed(seed) + np.random.seed(seed) + paddle.seed(seed) + + def download_model(url, output_dir, temp_tar): """ 下载模型，并将其解压到指定目录。 diff --git a/fastdeploy/worker/gpu_model_runner.py b/fastdeploy/worker/gpu_model_runner.py index 4ec2411ec44..03b5cd69ef7 100644 --- a/fastdeploy/worker/gpu_model_runner.py +++ b/fastdeploy/worker/gpu_model_runner.py @@ -127,9 +127,7 @@ def __init__( # Initialize share inputs self._init_share_inputs(self.parallel_config.max_num_seqs) self.infer_seed_increment = paddle.full( - shape=[self.parallel_config.max_num_seqs, 1], - fill_value=4, - dtype="int64", + shape=[self.parallel_config.max_num_seqs, 1], fill_value=4, dtype="int64" ) self.restore_chunked_prefill_request = dict() @@ -269,6 +267,7 @@ def insert_tasks_v1(self, req_dicts: List[Request]): self.share_inputs["first_token_ids"][idx : idx + 1] = self.share_inputs["input_ids"][idx : idx + 1, :1] self.share_inputs["ori_seq_lens_encoder"][idx : idx + 1] = length + print("seed", request.get("seed")) if request.get("seed") is not None: self.share_inputs["infer_seed"][idx : idx + 1] = request.get("seed") @@ -509,7 +508,7 @@ def _init_share_inputs(self, max_num_seqs: int): """ Initialize all share buffers for model inputs. """ - self.MAX_INFER_SEED = 9223372036854775806 + self.share_inputs = {} self.share_inputs["pre_ids"] = paddle.full( @@ -952,9 +951,10 @@ def _dummy_run( ids_remove_padding=self.share_inputs["ids_remove_padding"], forward_meta=self.forward_meta, ) + logits = self.model.compute_logits(model_output) hidden_states = rebuild_padding( - model_output, + logits, self.share_inputs["cum_offsets"], self.share_inputs["seq_lens_this_time"], self.share_inputs["seq_lens_decoder"], diff --git a/fastdeploy/worker/gpu_worker.py b/fastdeploy/worker/gpu_worker.py index 084b4f0f2d8..c1c8552444a 100644 --- a/fastdeploy/worker/gpu_worker.py +++ b/fastdeploy/worker/gpu_worker.py @@ -26,7 +26,7 @@ from fastdeploy.config import FDConfig from fastdeploy.engine.request import Request from fastdeploy.platforms import current_platform -from fastdeploy.utils import get_logger +from fastdeploy.utils import get_logger, set_random_seed from fastdeploy.worker.gpu_model_runner import GPUModelRunner from fastdeploy.worker.output import ModelRunnerOutput from fastdeploy.worker.worker_base import WorkerBase @@ -69,6 +69,7 @@ def init_device(self): else: raise RuntimeError(f"Not support device type: {self.device_config.device}") + set_random_seed(self.fd_config.model_config.seed) # Construct model runner self.model_runner: GPUModelRunner = GPUModelRunner( fd_config=self.fd_config, @@ -123,6 +124,7 @@ def determine_available_memory(self) -> int: # 2. Profile run self.model_runner.profile_run() + set_random_seed(self.fd_config.model_config.seed) # 3. Statistical memory information paddle_reserved_mem_after_run = paddle.device.cuda.max_memory_reserved(local_rank)