| | from transformers import PretrainedConfig |
| | from transformers.utils import logging |
| |
|
| |
|
| | logger = logging.get_logger(__name__) |
| |
|
| |
|
| | class D2CoderConfig(PretrainedConfig): |
| | r""" |
| | This is the configuration class to store the configuration of a [`D2LLM`]. It is used to instantiate a |
| | Qwen2 model according to the specified arguments, defining the model architecture. Instantiating a configuration |
| | with the defaults will yield a similar configuration to that of |
| | Qwen2-7B-beta [Qwen/Qwen2-7B-beta](https://huggingface.co/Qwen/Qwen2-7B-beta). |
| | |
| | Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the |
| | documentation from [`PretrainedConfig`] for more information. |
| | |
| | |
| | Args: |
| | vocab_size (`int`, *optional*, defaults to 151936): |
| | Vocabulary size of the Qwen2 model. Defines the number of different tokens that can be represented by the |
| | `inputs_ids` passed when calling [`D2LLM`] |
| | hidden_size (`int`, *optional*, defaults to 4096): |
| | Dimension of the hidden representations. |
| | intermediate_size (`int`, *optional*, defaults to 22016): |
| | Dimension of the MLP representations. |
| | num_hidden_layers (`int`, *optional*, defaults to 32): |
| | Number of hidden layers in the Transformer encoder. |
| | num_attention_heads (`int`, *optional*, defaults to 32): |
| | Number of attention heads for each attention layer in the Transformer encoder. |
| | num_key_value_heads (`int`, *optional*, defaults to 32): |
| | This is the number of key_value heads that should be used to implement Grouped Query Attention. If |
| | `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if |
| | `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When |
| | converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed |
| | by meanpooling all the original heads within that group. For more details checkout [this |
| | paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `32`. |
| | hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): |
| | The non-linear activation function (function or string) in the decoder. |
| | max_position_embeddings (`int`, *optional*, defaults to 32768): |
| | The maximum sequence length that this model might ever be used with. |
| | initializer_range (`float`, *optional*, defaults to 0.02): |
| | The standard deviation of the truncated_normal_initializer for initializing all weight matrices. |
| | rms_norm_eps (`float`, *optional*, defaults to 1e-06): |
| | The epsilon used by the rms normalization layers. |
| | use_cache (`bool`, *optional*, defaults to `True`): |
| | Whether or not the model should return the last key/values attentions (not used by all models). Only |
| | relevant if `config.is_decoder=True`. |
| | tie_word_embeddings (`bool`, *optional*, defaults to `False`): |
| | Whether the model's input and output word embeddings should be tied. |
| | rope_theta (`float`, *optional*, defaults to 10000.0): |
| | The base period of the RoPE embeddings. |
| | use_sliding_window (`bool`, *optional*, defaults to `False`): |
| | Whether to use sliding window attention. |
| | sliding_window (`int`, *optional*, defaults to 4096): |
| | Sliding window attention (SWA) window size. If not specified, will default to `4096`. |
| | max_window_layers (`int`, *optional*, defaults to 28): |
| | The number of layers that use SWA (Sliding Window Attention). The bottom layers use SWA while the top use full attention. |
| | attention_dropout (`float`, *optional*, defaults to 0.0): |
| | The dropout ratio for the attention probabilities. |
| | |
| | ```python |
| | >>> from transformers import Qwen2Model, Qwen2Config |
| | |
| | >>> # Initializing a Qwen2 style configuration |
| | >>> configuration = Qwen2Config() |
| | |
| | >>> # Initializing a model from the Qwen2-7B style configuration |
| | >>> model = Qwen2Model(configuration) |
| | |
| | >>> # Accessing the model configuration |
| | >>> configuration = model.config |
| | ```""" |
| |
|
| | model_type = "qwen2" |
| | keys_to_ignore_at_inference = ["past_key_values"] |
| |
|
| | def __init__( |
| | self, |
| | vocab_size=151936, |
| | hidden_size=4096, |
| | intermediate_size=22016, |
| | num_hidden_layers=32, |
| | num_attention_heads=32, |
| | num_key_value_heads=32, |
| | hidden_act="silu", |
| | max_position_embeddings=32768, |
| | initializer_range=0.02, |
| | rms_norm_eps=1e-6, |
| | use_cache=True, |
| | tie_word_embeddings=False, |
| | rope_theta=10000.0, |
| | use_sliding_window=False, |
| | sliding_window=4096, |
| | max_window_layers=28, |
| | attention_dropout=0.0, |
| | |
| | embedding_method="pma", |
| | inf_seq_length=1024, |
| | encoder_mode ="post_normal", |
| | num_encoder_layers =0, |
| | padding_side ="right", |
| | |
| | keep_max_layer=32, |
| | pma_num_heads=32, |
| | pma_ln=True, |
| | pma_norm=False, |
| | pma_norm_mode="post_normal", |
| | |
| | **kwargs, |
| | ): |
| | self.vocab_size = vocab_size |
| | self.max_position_embeddings = max_position_embeddings |
| | self.hidden_size = hidden_size |
| | self.intermediate_size = intermediate_size |
| | self.num_hidden_layers = num_hidden_layers |
| | self.num_attention_heads = num_attention_heads |
| | self.use_sliding_window = use_sliding_window |
| | self.sliding_window = sliding_window if use_sliding_window else None |
| | self.max_window_layers = max_window_layers |
| |
|
| | |
| | if num_key_value_heads is None: |
| | num_key_value_heads = num_attention_heads |
| |
|
| | self.num_key_value_heads = num_key_value_heads |
| | self.hidden_act = hidden_act |
| | self.initializer_range = initializer_range |
| | self.rms_norm_eps = rms_norm_eps |
| | self.use_cache = use_cache |
| | self.rope_theta = rope_theta |
| | self.attention_dropout = attention_dropout |
| |
|
| | self.embedding_method = config.embedding_method |
| | self.inf_seq_length = config.inf_seq_length |
| | self.encoder_mode = config.encoder_mode |
| | self.num_encoder_layers = config.num_encoder_layers |
| | self.padding_side = config.padding_side |
| |
|
| | self.keep_max_layer = config.keep_max_layer |
| | self.pma_num_heads = config.pma_num_heads |
| | self.pma_ln = config.pma_ln |
| | self.pma_norm = config.pma_norm |
| | self.pma_norm_mode = config.pma_norm_mode |
| |
|
| | super().__init__( |
| | tie_word_embeddings=tie_word_embeddings, |
| | **kwargs, |
| | ) |
| |
|
