Skip to content

configuration_utils

multimolecule.models.configuration_utils

HeadConfig

Bases: BaseHeadConfig

Configuration class for a prediction head.

参数：

名称 类型 描述 默认

num_labels

Number of labels to use in the last layer added to the model, typically for a classification task.

Head should look for Config.num_labels if is None.

 必需

problem_type

Problem type for XxxForYyyPrediction models. Can be one of "binary", "regression", "multiclass" or "multilabel".

Head should look for Config.problem_type if is None.

 必需

hidden_size

Dimensionality of the encoder layers and the pooler layer.

Head should look for Config.hidden_size if is None.

 必需

dropout

The dropout ratio for the hidden states.

 必需

transform

The transform operation applied to hidden states.

 必需

transform_act

The activation function of transform applied to hidden states.

 必需

bias

Whether to apply bias to the final prediction layer.

 必需

act

The activation function of the final prediction output.

 必需

layer_norm_eps

The epsilon used by the layer normalization layers.

 必需

output_name

The name of the tensor required in model outputs.

If is None, will use the default output name of the corresponding head.

 必需

type

The type of the head in the model.

This is used by PolyModel to construct heads.

 必需

loss_weight

The weight to scale the loss.

If is None, will not scale the loss.

 必需
源代码位于： multimolecule/modules/heads/config.py
Python
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
class HeadConfig(BaseHeadConfig):
    r"""
    Configuration class for a prediction head.

    Args:
        num_labels:
            Number of labels to use in the last layer added to the model, typically for a classification task.

            Head should look for [`Config.num_labels`][multimolecule.PreTrainedConfig] if is `None`.
        problem_type:
            Problem type for `XxxForYyyPrediction` models. Can be one of `"binary"`, `"regression"`,
            `"multiclass"` or `"multilabel"`.

            Head should look for [`Config.problem_type`][multimolecule.PreTrainedConfig] if is `None`.
        hidden_size:
            Dimensionality of the encoder layers and the pooler layer.

            Head should look for [`Config.hidden_size`][multimolecule.PreTrainedConfig] if is `None`.
        dropout:
            The dropout ratio for the hidden states.
        transform:
            The transform operation applied to hidden states.
        transform_act:
            The activation function of transform applied to hidden states.
        bias:
            Whether to apply bias to the final prediction layer.
        act:
            The activation function of the final prediction output.
        layer_norm_eps:
            The epsilon used by the layer normalization layers.
        output_name:
            The name of the tensor required in model outputs.

            If is `None`, will use the default output name of the corresponding head.
        type:
            The type of the head in the model.

            This is used by [`PolyModel`][multimolecule.PolyModel] to construct heads.
        loss_weight:
            The weight to scale the loss.

            If is `None`, will not scale the loss.
    """

    num_labels: Optional[int] = None
    problem_type: Optional[str] = None
    hidden_size: Optional[int] = None
    dropout: float = 0.0
    transform: Optional[str] = None
    transform_act: Optional[str] = "gelu"
    bias: bool = True
    act: Optional[str] = None
    layer_norm_eps: float = 1e-12
    output_name: Optional[str] = None
    type: Optional[str] = None
    loss_weight: Optional[float] = None

MaskedLMHeadConfig

Bases: BaseHeadConfig

Configuration class for a Masked Language Modeling head.

参数：

名称 类型 描述 默认

hidden_size

Dimensionality of the encoder layers and the pooler layer.

Head should look for Config.hidden_size if is None.

 必需

dropout

The dropout ratio for the hidden states.

 必需

transform

The transform operation applied to hidden states.

 必需

transform_act

The activation function of transform applied to hidden states.

 必需

bias

Whether to apply bias to the final prediction layer.

 必需

act

The activation function of the final prediction output.

 必需

layer_norm_eps

The epsilon used by the layer normalization layers.

 必需

output_name

The name of the tensor required in model outputs.

If is None, will use the default output name of the corresponding head.

 必需

loss_weight

The weight to scale the loss.

If is None, will not scale the loss.

 必需
源代码位于： multimolecule/modules/heads/config.py
Python
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
class MaskedLMHeadConfig(BaseHeadConfig):
    r"""
    Configuration class for a Masked Language Modeling head.

    Args:
        hidden_size:
            Dimensionality of the encoder layers and the pooler layer.

            Head should look for [`Config.hidden_size`][multimolecule.PreTrainedConfig] if is `None`.
        dropout:
            The dropout ratio for the hidden states.
        transform:
            The transform operation applied to hidden states.
        transform_act:
            The activation function of transform applied to hidden states.
        bias:
            Whether to apply bias to the final prediction layer.
        act:
            The activation function of the final prediction output.
        layer_norm_eps:
            The epsilon used by the layer normalization layers.
        output_name:
            The name of the tensor required in model outputs.

            If is `None`, will use the default output name of the corresponding head.
        loss_weight:
            The weight to scale the loss.

            If is `None`, will not scale the loss.
    """

    hidden_size: Optional[int] = None
    dropout: float = 0.0
    transform: Optional[str] = "nonlinear"
    transform_act: Optional[str] = "gelu"
    bias: bool = True
    act: Optional[str] = None
    layer_norm_eps: float = 1e-12
    output_name: Optional[str] = None
    loss_weight: Optional[float] = None

PreTrainedConfig

Bases: PretrainedConfig

Base class for all model configuration classes.

参数：

名称 类型 描述 默认

pad_token_id

int

The ID of the padding token.

 0

bos_token_id

int

The ID of the beginning-of-sequence token.

 1

eos_token_id

int

The ID of the end-of-sequence token.

 2

unk_token_id

int

The ID of the unknown token.

 3

mask_token_id

int

The ID of the mask token.

 4

null_token_id

int

The ID of the null or placeholder token.

 5

num_labels

int

Default number of labels for prediction heads.

 1
源代码位于： multimolecule/models/configuration_utils.py
Python
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
class PreTrainedConfig(PretrainedConfig):
    r"""
    Base class for all model configuration classes.

    Args:
        pad_token_id:
            The ID of the padding token.
        bos_token_id:
            The ID of the beginning-of-sequence token.
        eos_token_id:
            The ID of the end-of-sequence token.
        unk_token_id:
            The ID of the unknown token.
        mask_token_id:
            The ID of the mask token.
        null_token_id:
            The ID of the null or placeholder token.
        num_labels:
            Default number of labels for prediction heads.
    """

    head: HeadConfig | None = None
    num_labels: int = 1

    pad_token_id: int = 0
    bos_token_id: int = 1
    eos_token_id: int = 2
    unk_token_id: int = 3
    mask_token_id: int = 4
    null_token_id: int = 5

    def __init__(
        self,
        pad_token_id: int = 0,
        bos_token_id: int = 1,
        eos_token_id: int = 2,
        unk_token_id: int = 3,
        mask_token_id: int = 4,
        null_token_id: int = 5,
        num_labels: int = 1,
        **kwargs,
    ):
        if "attn_implementation" not in kwargs and "_attn_implementation" not in kwargs:
            kwargs["attn_implementation"] = "eager"
        kwargs.setdefault("tie_word_embeddings", True)
        super().__init__(
            pad_token_id=pad_token_id,
            bos_token_id=bos_token_id,
            eos_token_id=eos_token_id,
            unk_token_id=unk_token_id,
            mask_token_id=mask_token_id,
            null_token_id=null_token_id,
            num_labels=num_labels,
            **kwargs,
        )

    def to_dict(self):
        output = super().to_dict()
        for k, v in output.items():
            if hasattr(v, "to_dict"):
                output[k] = v.to_dict()
            if is_dataclass(v):
                output[k] = asdict(v)
        return output

validate_attention_dimensions

Python
validate_attention_dimensions(hidden_size: int, num_attention_heads: int) -> None

Validate the standard transformer hidden-size/head-count relationship.

源代码位于： multimolecule/models/configuration_utils.py
Python
40
41
42
43
44
45
46
47
def validate_attention_dimensions(hidden_size: int, num_attention_heads: int) -> None:
    """Validate the standard transformer hidden-size/head-count relationship."""
    if num_attention_heads <= 0:
        raise ValueError(f"num_attention_heads ({num_attention_heads}) must be positive.")
    if hidden_size % num_attention_heads != 0:
        raise ValueError(
            f"hidden_size ({hidden_size}) must be divisible by num_attention_heads ({num_attention_heads})."
        )