Module pysimt.layers.positionwise_ff

Positionwise feed-forward layer.

"""Positionwise feed-forward layer."""

from torch import nn

from . import FF
from .transformers import BaseSublayer


class PositionwiseFF(nn.Module):
    """Positionwise Feed-forward layer.

    Arguments:

    Input:

    Output:
    """

    def __init__(self, model_dim, ff_dim, activ='gelu', dropout=0.1):
        """
        Creates a PositionwiseFF.
        :param model_dim: The model dimensions.
        :param ff_dim: The feedforward dimensions.
        :param activ: The activation function. Default: gelu
        :param dropout: The amount of dropout. Default: 0.1
        """
        super().__init__()
        self.model_dim = model_dim
        self.ff_dim = ff_dim
        self.activ = activ

        # Create the layers
        self.layers = nn.Sequential(
            FF(self.model_dim, self.ff_dim, activ=self.activ),
            nn.Dropout(dropout),
            FF(self.ff_dim, self.model_dim, activ=None),
        )

    def forward(self, x):
        return self.layers(x)


class PositionwiseSublayer(BaseSublayer):
    def __init__(self, model_dim, ff_dim, ff_activ='gelu', dropout=0.1, is_pre_norm=False):
        """
        Creates a PositionwiseSublayer.
        :param model_dim: The model dimensions.
        :param ff_dim: The dimensions of the feed forward network.
        :param ff_activ: The activation of the feed forward network.
        :param dropout: The dropout rate.
        :param is_pre_norm: Whether the layer type is pre_norm. Default: True.
        """
        super().__init__(model_dim, dropout, is_pre_norm)
        self.feed_forward = PositionwiseFF(model_dim, ff_dim, ff_activ, dropout=dropout)

    def forward(self, x, mask=None):
        """
        Performs a forward pass over the PositionwiseSublayer.
        :param x: The input x.
        :param mask: The input mask.
        :return: The output from the forward pass of the PositionwiseSublayer.
        """
        residual = x
        x = self.apply_pre_norm_if_needed(x)
        x = self.feed_forward(x)
        x = self.apply_residual(residual, x)
        x = self.apply_post_norm_if_needed(x)
        return x

Classes

class PositionwiseFF (model_dim, ff_dim, activ='gelu', dropout=0.1)

Positionwise Feed-forward layer.

Arguments:

Input:

Output:

Creates a PositionwiseFF. :param model_dim: The model dimensions. :param ff_dim: The feedforward dimensions. :param activ: The activation function. Default: gelu :param dropout: The amount of dropout. Default: 0.1

Expand source code

class PositionwiseFF(nn.Module):
    """Positionwise Feed-forward layer.

    Arguments:

    Input:

    Output:
    """

    def __init__(self, model_dim, ff_dim, activ='gelu', dropout=0.1):
        """
        Creates a PositionwiseFF.
        :param model_dim: The model dimensions.
        :param ff_dim: The feedforward dimensions.
        :param activ: The activation function. Default: gelu
        :param dropout: The amount of dropout. Default: 0.1
        """
        super().__init__()
        self.model_dim = model_dim
        self.ff_dim = ff_dim
        self.activ = activ

        # Create the layers
        self.layers = nn.Sequential(
            FF(self.model_dim, self.ff_dim, activ=self.activ),
            nn.Dropout(dropout),
            FF(self.ff_dim, self.model_dim, activ=None),
        )

    def forward(self, x):
        return self.layers(x)

Ancestors

• torch.nn.modules.module.Module

Class variables

var dump_patches : bool

var training : bool

Methods

def forward(self, x) ‑> Callable[..., Any]

Defines the computation performed at every call.

Should be overridden by all subclasses.

Note

Although the recipe for forward pass needs to be defined within this function, one should call the :class:Module instance afterwards instead of this since the former takes care of running the registered hooks while the latter silently ignores them.

Expand source code

def forward(self, x):
    return self.layers(x)

class PositionwiseSublayer (model_dim, ff_dim, ff_activ='gelu', dropout=0.1, is_pre_norm=False)

Base class for all neural network modules.

Your models should also subclass this class.

Modules can also contain other Modules, allowing to nest them in a tree structure. You can assign the submodules as regular attributes::

import torch.nn as nn
import torch.nn.functional as F

class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.conv1 = nn.Conv2d(1, 20, 5)
        self.conv2 = nn.Conv2d(20, 20, 5)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        return F.relu(self.conv2(x))

Submodules assigned in this way will be registered, and will have their parameters converted too when you call :meth:to, etc.

:ivar training: Boolean represents whether this module is in training or evaluation mode. :vartype training: bool

Creates a PositionwiseSublayer. :param model_dim: The model dimensions. :param ff_dim: The dimensions of the feed forward network. :param ff_activ: The activation of the feed forward network. :param dropout: The dropout rate. :param is_pre_norm: Whether the layer type is pre_norm. Default: True.

Expand source code

class PositionwiseSublayer(BaseSublayer):
    def __init__(self, model_dim, ff_dim, ff_activ='gelu', dropout=0.1, is_pre_norm=False):
        """
        Creates a PositionwiseSublayer.
        :param model_dim: The model dimensions.
        :param ff_dim: The dimensions of the feed forward network.
        :param ff_activ: The activation of the feed forward network.
        :param dropout: The dropout rate.
        :param is_pre_norm: Whether the layer type is pre_norm. Default: True.
        """
        super().__init__(model_dim, dropout, is_pre_norm)
        self.feed_forward = PositionwiseFF(model_dim, ff_dim, ff_activ, dropout=dropout)

    def forward(self, x, mask=None):
        """
        Performs a forward pass over the PositionwiseSublayer.
        :param x: The input x.
        :param mask: The input mask.
        :return: The output from the forward pass of the PositionwiseSublayer.
        """
        residual = x
        x = self.apply_pre_norm_if_needed(x)
        x = self.feed_forward(x)
        x = self.apply_residual(residual, x)
        x = self.apply_post_norm_if_needed(x)
        return x

Ancestors

• BaseSublayer
• torch.nn.modules.module.Module

Class variables

var dump_patches : bool

var training : bool

Methods

def forward(self, x, mask=None) ‑> Callable[..., Any]

Performs a forward pass over the PositionwiseSublayer. :param x: The input x. :param mask: The input mask. :return: The output from the forward pass of the PositionwiseSublayer.

Expand source code

def forward(self, x, mask=None):
    """
    Performs a forward pass over the PositionwiseSublayer.
    :param x: The input x.
    :param mask: The input mask.
    :return: The output from the forward pass of the PositionwiseSublayer.
    """
    residual = x
    x = self.apply_pre_norm_if_needed(x)
    x = self.feed_forward(x)
    x = self.apply_residual(residual, x)
    x = self.apply_post_norm_if_needed(x)
    return x

Inherited members

• BaseSublayer:
  • apply_post_norm_if_needed
  • apply_pre_norm_if_needed
  • apply_residual