diff --git a/docs/source/_rst/_code.rst b/docs/source/_rst/_code.rst index 813f1e46b..211398d9d 100644 --- a/docs/source/_rst/_code.rst +++ b/docs/source/_rst/_code.rst @@ -168,25 +168,26 @@ Optimizers and Schedulers TorchScheduler -Adaptive Activation Functions +Adaptive Functions ------------------------------- .. toctree:: :titlesonly: - Adaptive Function Interface - Adaptive ReLU - Adaptive Sigmoid - Adaptive Tanh - Adaptive SiLU - Adaptive Mish - Adaptive ELU - Adaptive CELU - Adaptive GELU - Adaptive Softmin - Adaptive Softmax - Adaptive SIREN - Adaptive Exp + Adaptive Function Interface + Base Adaptive Function + Adaptive CELU + Adaptive ELU + Adaptive Exp + Adaptive GELU + Adaptive Mish + Adaptive ReLU + Adaptive Sigmoid + Adaptive SiLU + Adaptive SIREN + Adaptive Softmax + Adaptive Softmin + Adaptive Tanh Equations and Differential Operators diff --git a/docs/source/_rst/adaptive_function/AdaptiveActivationFunctionInterface.rst b/docs/source/_rst/adaptive_function/AdaptiveActivationFunctionInterface.rst deleted file mode 100644 index db035b46b..000000000 --- a/docs/source/_rst/adaptive_function/AdaptiveActivationFunctionInterface.rst +++ /dev/null @@ -1,8 +0,0 @@ -AdaptiveActivationFunctionInterface -======================================= - -.. currentmodule:: pina.adaptive_function.adaptive_function_interface - -.. automodule:: pina._src.adaptive_function.adaptive_function_interface - :members: - :show-inheritance: diff --git a/docs/source/_rst/adaptive_function/AdaptiveCELU.rst b/docs/source/_rst/adaptive_function/AdaptiveCELU.rst deleted file mode 100644 index 5c04ecde3..000000000 --- a/docs/source/_rst/adaptive_function/AdaptiveCELU.rst +++ /dev/null @@ -1,9 +0,0 @@ -AdaptiveCELU -============ - -.. currentmodule:: pina.adaptive_function - -.. autoclass:: pina._src.adaptive_function.adaptive_function.AdaptiveCELU - :members: - :show-inheritance: - :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_function/AdaptiveELU.rst b/docs/source/_rst/adaptive_function/AdaptiveELU.rst deleted file mode 100644 index 2b27c4038..000000000 --- a/docs/source/_rst/adaptive_function/AdaptiveELU.rst +++ /dev/null @@ -1,9 +0,0 @@ -AdaptiveELU -=========== - -.. currentmodule:: pina.adaptive_function.adaptive_function - -.. autoclass:: pina._src.adaptive_function.adaptive_function.AdaptiveELU - :members: - :show-inheritance: - :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_function/AdaptiveExp.rst b/docs/source/_rst/adaptive_function/AdaptiveExp.rst deleted file mode 100644 index 000f5bab2..000000000 --- a/docs/source/_rst/adaptive_function/AdaptiveExp.rst +++ /dev/null @@ -1,9 +0,0 @@ -AdaptiveExp -=========== - -.. currentmodule:: pina.adaptive_function.adaptive_function - -.. autoclass:: pina._src.adaptive_function.adaptive_function.AdaptiveExp - :members: - :show-inheritance: - :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_function/AdaptiveGELU.rst b/docs/source/_rst/adaptive_function/AdaptiveGELU.rst deleted file mode 100644 index 35ae98382..000000000 --- a/docs/source/_rst/adaptive_function/AdaptiveGELU.rst +++ /dev/null @@ -1,9 +0,0 @@ -AdaptiveGELU -============ - -.. currentmodule:: pina.adaptive_function.adaptive_function - -.. autoclass:: pina._src.adaptive_function.adaptive_function.AdaptiveGELU - :members: - :show-inheritance: - :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_function/AdaptiveMish.rst b/docs/source/_rst/adaptive_function/AdaptiveMish.rst deleted file mode 100644 index 6b440f5d2..000000000 --- a/docs/source/_rst/adaptive_function/AdaptiveMish.rst +++ /dev/null @@ -1,9 +0,0 @@ -AdaptiveMish -============ - -.. currentmodule:: pina.adaptive_function.adaptive_function - -.. autoclass:: pina._src.adaptive_function.adaptive_function.AdaptiveMish - :members: - :show-inheritance: - :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_function/AdaptiveReLU.rst b/docs/source/_rst/adaptive_function/AdaptiveReLU.rst deleted file mode 100644 index 379ee1d66..000000000 --- a/docs/source/_rst/adaptive_function/AdaptiveReLU.rst +++ /dev/null @@ -1,9 +0,0 @@ -AdaptiveReLU -============ - -.. currentmodule:: pina.adaptive_function.adaptive_function - -.. autoclass:: pina._src.adaptive_function.adaptive_function.AdaptiveReLU - :members: - :show-inheritance: - :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_function/AdaptiveSIREN.rst b/docs/source/_rst/adaptive_function/AdaptiveSIREN.rst deleted file mode 100644 index 6e4aaf6f0..000000000 --- a/docs/source/_rst/adaptive_function/AdaptiveSIREN.rst +++ /dev/null @@ -1,9 +0,0 @@ -AdaptiveSIREN -============= - -.. currentmodule:: pina.adaptive_function.adaptive_function - -.. autoclass:: pina._src.adaptive_function.adaptive_function.AdaptiveSIREN - :members: - :show-inheritance: - :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_function/AdaptiveSiLU.rst b/docs/source/_rst/adaptive_function/AdaptiveSiLU.rst deleted file mode 100644 index b1fa345f1..000000000 --- a/docs/source/_rst/adaptive_function/AdaptiveSiLU.rst +++ /dev/null @@ -1,9 +0,0 @@ -AdaptiveSiLU -============ - -.. currentmodule:: pina.adaptive_function.adaptive_function - -.. autoclass:: pina._src.adaptive_function.adaptive_function.AdaptiveSiLU - :members: - :show-inheritance: - :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_function/AdaptiveSigmoid.rst b/docs/source/_rst/adaptive_function/AdaptiveSigmoid.rst deleted file mode 100644 index 3a2c19a9b..000000000 --- a/docs/source/_rst/adaptive_function/AdaptiveSigmoid.rst +++ /dev/null @@ -1,9 +0,0 @@ -AdaptiveSigmoid -=============== - -.. currentmodule:: pina.adaptive_function.adaptive_function - -.. autoclass:: pina._src.adaptive_function.adaptive_function.AdaptiveSigmoid - :members: - :show-inheritance: - :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_function/AdaptiveSoftmax.rst b/docs/source/_rst/adaptive_function/AdaptiveSoftmax.rst deleted file mode 100644 index 0a2352508..000000000 --- a/docs/source/_rst/adaptive_function/AdaptiveSoftmax.rst +++ /dev/null @@ -1,9 +0,0 @@ -AdaptiveSoftmax -=============== - -.. currentmodule:: pina.adaptive_function.adaptive_function - -.. autoclass:: pina._src.adaptive_function.adaptive_function.AdaptiveSoftmax - :members: - :show-inheritance: - :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_function/AdaptiveSoftmin.rst b/docs/source/_rst/adaptive_function/AdaptiveSoftmin.rst deleted file mode 100644 index d842c5f26..000000000 --- a/docs/source/_rst/adaptive_function/AdaptiveSoftmin.rst +++ /dev/null @@ -1,9 +0,0 @@ -AdaptiveSoftmin -=============== - -.. currentmodule:: pina.adaptive_function.adaptive_function - -.. autoclass:: pina._src.adaptive_function.adaptive_function.AdaptiveSoftmin - :members: - :show-inheritance: - :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_function/AdaptiveTanh.rst b/docs/source/_rst/adaptive_function/AdaptiveTanh.rst deleted file mode 100644 index ca183abec..000000000 --- a/docs/source/_rst/adaptive_function/AdaptiveTanh.rst +++ /dev/null @@ -1,9 +0,0 @@ -AdaptiveTanh -============ - -.. currentmodule:: pina.adaptive_function.adaptive_function - -.. autoclass:: pina._src.adaptive_function.adaptive_function.AdaptiveTanh - :members: - :show-inheritance: - :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_function/adaptive_celu.rst b/docs/source/_rst/adaptive_function/adaptive_celu.rst new file mode 100644 index 000000000..b04bcf42b --- /dev/null +++ b/docs/source/_rst/adaptive_function/adaptive_celu.rst @@ -0,0 +1,9 @@ +Adaptive CELU +================== +.. currentmodule:: pina.adaptive_function.adaptive_celu + +.. automodule:: pina._src.adaptive_function.adaptive_celu + +.. autoclass:: pina._src.adaptive_function.adaptive_celu.AdaptiveCELU + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/_rst/adaptive_function/adaptive_elu.rst b/docs/source/_rst/adaptive_function/adaptive_elu.rst new file mode 100644 index 000000000..e758b20b3 --- /dev/null +++ b/docs/source/_rst/adaptive_function/adaptive_elu.rst @@ -0,0 +1,9 @@ +Adaptive ELU +============================= +.. currentmodule:: pina.adaptive_function.adaptive_elu + +.. automodule:: pina._src.adaptive_function.adaptive_elu + +.. autoclass:: pina._src.adaptive_function.adaptive_elu.AdaptiveELU + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/_rst/adaptive_function/adaptive_exp.rst b/docs/source/_rst/adaptive_function/adaptive_exp.rst new file mode 100644 index 000000000..3feeb6192 --- /dev/null +++ b/docs/source/_rst/adaptive_function/adaptive_exp.rst @@ -0,0 +1,9 @@ +Adaptive Exp +============================= +.. currentmodule:: pina.adaptive_function.adaptive_exp + +.. automodule:: pina._src.adaptive_function.adaptive_exp + +.. autoclass:: pina._src.adaptive_function.adaptive_exp.AdaptiveExp + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/_rst/adaptive_function/adaptive_function_interface.rst b/docs/source/_rst/adaptive_function/adaptive_function_interface.rst new file mode 100644 index 000000000..e7859c0d2 --- /dev/null +++ b/docs/source/_rst/adaptive_function/adaptive_function_interface.rst @@ -0,0 +1,9 @@ +Adaptive Function Interface +============================= +.. currentmodule:: pina.adaptive_function.adaptive_function_interface + +.. automodule:: pina._src.adaptive_function.adaptive_function_interface + +.. autoclass:: pina._src.adaptive_function.adaptive_function_interface.AdaptiveFunctionInterface + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/_rst/adaptive_function/adaptive_gelu.rst b/docs/source/_rst/adaptive_function/adaptive_gelu.rst new file mode 100644 index 000000000..a07960373 --- /dev/null +++ b/docs/source/_rst/adaptive_function/adaptive_gelu.rst @@ -0,0 +1,9 @@ +Adaptive GELU +============================= +.. currentmodule:: pina.adaptive_function.adaptive_gelu + +.. automodule:: pina._src.adaptive_function.adaptive_gelu + +.. autoclass:: pina._src.adaptive_function.adaptive_gelu.AdaptiveGELU + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/_rst/adaptive_function/adaptive_mish.rst b/docs/source/_rst/adaptive_function/adaptive_mish.rst new file mode 100644 index 000000000..f56c911fb --- /dev/null +++ b/docs/source/_rst/adaptive_function/adaptive_mish.rst @@ -0,0 +1,9 @@ +Adaptive Mish +============================= +.. currentmodule:: pina.adaptive_function.adaptive_mish + +.. automodule:: pina._src.adaptive_function.adaptive_mish + +.. autoclass:: pina._src.adaptive_function.adaptive_mish.AdaptiveMish + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/_rst/adaptive_function/adaptive_relu.rst b/docs/source/_rst/adaptive_function/adaptive_relu.rst new file mode 100644 index 000000000..a2032f344 --- /dev/null +++ b/docs/source/_rst/adaptive_function/adaptive_relu.rst @@ -0,0 +1,9 @@ +Adaptive ReLU +============================= +.. currentmodule:: pina.adaptive_function.adaptive_relu + +.. automodule:: pina._src.adaptive_function.adaptive_relu + +.. autoclass:: pina._src.adaptive_function.adaptive_relu.AdaptiveReLU + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/_rst/adaptive_function/adaptive_sigmoid.rst b/docs/source/_rst/adaptive_function/adaptive_sigmoid.rst new file mode 100644 index 000000000..8aef91c0d --- /dev/null +++ b/docs/source/_rst/adaptive_function/adaptive_sigmoid.rst @@ -0,0 +1,9 @@ +Adaptive Sigmoid +============================= +.. currentmodule:: pina.adaptive_function.adaptive_sigmoid + +.. automodule:: pina._src.adaptive_function.adaptive_sigmoid + +.. autoclass:: pina._src.adaptive_function.adaptive_sigmoid.AdaptiveSigmoid + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/_rst/adaptive_function/adaptive_silu.rst b/docs/source/_rst/adaptive_function/adaptive_silu.rst new file mode 100644 index 000000000..2d22dcf20 --- /dev/null +++ b/docs/source/_rst/adaptive_function/adaptive_silu.rst @@ -0,0 +1,9 @@ +Adaptive SiLU +============================= +.. currentmodule:: pina.adaptive_function.adaptive_silu + +.. automodule:: pina._src.adaptive_function.adaptive_silu + +.. autoclass:: pina._src.adaptive_function.adaptive_silu.AdaptiveSiLU + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/_rst/adaptive_function/adaptive_siren.rst b/docs/source/_rst/adaptive_function/adaptive_siren.rst new file mode 100644 index 000000000..167cd79ff --- /dev/null +++ b/docs/source/_rst/adaptive_function/adaptive_siren.rst @@ -0,0 +1,9 @@ +Adaptive SIREN +============================= +.. currentmodule:: pina.adaptive_function.adaptive_siren + +.. automodule:: pina._src.adaptive_function.adaptive_siren + +.. autoclass:: pina._src.adaptive_function.adaptive_siren.AdaptiveSIREN + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/_rst/adaptive_function/adaptive_softmax.rst b/docs/source/_rst/adaptive_function/adaptive_softmax.rst new file mode 100644 index 000000000..8797acae9 --- /dev/null +++ b/docs/source/_rst/adaptive_function/adaptive_softmax.rst @@ -0,0 +1,9 @@ +Adaptive Softmax +============================= +.. currentmodule:: pina.adaptive_function.adaptive_softmax + +.. automodule:: pina._src.adaptive_function.adaptive_softmax + +.. autoclass:: pina._src.adaptive_function.adaptive_softmax.AdaptiveSoftmax + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/_rst/adaptive_function/adaptive_softmin.rst b/docs/source/_rst/adaptive_function/adaptive_softmin.rst new file mode 100644 index 000000000..72ed8ae1f --- /dev/null +++ b/docs/source/_rst/adaptive_function/adaptive_softmin.rst @@ -0,0 +1,9 @@ +Adaptive Softmin +============================= +.. currentmodule:: pina.adaptive_function.adaptive_softmin + +.. automodule:: pina._src.adaptive_function.adaptive_softmin + +.. autoclass:: pina._src.adaptive_function.adaptive_softmin.AdaptiveSoftmin + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/_rst/adaptive_function/adaptive_tanh.rst b/docs/source/_rst/adaptive_function/adaptive_tanh.rst new file mode 100644 index 000000000..dbd9e4313 --- /dev/null +++ b/docs/source/_rst/adaptive_function/adaptive_tanh.rst @@ -0,0 +1,9 @@ +Adaptive Tanh +============================= +.. currentmodule:: pina.adaptive_function.adaptive_tanh + +.. automodule:: pina._src.adaptive_function.adaptive_tanh + +.. autoclass:: pina._src.adaptive_function.adaptive_tanh.AdaptiveTanh + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/_rst/adaptive_function/base_adaptive_function.rst b/docs/source/_rst/adaptive_function/base_adaptive_function.rst new file mode 100644 index 000000000..6b1e6cee7 --- /dev/null +++ b/docs/source/_rst/adaptive_function/base_adaptive_function.rst @@ -0,0 +1,9 @@ +Base Adaptive Function +============================= +.. currentmodule:: pina.adaptive_function.base_adaptive_function + +.. automodule:: pina._src.adaptive_function.base_adaptive_function + +.. autoclass:: pina._src.adaptive_function.base_adaptive_function.BaseAdaptiveFunction + :members: + :show-inheritance: \ No newline at end of file diff --git a/pina/_src/adaptive_function/adaptive_celu.py b/pina/_src/adaptive_function/adaptive_celu.py new file mode 100644 index 000000000..bb460933c --- /dev/null +++ b/pina/_src/adaptive_function/adaptive_celu.py @@ -0,0 +1,77 @@ +"""Module for the Adaptive CELU activation function.""" + +import torch +from pina._src.adaptive_function.base_adaptive_function import ( + BaseAdaptiveFunction, +) + + +class AdaptiveCELU(BaseAdaptiveFunction): + r""" + Adaptive, trainable variant of the :class:`~torch.nn.CELU` activation. + + This module extends the standard CELU by introducing learnable scaling + and shifting parameters applied to both the input and the output. + + Given the function :math:`\text{CELU}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the corresponding adaptive activation + :math:`\text{CELU}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` is + defined as: + + .. math:: + \text{CELU}_{\text{adaptive}}({x})=\alpha\,\text{CELU}(\beta{x}+\gamma), + + where :math:`\alpha`, :math:`\beta`, and :math:`\gamma` are trainable + parameters controlling output scaling, input scaling, and input shifting, + respectively. + + The CELU function is defined elementwise as: + + .. math:: + \text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x) - 1)) + + .. seealso:: + + **Original reference**: Godfrey, L. B., Gashler, M. S. (2015). + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 7th international joint conference on knowledge discovery, knowledge + engineering and knowledge management (IC3K), Vol. 1. + DOI: `arXiv preprint arXiv:1602.01321. + `_. + + **Original reference**: Jagtap, A. D., Karniadakis, G. E. (2020). + *Adaptive activation functions accelerate convergence in deep and + physics-informed neural networks*. + Journal of Computational Physics, 404. + DOI: `JCP 10.1016 `_. + """ + + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + """ + Initialization of the :class:`AdaptiveCELU` class. + + :param alpha: The output scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type alpha: int | float + :param beta: The input scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type beta: int | float + :param gamma: The input shifting parameter of the adaptive function. + If ``None``, it is initialized to ``0``. Default is ``None``. + :type gamma: int | float + :param fixed: The names of parameters to keep fixed during training. + These parameters will not be optimized and will have + ``requires_grad=False``. Available options are ``"alpha"``, + ``"beta"``, and ``"gamma"``. If ``None``, all parameters are + trainable. Default is ``None``. + :type fixed: str | list[str] + :raises ValueError: If alpha, when provided, is not a number. + :raises ValueError: If beta, when provided, is not a number. + :raises ValueError: If gamma, when provided, is not a number. + :raises ValueError: If fixed, when provided, is neither a string nor a + list of strings. + :raises ValueError: If fixed contains invalid parameter names. + """ + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.CELU() diff --git a/pina/_src/adaptive_function/adaptive_elu.py b/pina/_src/adaptive_function/adaptive_elu.py new file mode 100644 index 000000000..12b40fa46 --- /dev/null +++ b/pina/_src/adaptive_function/adaptive_elu.py @@ -0,0 +1,80 @@ +"""Module for the Adaptive ELU activation function.""" + +import torch +from pina._src.adaptive_function.base_adaptive_function import ( + BaseAdaptiveFunction, +) + + +class AdaptiveELU(BaseAdaptiveFunction): + r""" + Adaptive, trainable variant of the :class:`~torch.nn.ELU` activation. + + This module extends the standard ELU by introducing learnable scaling + and shifting parameters applied to both the input and the output. + + Given the function :math:`\text{ELU}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the corresponding adaptive activation + :math:`\text{ELU}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` is + defined as: + + .. math:: + \text{ELU}_{\text{adaptive}}({x}) = \alpha\,\text{ELU}(\beta{x}+\gamma), + + where :math:`\alpha`, :math:`\beta`, and :math:`\gamma` are trainable + parameters controlling output scaling, input scaling, and input shifting, + respectively. + + The ELU function is defined elementwise as: + + .. math:: + \text{ELU}(x) = \begin{cases} + x, & \text{ if }x > 0\\ + \exp(x) - 1, & \text{ if }x \leq 0 + \end{cases} + + .. seealso:: + + **Original reference**: Godfrey, L. B., Gashler, M. S. (2015). + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 7th international joint conference on knowledge discovery, knowledge + engineering and knowledge management (IC3K), Vol. 1. + DOI: `arXiv preprint arXiv:1602.01321. + `_. + + **Original reference**: Jagtap, A. D., Karniadakis, G. E. (2020). + *Adaptive activation functions accelerate convergence in deep and + physics-informed neural networks*. + Journal of Computational Physics, 404. + DOI: `JCP 10.1016 `_. + """ + + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + """ + Initialization of the :class:`AdaptiveELU` class. + + :param alpha: The output scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type alpha: int | float + :param beta: The input scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type beta: int | float + :param gamma: The input shifting parameter of the adaptive function. + If ``None``, it is initialized to ``0``. Default is ``None``. + :type gamma: int | float + :param fixed: The names of parameters to keep fixed during training. + These parameters will not be optimized and will have + ``requires_grad=False``. Available options are ``"alpha"``, + ``"beta"``, and ``"gamma"``. If ``None``, all parameters are + trainable. Default is ``None``. + :type fixed: str | list[str] + :raises ValueError: If alpha, when provided, is not a number. + :raises ValueError: If beta, when provided, is not a number. + :raises ValueError: If gamma, when provided, is not a number. + :raises ValueError: If fixed, when provided, is neither a string nor a + list of strings. + :raises ValueError: If fixed contains invalid parameter names. + """ + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.ELU() diff --git a/pina/_src/adaptive_function/adaptive_exp.py b/pina/_src/adaptive_function/adaptive_exp.py new file mode 100644 index 000000000..c6484f8c9 --- /dev/null +++ b/pina/_src/adaptive_function/adaptive_exp.py @@ -0,0 +1,73 @@ +"""Module for the Adaptive Exp activation function.""" + +import torch +from pina._src.adaptive_function.base_adaptive_function import ( + BaseAdaptiveFunction, +) + + +class AdaptiveExp(BaseAdaptiveFunction): + r""" + Adaptive, trainable variant of the :obj:`~torch.exp` activation. + + This module extends the standard exponential function by introducing + learnable scaling and shifting parameters applied to both the input and the + output. + + Given the function :math:`\text{exp}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the corresponding adaptive activation + :math:`\text{exp}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` + is defined as: + + .. math:: + \text{exp}_{\text{adaptive}}({x}) = \alpha\,\text{exp}(\beta{x}+\gamma), + + where :math:`\alpha`, :math:`\beta`, and :math:`\gamma` are trainable + parameters controlling output scaling, input scaling, and input shifting, + respectively. + + .. seealso:: + + **Original reference**: Godfrey, L. B., Gashler, M. S. (2015). + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 7th international joint conference on knowledge discovery, knowledge + engineering and knowledge management (IC3K), Vol. 1. + DOI: `arXiv preprint arXiv:1602.01321. + `_. + + **Original reference**: Jagtap, A. D., Karniadakis, G. E. (2020). + *Adaptive activation functions accelerate convergence in deep and + physics-informed neural networks*. + Journal of Computational Physics, 404. + DOI: `JCP 10.1016 `_. + """ + + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + """ + Initialization of the :class:`AdaptiveExp` class. + + :param alpha: The output scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type alpha: int | float + :param beta: The input scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type beta: int | float + :param gamma: The input shifting parameter of the adaptive function. + If ``None``, it is initialized to ``0``. Default is ``None``. + :type gamma: int | float + :param fixed: The names of parameters to keep fixed during training. + These parameters will not be optimized and will have + ``requires_grad=False``. Available options are ``"alpha"``, + ``"beta"``, and ``"gamma"``. If ``None``, all parameters are + trainable. Default is ``None``. + :type fixed: str | list[str] + :raises ValueError: If alpha, when provided, is not a number. + :raises ValueError: If beta, when provided, is not a number. + :raises ValueError: If gamma, when provided, is not a number. + :raises ValueError: If fixed, when provided, is neither a string nor a + list of strings. + :raises ValueError: If fixed contains invalid parameter names. + """ + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.exp diff --git a/pina/_src/adaptive_function/adaptive_function.py b/pina/_src/adaptive_function/adaptive_function.py deleted file mode 100644 index 21f45fd1e..000000000 --- a/pina/_src/adaptive_function/adaptive_function.py +++ /dev/null @@ -1,511 +0,0 @@ -"""Module for the Adaptive Functions.""" - -import torch -from pina._src.core.utils import check_consistency -from pina._src.adaptive_function.adaptive_function_interface import ( - AdaptiveActivationFunctionInterface, -) - - -class AdaptiveReLU(AdaptiveActivationFunctionInterface): - r""" - Adaptive trainable :class:`~torch.nn.ReLU` activation function. - - Given the function :math:`\text{ReLU}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, - the adaptive function - :math:`\text{ReLU}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` - is defined as: - - .. math:: - \text{ReLU}_{\text{adaptive}}({x})=\alpha\,\text{ReLU}(\beta{x}+\gamma), - - where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the - ReLU function is defined as: - - .. math:: - \text{ReLU}(x) = \max(0, x) - - .. seealso:: - - **Original reference**: Godfrey, Luke B., and Michael S. Gashler. - *A continuum among logarithmic, linear, and exponential functions, - and its potential to improve generalization in neural networks.* - 2015 7th international joint conference on knowledge discovery, - knowledge engineering and knowledge management (IC3K). - Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. - `_. - - Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive - activation functions accelerate convergence in deep and - physics-informed neural networks*. Journal of - Computational Physics 404 (2020): 109136. - DOI: `JCP 10.1016 - `_. - """ - - def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): - super().__init__(alpha, beta, gamma, fixed) - self._func = torch.nn.ReLU() - - -class AdaptiveSigmoid(AdaptiveActivationFunctionInterface): - r""" - Adaptive trainable :class:`~torch.nn.Sigmoid` activation function. - - Given the function - :math:`\text{Sigmoid}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, - the adaptive function - :math:`\text{Sigmoid}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` - is defined as: - - .. math:: - \text{Sigmoid}_{\text{adaptive}}({x})= - \alpha\,\text{Sigmoid}(\beta{x}+\gamma), - - where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the - Sigmoid function is defined as: - - .. math:: - \text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)} - - .. seealso:: - - **Original reference**: Godfrey, Luke B., and Michael S. Gashler. - *A continuum among logarithmic, linear, and exponential functions, - and its potential to improve generalization in neural networks.* - 2015 7th international joint conference on knowledge discovery, - knowledge engineering and knowledge management (IC3K). - Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. - `_. - - Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive - activation functions accelerate convergence in deep and - physics-informed neural networks*. Journal of - Computational Physics 404 (2020): 109136. - DOI: `JCP 10.1016 - `_. - """ - - def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): - super().__init__(alpha, beta, gamma, fixed) - self._func = torch.nn.Sigmoid() - - -class AdaptiveTanh(AdaptiveActivationFunctionInterface): - r""" - Adaptive trainable :class:`~torch.nn.Tanh` activation function. - - Given the function :math:`\text{Tanh}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, - the adaptive function - :math:`\text{Tanh}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` - is defined as: - - .. math:: - \text{Tanh}_{\text{adaptive}}({x})=\alpha\,\text{Tanh}(\beta{x}+\gamma), - - where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the - Tanh function is defined as: - - .. math:: - \text{Tanh}(x) = \frac{\exp(x) - \exp(-x)} {\exp(x) + \exp(-x)} - - .. seealso:: - - **Original reference**: Godfrey, Luke B., and Michael S. Gashler. - *A continuum among logarithmic, linear, and exponential functions, - and its potential to improve generalization in neural networks.* - 2015 7th international joint conference on knowledge discovery, - knowledge engineering and knowledge management (IC3K). - Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. - `_. - - Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive - activation functions accelerate convergence in deep and - physics-informed neural networks*. Journal of - Computational Physics 404 (2020): 109136. - DOI: `JCP 10.1016 - `_. - """ - - def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): - super().__init__(alpha, beta, gamma, fixed) - self._func = torch.nn.Tanh() - - -class AdaptiveSiLU(AdaptiveActivationFunctionInterface): - r""" - Adaptive trainable :class:`~torch.nn.SiLU` activation function. - - Given the function :math:`\text{SiLU}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, - the adaptive function - :math:`\text{SiLU}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` - is defined as: - - .. math:: - \text{SiLU}_{\text{adaptive}}({x})=\alpha\,\text{SiLU}(\beta{x}+\gamma), - - where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the - SiLU function is defined as: - - .. math:: - \text{SiLU}(x) = x * \sigma(x), \text{where }\sigma(x) - \text{ is the logistic sigmoid.} - - .. seealso:: - - **Original reference**: Godfrey, Luke B., and Michael S. Gashler. - *A continuum among logarithmic, linear, and exponential functions, - and its potential to improve generalization in neural networks.* - 2015 7th international joint conference on knowledge discovery, - knowledge engineering and knowledge management (IC3K). - Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. - `_. - - Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive - activation functions accelerate convergence in deep and - physics-informed neural networks*. Journal of - Computational Physics 404 (2020): 109136. - DOI: `JCP 10.1016 - `_. - """ - - def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): - super().__init__(alpha, beta, gamma, fixed) - self._func = torch.nn.SiLU() - - -class AdaptiveMish(AdaptiveActivationFunctionInterface): - r""" - Adaptive trainable :class:`~torch.nn.Mish` activation function. - - Given the function :math:`\text{Mish}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, - the adaptive function - :math:`\text{Mish}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` - is defined as: - - .. math:: - \text{Mish}_{\text{adaptive}}({x})=\alpha\,\text{Mish}(\beta{x}+\gamma), - - where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the - Mish function is defined as: - - .. math:: - \text{Mish}(x) = x * \text{Tanh}(x) - - .. seealso:: - - **Original reference**: Godfrey, Luke B., and Michael S. Gashler. - *A continuum among logarithmic, linear, and exponential functions, - and its potential to improve generalization in neural networks.* - 2015 7th international joint conference on knowledge discovery, - knowledge engineering and knowledge management (IC3K). - Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. - `_. - - Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive - activation functions accelerate convergence in deep and - physics-informed neural networks*. Journal of - Computational Physics 404 (2020): 109136. - DOI: `JCP 10.1016 - `_. - """ - - def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): - super().__init__(alpha, beta, gamma, fixed) - self._func = torch.nn.Mish() - - -class AdaptiveELU(AdaptiveActivationFunctionInterface): - r""" - Adaptive trainable :class:`~torch.nn.ELU` activation function. - - Given the function :math:`\text{ELU}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, - the adaptive function - :math:`\text{ELU}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` - is defined as: - - .. math:: - \text{ELU}_{\text{adaptive}}({x}) = \alpha\,\text{ELU}(\beta{x}+\gamma), - - where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the - ELU function is defined as: - - .. math:: - \text{ELU}(x) = \begin{cases} - x, & \text{ if }x > 0\\ - \exp(x) - 1, & \text{ if }x \leq 0 - \end{cases} - - .. seealso:: - - **Original reference**: Godfrey, Luke B., and Michael S. Gashler. - *A continuum among logarithmic, linear, and exponential functions, - and its potential to improve generalization in neural networks.* - 2015 7th international joint conference on knowledge discovery, - knowledge engineering and knowledge management (IC3K). - Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. - `_. - - Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive - activation functions accelerate convergence in deep and - physics-informed neural networks*. Journal of - Computational Physics 404 (2020): 109136. - DOI: `JCP 10.1016 - `_. - """ - - def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): - super().__init__(alpha, beta, gamma, fixed) - self._func = torch.nn.ELU() - - -class AdaptiveCELU(AdaptiveActivationFunctionInterface): - r""" - Adaptive trainable :class:`~torch.nn.CELU` activation function. - - Given the function :math:`\text{CELU}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, - the adaptive function - :math:`\text{CELU}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` - is defined as: - - .. math:: - \text{CELU}_{\text{adaptive}}({x})=\alpha\,\text{CELU}(\beta{x}+\gamma), - - where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the - CELU function is defined as: - - .. math:: - \text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x) - 1)) - - .. seealso:: - - **Original reference**: Godfrey, Luke B., and Michael S. Gashler. - *A continuum among logarithmic, linear, and exponential functions, - and its potential to improve generalization in neural networks.* - 2015 7th international joint conference on knowledge discovery, - knowledge engineering and knowledge management (IC3K). - Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. - `_. - - Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive - activation functions accelerate convergence in deep and - physics-informed neural networks*. Journal of - Computational Physics 404 (2020): 109136. - DOI: `JCP 10.1016 - `_. - """ - - def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): - super().__init__(alpha, beta, gamma, fixed) - self._func = torch.nn.CELU() - - -class AdaptiveGELU(AdaptiveActivationFunctionInterface): - r""" - Adaptive trainable :class:`~torch.nn.GELU` activation function. - - Given the function :math:`\text{GELU}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, - the adaptive function - :math:`\text{GELU}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` - is defined as: - - .. math:: - \text{GELU}_{\text{adaptive}}({x})=\alpha\,\text{GELU}(\beta{x}+\gamma), - - where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the - GELU function is defined as: - - .. math:: - \text{GELU}(x)=0.5*x*(1+\text{Tanh}(\sqrt{2 / \pi}*(x+0.044715*x^3))) - - - .. seealso:: - - **Original reference**: Godfrey, Luke B., and Michael S. Gashler. - *A continuum among logarithmic, linear, and exponential functions, - and its potential to improve generalization in neural networks.* - 2015 7th international joint conference on knowledge discovery, - knowledge engineering and knowledge management (IC3K). - Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. - `_. - - Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive - activation functions accelerate convergence in deep and - physics-informed neural networks*. Journal of - Computational Physics 404 (2020): 109136. - DOI: `JCP 10.1016 - `_. - """ - - def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): - super().__init__(alpha, beta, gamma, fixed) - self._func = torch.nn.GELU() - - -class AdaptiveSoftmin(AdaptiveActivationFunctionInterface): - r""" - Adaptive trainable :class:`~torch.nn.Softmin` activation function. - - Given the function - :math:`\text{Softmin}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, - the adaptive function - :math:`\text{Softmin}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` - is defined as: - - .. math:: - \text{Softmin}_{\text{adaptive}}({x})=\alpha\, - \text{Softmin}(\beta{x}+\gamma), - - where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the - Softmin function is defined as: - - .. math:: - \text{Softmin}(x_{i}) = \frac{\exp(-x_i)}{\sum_j \exp(-x_j)} - - .. seealso:: - - **Original reference**: Godfrey, Luke B., and Michael S. Gashler. - *A continuum among logarithmic, linear, and exponential functions, - and its potential to improve generalization in neural networks.* - 2015 7th international joint conference on knowledge discovery, - knowledge engineering and knowledge management (IC3K). - Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. - `_. - - Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive - activation functions accelerate convergence in deep and - physics-informed neural networks*. Journal of - Computational Physics 404 (2020): 109136. - DOI: `JCP 10.1016 - `_. - """ - - def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): - super().__init__(alpha, beta, gamma, fixed) - self._func = torch.nn.Softmin() - - -class AdaptiveSoftmax(AdaptiveActivationFunctionInterface): - r""" - Adaptive trainable :class:`~torch.nn.Softmax` activation function. - - Given the function - :math:`\text{Softmax}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, - the adaptive function - :math:`\text{Softmax}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` - is defined as: - - .. math:: - \text{Softmax}_{\text{adaptive}}({x})=\alpha\, - \text{Softmax}(\beta{x}+\gamma), - - where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the - Softmax function is defined as: - - .. math:: - \text{Softmax}(x_{i}) = \frac{\exp(x_i)}{\sum_j \exp(x_j)} - - .. seealso:: - - **Original reference**: Godfrey, Luke B., and Michael S. Gashler. - *A continuum among logarithmic, linear, and exponential functions, - and its potential to improve generalization in neural networks.* - 2015 7th international joint conference on knowledge discovery, - knowledge engineering and knowledge management (IC3K). - Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. - `_. - - Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive - activation functions accelerate convergence in deep and - physics-informed neural networks*. Journal of - Computational Physics 404 (2020): 109136. - DOI: `JCP 10.1016 - `_. - """ - - def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): - super().__init__(alpha, beta, gamma, fixed) - self._func = torch.nn.Softmax() - - -class AdaptiveSIREN(AdaptiveActivationFunctionInterface): - r""" - Adaptive trainable :obj:`~torch.sin` function. - - Given the function :math:`\text{sin}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, - the adaptive function - :math:`\text{sin}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` - is defined as: - - .. math:: - \text{sin}_{\text{adaptive}}({x}) = \alpha\,\text{sin}(\beta{x}+\gamma), - - where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters. - - .. seealso:: - - **Original reference**: Godfrey, Luke B., and Michael S. Gashler. - *A continuum among logarithmic, linear, and exponential functions, - and its potential to improve generalization in neural networks.* - 2015 7th international joint conference on knowledge discovery, - knowledge engineering and knowledge management (IC3K). - Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. - `_. - - Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive - activation functions accelerate convergence in deep and - physics-informed neural networks*. Journal of - Computational Physics 404 (2020): 109136. - DOI: `JCP 10.1016 - `_. - """ - - def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): - super().__init__(alpha, beta, gamma, fixed) - self._func = torch.sin - - -class AdaptiveExp(AdaptiveActivationFunctionInterface): - r""" - Adaptive trainable :obj:`~torch.exp` function. - - Given the function :math:`\text{exp}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, - the adaptive function - :math:`\text{exp}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` - is defined as: - - .. math:: - \text{exp}_{\text{adaptive}}({x}) = \alpha\,\text{exp}(\beta{x}), - - where :math:`\alpha,\,\beta` are trainable parameters. - - .. seealso:: - - **Original reference**: Godfrey, Luke B., and Michael S. Gashler. - *A continuum among logarithmic, linear, and exponential functions, - and its potential to improve generalization in neural networks.* - 2015 7th international joint conference on knowledge discovery, - knowledge engineering and knowledge management (IC3K). - Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. - `_. - - Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive - activation functions accelerate convergence in deep and - physics-informed neural networks*. Journal of - Computational Physics 404 (2020): 109136. - DOI: `JCP 10.1016 - `_. - """ - - def __init__(self, alpha=None, beta=None, fixed=None): - - # only alpha, and beta parameters (gamma=0 fixed) - if fixed is None: - fixed = ["gamma"] - else: - check_consistency(fixed, str) - fixed = list(fixed) + ["gamma"] - - # calling super - super().__init__(alpha, beta, 0.0, fixed) - self._func = torch.exp diff --git a/pina/_src/adaptive_function/adaptive_function_interface.py b/pina/_src/adaptive_function/adaptive_function_interface.py index d73382cb6..d53694bcd 100644 --- a/pina/_src/adaptive_function/adaptive_function_interface.py +++ b/pina/_src/adaptive_function/adaptive_function_interface.py @@ -1,151 +1,50 @@ -"""Module for the Adaptive Function interface.""" +"""Module for the Adaptive Function Interface.""" -from abc import ABCMeta -import torch -from pina._src.core.utils import check_consistency, is_function +from abc import ABCMeta, abstractmethod -class AdaptiveActivationFunctionInterface(torch.nn.Module, metaclass=ABCMeta): - r""" - The :class:`AdaptiveActivationFunctionInterface` - class makes a :class:`torch.nn.Module` activation function into an adaptive - trainable activation function. If one wants to create an adpative activation - function, this class must be use as base class. - - Given a function :math:`f:\mathbb{R}^n\rightarrow\mathbb{R}^m`, the adaptive - function :math:`f_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^m` - is defined as: - - .. math:: - f_{\text{adaptive}}(\mathbf{x}) = \alpha\,f(\beta\mathbf{x}+\gamma), - - where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters. - - .. seealso:: - - **Original reference**: Godfrey, Luke B., and Michael S. Gashler. - *A continuum among logarithmic, linear, and exponential functions, - and its potential to improve generalization in neural networks.* - 2015 7th international joint conference on knowledge discovery, - knowledge engineering and knowledge management (IC3K). - Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. - `_. - - Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive - activation functions accelerate convergence in deep and - physics-informed neural networks*. Journal of - Computational Physics 404 (2020): 109136. - DOI: `JCP 10.1016 - `_. +class AdaptiveFunctionInterface(metaclass=ABCMeta): + """ + Abstract interface for all adaptive functions. """ - def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): - """ - Initializes the Adaptive Function. - - :param float | complex alpha: Scaling parameter alpha. - Defaults to ``None``. When ``None`` is passed, - the variable is initialized to 1. - :param float | complex beta: Scaling parameter beta. - Defaults to ``None``. When ``None`` is passed, - the variable is initialized to 1. - :param float | complex gamma: Shifting parameter gamma. - Defaults to ``None``. When ``None`` is passed, - the variable is initialized to 1. - :param list fixed: List of parameters to fix during training, - i.e. not optimized (``requires_grad`` set to ``False``). - Options are ``alpha``, ``beta``, ``gamma``. Defaults to None. - """ - super().__init__() - - # see if there are fixed variables - if fixed is not None: - check_consistency(fixed, str) - if not all(key in ["alpha", "beta", "gamma"] for key in fixed): - raise TypeError( - "Fixed keys must be in [`alpha`, `beta`, `gamma`]." - ) - - # initialize alpha, beta, gamma if they are None - if alpha is None: - alpha = 1.0 - if beta is None: - beta = 1.0 - if gamma is None: - gamma = 0.0 - - # checking consistency - check_consistency(alpha, (float, complex)) - check_consistency(beta, (float, complex)) - check_consistency(gamma, (float, complex)) - - # registering as tensors - alpha = torch.tensor(alpha, requires_grad=False) - beta = torch.tensor(beta, requires_grad=False) - gamma = torch.tensor(gamma, requires_grad=False) - - # setting not fixed variables as torch.nn.Parameter with gradient - # registering the buffer for the one which are fixed, buffers by - # default are saved alongside trainable parameters - if "alpha" not in (fixed or []): - self._alpha = torch.nn.Parameter(alpha, requires_grad=True) - else: - self.register_buffer("alpha", alpha) - - if "beta" not in (fixed or []): - self._beta = torch.nn.Parameter(beta, requires_grad=True) - else: - self.register_buffer("beta", beta) - - if "gamma" not in (fixed or []): - self._gamma = torch.nn.Parameter(gamma, requires_grad=True) - else: - self.register_buffer("gamma", gamma) - + @abstractmethod def forward(self, x): """ - Define the computation performed at every call. - The function to the input elementwise. + Compute the transformation of the adaptive function on the input. - :param x: The input tensor to evaluate the activation function. + :param x: The input tensor to evaluate the adaptive function. :type x: torch.Tensor | LabelTensor + :return: The output of the adaptive function. + :rtype: torch.Tensor | LabelTensor """ - return self.alpha * (self._func(self.beta * x + self.gamma)) @property + @abstractmethod def alpha(self): """ - The alpha variable. + The output scaling parameter of the adaptive function. + + :return: The alpha parameter. + :rtype: torch.nn.Parameter | torch.Tensor """ - return self._alpha @property + @abstractmethod def beta(self): """ - The beta variable. - """ - return self._beta + The input scaling parameter of the adaptive function. - @property - def gamma(self): - """ - The gamma variable. + :return: The beta parameter. + :rtype: torch.nn.Parameter | torch.Tensor """ - return self._gamma @property - def func(self): - """ - The callable activation function. + @abstractmethod + def gamma(self): """ - return self._func + The input shifting parameter of the adaptive function. - @func.setter - def func(self, value): - """ - Set the activation function. + :return: The gamma parameter. + :rtype: torch.nn.Parameter | torch.Tensor """ - if not is_function(value): - raise TypeError("The function must be callable.") - self._func = value - return self._func diff --git a/pina/_src/adaptive_function/adaptive_gelu.py b/pina/_src/adaptive_function/adaptive_gelu.py new file mode 100644 index 000000000..148d43d52 --- /dev/null +++ b/pina/_src/adaptive_function/adaptive_gelu.py @@ -0,0 +1,78 @@ +"""Module for the Adaptive GELU activation function.""" + +import torch +from pina._src.adaptive_function.base_adaptive_function import ( + BaseAdaptiveFunction, +) + + +class AdaptiveGELU(BaseAdaptiveFunction): + r""" + Adaptive, trainable variant of the :class:`~torch.nn.GELU` activation. + + This module extends the standard GELU by introducing learnable scaling + and shifting parameters applied to both the input and the output. + + Given the function :math:`\text{GELU}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the corresponding adaptive activation + :math:`\text{GELU}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` is + defined as: + + .. math:: + \text{GELU}_{\text{adaptive}}({x})=\alpha\,\text{GELU}(\beta{x}+\gamma), + + where :math:`\alpha`, :math:`\beta`, and :math:`\gamma` are trainable + parameters controlling output scaling, input scaling, and input shifting, + respectively. + + The GELU function is defined elementwise as: + + .. math:: + \text{GELU}(x)=0.5*x*(1+\text{Tanh}(\sqrt{2 / \pi}*(x+0.044715*x^3))) + + + .. seealso:: + + **Original reference**: Godfrey, L. B., Gashler, M. S. (2015). + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 7th international joint conference on knowledge discovery, knowledge + engineering and knowledge management (IC3K), Vol. 1. + DOI: `arXiv preprint arXiv:1602.01321. + `_. + + **Original reference**: Jagtap, A. D., Karniadakis, G. E. (2020). + *Adaptive activation functions accelerate convergence in deep and + physics-informed neural networks*. + Journal of Computational Physics, 404. + DOI: `JCP 10.1016 `_. + """ + + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + """ + Initialization of the :class:`AdaptiveGELU` class. + + :param alpha: The output scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type alpha: int | float + :param beta: The input scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type beta: int | float + :param gamma: The input shifting parameter of the adaptive function. + If ``None``, it is initialized to ``0``. Default is ``None``. + :type gamma: int | float + :param fixed: The names of parameters to keep fixed during training. + These parameters will not be optimized and will have + ``requires_grad=False``. Available options are ``"alpha"``, + ``"beta"``, and ``"gamma"``. If ``None``, all parameters are + trainable. Default is ``None``. + :type fixed: str | list[str] + :raises ValueError: If alpha, when provided, is not a number. + :raises ValueError: If beta, when provided, is not a number. + :raises ValueError: If gamma, when provided, is not a number. + :raises ValueError: If fixed, when provided, is neither a string nor a + list of strings. + :raises ValueError: If fixed contains invalid parameter names. + """ + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.GELU() diff --git a/pina/_src/adaptive_function/adaptive_mish.py b/pina/_src/adaptive_function/adaptive_mish.py new file mode 100644 index 000000000..1c7278a1e --- /dev/null +++ b/pina/_src/adaptive_function/adaptive_mish.py @@ -0,0 +1,77 @@ +"""Module for the Adaptive Mish activation function.""" + +import torch +from pina._src.adaptive_function.base_adaptive_function import ( + BaseAdaptiveFunction, +) + + +class AdaptiveMish(BaseAdaptiveFunction): + r""" + Adaptive, trainable variant of the :class:`~torch.nn.Mish` activation. + + This module extends the standard Mish by introducing learnable scaling + and shifting parameters applied to both the input and the output. + + Given the function :math:`\text{Mish}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the corresponding adaptive activation + :math:`\text{Mish}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` is + defined as: + + .. math:: + \text{Mish}_{\text{adaptive}}({x})=\alpha\,\text{Mish}(\beta{x}+\gamma), + + where :math:`\alpha`, :math:`\beta`, and :math:`\gamma` are trainable + parameters controlling output scaling, input scaling, and input shifting, + respectively. + + The Mish function is defined elementwise as: + + .. math:: + \text{Mish}(x) = x * \text{Tanh}(x) + + .. seealso:: + + **Original reference**: Godfrey, L. B., Gashler, M. S. (2015). + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 7th international joint conference on knowledge discovery, knowledge + engineering and knowledge management (IC3K), Vol. 1. + DOI: `arXiv preprint arXiv:1602.01321. + `_. + + **Original reference**: Jagtap, A. D., Karniadakis, G. E. (2020). + *Adaptive activation functions accelerate convergence in deep and + physics-informed neural networks*. + Journal of Computational Physics, 404. + DOI: `JCP 10.1016 `_. + """ + + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + """ + Initialization of the :class:`AdaptiveMish` class. + + :param alpha: The output scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type alpha: int | float + :param beta: The input scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type beta: int | float + :param gamma: The input shifting parameter of the adaptive function. + If ``None``, it is initialized to ``0``. Default is ``None``. + :type gamma: int | float + :param fixed: The names of parameters to keep fixed during training. + These parameters will not be optimized and will have + ``requires_grad=False``. Available options are ``"alpha"``, + ``"beta"``, and ``"gamma"``. If ``None``, all parameters are + trainable. Default is ``None``. + :type fixed: str | list[str] + :raises ValueError: If alpha, when provided, is not a number. + :raises ValueError: If beta, when provided, is not a number. + :raises ValueError: If gamma, when provided, is not a number. + :raises ValueError: If fixed, when provided, is neither a string nor a + list of strings. + :raises ValueError: If fixed contains invalid parameter names. + """ + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.Mish() diff --git a/pina/_src/adaptive_function/adaptive_relu.py b/pina/_src/adaptive_function/adaptive_relu.py new file mode 100644 index 000000000..bd8ec0879 --- /dev/null +++ b/pina/_src/adaptive_function/adaptive_relu.py @@ -0,0 +1,78 @@ +"""Module for the Adaptive ReLU activation function.""" + +import torch +from pina._src.adaptive_function.base_adaptive_function import ( + BaseAdaptiveFunction, +) + + +class AdaptiveReLU(BaseAdaptiveFunction): + r""" + Adaptive, trainable variant of the :class:`torch.nn.ReLU` activation. + + This module extends the standard ReLU by introducing learnable scaling + and shifting parameters applied to both the input and the output. + + Given the function :math:`\text{ReLU}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the corresponding adaptive activation + :math:`\text{ReLU}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` is + defined as: + + .. math:: + \text{ReLU}_{\text{adaptive}}(x) = + \alpha \, \text{ReLU}(\beta x + \gamma), + + where :math:`\alpha`, :math:`\beta`, and :math:`\gamma` are trainable + parameters controlling output scaling, input scaling, and input shifting, + respectively. + + The ReLU function is defined elementwise as: + + .. math:: + \text{ReLU}(x) = \max(0, x). + + .. seealso:: + + **Original reference**: Godfrey, L. B., Gashler, M. S. (2015). + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 7th international joint conference on knowledge discovery, knowledge + engineering and knowledge management (IC3K), Vol. 1. + DOI: `arXiv preprint arXiv:1602.01321. + `_. + + **Original reference**: Jagtap, A. D., Karniadakis, G. E. (2020). + *Adaptive activation functions accelerate convergence in deep and + physics-informed neural networks*. + Journal of Computational Physics, 404. + DOI: `JCP 10.1016 `_. + """ + + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + """ + Initialization of the :class:`AdaptiveReLU` class. + + :param alpha: The output scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type alpha: int | float + :param beta: The input scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type beta: int | float + :param gamma: The input shifting parameter of the adaptive function. + If ``None``, it is initialized to ``0``. Default is ``None``. + :type gamma: int | float + :param fixed: The names of parameters to keep fixed during training. + These parameters will not be optimized and will have + ``requires_grad=False``. Available options are ``"alpha"``, + ``"beta"``, and ``"gamma"``. If ``None``, all parameters are + trainable. Default is ``None``. + :type fixed: str | list[str] + :raises ValueError: If alpha, when provided, is not a number. + :raises ValueError: If beta, when provided, is not a number. + :raises ValueError: If gamma, when provided, is not a number. + :raises ValueError: If fixed, when provided, is neither a string nor a + list of strings. + :raises ValueError: If fixed contains invalid parameter names. + """ + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.ReLU() diff --git a/pina/_src/adaptive_function/adaptive_sigmoid.py b/pina/_src/adaptive_function/adaptive_sigmoid.py new file mode 100644 index 000000000..c88eafab2 --- /dev/null +++ b/pina/_src/adaptive_function/adaptive_sigmoid.py @@ -0,0 +1,79 @@ +"""Module for the Adaptive Sigmoid activation function.""" + +import torch +from pina._src.adaptive_function.base_adaptive_function import ( + BaseAdaptiveFunction, +) + + +class AdaptiveSigmoid(BaseAdaptiveFunction): + r""" + Adaptive, trainable variant of the :class:`~torch.nn.Sigmoid` activation. + + This module extends the standard Sigmoid by introducing learnable scaling + and shifting parameters applied to both the input and the output. + + Given the function + :math:`\text{Sigmoid}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, the + corresponding adaptive activation + :math:`\text{Sigmoid}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` + is defined as: + + .. math:: + \text{Sigmoid}_{\text{adaptive}}({x})= + \alpha\,\text{Sigmoid}(\beta{x}+\gamma), + + where :math:`\alpha`, :math:`\beta`, and :math:`\gamma` are trainable + parameters controlling output scaling, input scaling, and input shifting, + respectively. + + The Sigmoid function is defined elementwise as: + + .. math:: + \text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)} + + .. seealso:: + + **Original reference**: Godfrey, L. B., Gashler, M. S. (2015). + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 7th international joint conference on knowledge discovery, knowledge + engineering and knowledge management (IC3K), Vol. 1. + DOI: `arXiv preprint arXiv:1602.01321. + `_. + + **Original reference**: Jagtap, A. D., Karniadakis, G. E. (2020). + *Adaptive activation functions accelerate convergence in deep and + physics-informed neural networks*. + Journal of Computational Physics, 404. + DOI: `JCP 10.1016 `_. + """ + + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + """ + Initialization of the :class:`AdaptiveSigmoid` class. + + :param alpha: The output scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type alpha: int | float + :param beta: The input scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type beta: int | float + :param gamma: The input shifting parameter of the adaptive function. + If ``None``, it is initialized to ``0``. Default is ``None``. + :type gamma: int | float + :param fixed: The names of parameters to keep fixed during training. + These parameters will not be optimized and will have + ``requires_grad=False``. Available options are ``"alpha"``, + ``"beta"``, and ``"gamma"``. If ``None``, all parameters are + trainable. Default is ``None``. + :type fixed: str | list[str] + :raises ValueError: If alpha, when provided, is not a number. + :raises ValueError: If beta, when provided, is not a number. + :raises ValueError: If gamma, when provided, is not a number. + :raises ValueError: If fixed, when provided, is neither a string nor a + list of strings. + :raises ValueError: If fixed contains invalid parameter names. + """ + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.Sigmoid() diff --git a/pina/_src/adaptive_function/adaptive_silu.py b/pina/_src/adaptive_function/adaptive_silu.py new file mode 100644 index 000000000..d35b867a6 --- /dev/null +++ b/pina/_src/adaptive_function/adaptive_silu.py @@ -0,0 +1,79 @@ +"""Module for the Adaptive SiLU activation function.""" + +import torch +from pina._src.adaptive_function.base_adaptive_function import ( + BaseAdaptiveFunction, +) + + +class AdaptiveSiLU(BaseAdaptiveFunction): + r""" + Adaptive, trainable variant of the :class:`~torch.nn.SiLU` activation. + + This module extends the standard SiLU by introducing learnable scaling + and shifting parameters applied to both the input and the output. + + Given the function :math:`\text{SiLU}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the corresponding adaptive activation + :math:`\text{SiLU}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` is + defined as: + + .. math:: + \text{SiLU}_{\text{adaptive}}({x})=\alpha\,\text{SiLU}(\beta{x}+\gamma), + + where :math:`\alpha`, :math:`\beta`, and :math:`\gamma` are trainable + parameters controlling output scaling, input scaling, and input shifting, + respectively. + + The SiLU function is defined elementwise as: + + .. math:: + \text{SiLU}(x) = x * \sigma(x), + + where :math:`\sigma(x)` is the logistic sigmoid function. + + .. seealso:: + + **Original reference**: Godfrey, L. B., Gashler, M. S. (2015). + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 7th international joint conference on knowledge discovery, knowledge + engineering and knowledge management (IC3K), Vol. 1. + DOI: `arXiv preprint arXiv:1602.01321. + `_. + + **Original reference**: Jagtap, A. D., Karniadakis, G. E. (2020). + *Adaptive activation functions accelerate convergence in deep and + physics-informed neural networks*. + Journal of Computational Physics, 404. + DOI: `JCP 10.1016 `_. + """ + + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + """ + Initialization of the :class:`AdaptiveSiLU` class. + + :param alpha: The output scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type alpha: int | float + :param beta: The input scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type beta: int | float + :param gamma: The input shifting parameter of the adaptive function. + If ``None``, it is initialized to ``0``. Default is ``None``. + :type gamma: int | float + :param fixed: The names of parameters to keep fixed during training. + These parameters will not be optimized and will have + ``requires_grad=False``. Available options are ``"alpha"``, + ``"beta"``, and ``"gamma"``. If ``None``, all parameters are + trainable. Default is ``None``. + :type fixed: str | list[str] + :raises ValueError: If alpha, when provided, is not a number. + :raises ValueError: If beta, when provided, is not a number. + :raises ValueError: If gamma, when provided, is not a number. + :raises ValueError: If fixed, when provided, is neither a string nor a + list of strings. + :raises ValueError: If fixed contains invalid parameter names. + """ + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.SiLU() diff --git a/pina/_src/adaptive_function/adaptive_siren.py b/pina/_src/adaptive_function/adaptive_siren.py new file mode 100644 index 000000000..dfb42b4b9 --- /dev/null +++ b/pina/_src/adaptive_function/adaptive_siren.py @@ -0,0 +1,72 @@ +"""Module for the Adaptive SIREN activation function.""" + +import torch +from pina._src.adaptive_function.base_adaptive_function import ( + BaseAdaptiveFunction, +) + + +class AdaptiveSIREN(BaseAdaptiveFunction): + r""" + Adaptive, trainable variant of the :obj:`~torch.sin` activation. + + This module extends the standard SIREN by introducing learnable scaling + and shifting parameters applied to both the input and the output. + + Given the function :math:`\text{sin}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the corresponding adaptive activation + :math:`\text{sin}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` + is defined as: + + .. math:: + \text{sin}_{\text{adaptive}}({x}) = \alpha\,\text{sin}(\beta{x}+\gamma), + + where :math:`\alpha`, :math:`\beta`, and :math:`\gamma` are trainable + parameters controlling output scaling, input scaling, and input shifting, + respectively. + + .. seealso:: + + **Original reference**: Godfrey, L. B., Gashler, M. S. (2015). + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 7th international joint conference on knowledge discovery, knowledge + engineering and knowledge management (IC3K), Vol. 1. + DOI: `arXiv preprint arXiv:1602.01321. + `_. + + **Original reference**: Jagtap, A. D., Karniadakis, G. E. (2020). + *Adaptive activation functions accelerate convergence in deep and + physics-informed neural networks*. + Journal of Computational Physics, 404. + DOI: `JCP 10.1016 `_. + """ + + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + """ + Initialization of the :class:`AdaptiveSIREN` class. + + :param alpha: The output scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type alpha: int | float + :param beta: The input scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type beta: int | float + :param gamma: The input shifting parameter of the adaptive function. + If ``None``, it is initialized to ``0``. Default is ``None``. + :type gamma: int | float + :param fixed: The names of parameters to keep fixed during training. + These parameters will not be optimized and will have + ``requires_grad=False``. Available options are ``"alpha"``, + ``"beta"``, and ``"gamma"``. If ``None``, all parameters are + trainable. Default is ``None``. + :type fixed: str | list[str] + :raises ValueError: If alpha, when provided, is not a number. + :raises ValueError: If beta, when provided, is not a number. + :raises ValueError: If gamma, when provided, is not a number. + :raises ValueError: If fixed, when provided, is neither a string nor a + list of strings. + :raises ValueError: If fixed contains invalid parameter names. + """ + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.sin diff --git a/pina/_src/adaptive_function/adaptive_softmax.py b/pina/_src/adaptive_function/adaptive_softmax.py new file mode 100644 index 000000000..7f2ad156f --- /dev/null +++ b/pina/_src/adaptive_function/adaptive_softmax.py @@ -0,0 +1,79 @@ +"""Module for the Adaptive Softmax activation function.""" + +import torch +from pina._src.adaptive_function.base_adaptive_function import ( + BaseAdaptiveFunction, +) + + +class AdaptiveSoftmax(BaseAdaptiveFunction): + r""" + Adaptive, trainable variant of the :class:`~torch.nn.Softmax` activation. + + This module extends the standard Softmax by introducing learnable scaling + and shifting parameters applied to both the input and the output. + + Given the function + :math:`\text{Softmax}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, the + corresponding adaptive activation + :math:`\text{Softmax}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` + is defined as: + + .. math:: + \text{Softmax}_{\text{adaptive}}({x})=\alpha\, + \text{Softmax}(\beta{x}+\gamma), + + where :math:`\alpha`, :math:`\beta`, and :math:`\gamma` are trainable + parameters controlling output scaling, input scaling, and input shifting, + respectively. + + The Softmax function is defined elementwise as: + + .. math:: + \text{Softmax}(x_i) = \frac{\exp(x_i)}{\sum_j \exp(x_j)} + + .. seealso:: + + **Original reference**: Godfrey, L. B., Gashler, M. S. (2015). + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 7th international joint conference on knowledge discovery, knowledge + engineering and knowledge management (IC3K), Vol. 1. + DOI: `arXiv preprint arXiv:1602.01321. + `_. + + **Original reference**: Jagtap, A. D., Karniadakis, G. E. (2020). + *Adaptive activation functions accelerate convergence in deep and + physics-informed neural networks*. + Journal of Computational Physics, 404. + DOI: `JCP 10.1016 `_. + """ + + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + """ + Initialization of the :class:`AdaptiveSoftmax` class. + + :param alpha: The output scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type alpha: int | float + :param beta: The input scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type beta: int | float + :param gamma: The input shifting parameter of the adaptive function. + If ``None``, it is initialized to ``0``. Default is ``None``. + :type gamma: int | float + :param fixed: The names of parameters to keep fixed during training. + These parameters will not be optimized and will have + ``requires_grad=False``. Available options are ``"alpha"``, + ``"beta"``, and ``"gamma"``. If ``None``, all parameters are + trainable. Default is ``None``. + :type fixed: str | list[str] + :raises ValueError: If alpha, when provided, is not a number. + :raises ValueError: If beta, when provided, is not a number. + :raises ValueError: If gamma, when provided, is not a number. + :raises ValueError: If fixed, when provided, is neither a string nor a + list of strings. + :raises ValueError: If fixed contains invalid parameter names. + """ + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.Softmax(dim=-1) diff --git a/pina/_src/adaptive_function/adaptive_softmin.py b/pina/_src/adaptive_function/adaptive_softmin.py new file mode 100644 index 000000000..b07e27bbf --- /dev/null +++ b/pina/_src/adaptive_function/adaptive_softmin.py @@ -0,0 +1,79 @@ +"""Module for the Adaptive Softmin activation function.""" + +import torch +from pina._src.adaptive_function.base_adaptive_function import ( + BaseAdaptiveFunction, +) + + +class AdaptiveSoftmin(BaseAdaptiveFunction): + r""" + Adaptive, trainable variant of the :class:`~torch.nn.Softmin` activation. + + This module extends the standard Softmin by introducing learnable scaling + and shifting parameters applied to both the input and the output. + + Given the function + :math:`\text{Softmin}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, the + corresponding adaptive activation + :math:`\text{Softmin}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` + is defined as: + + .. math:: + \text{Softmin}_{\text{adaptive}}({x})=\alpha\, + \text{Softmin}(\beta{x}+\gamma), + + where :math:`\alpha`, :math:`\beta`, and :math:`\gamma` are trainable + parameters controlling output scaling, input scaling, and input shifting, + respectively. + + The Softmin function is defined elementwise as: + + .. math:: + \text{Softmin}(x_i) = \frac{\exp(-x_i)}{\sum_j \exp(-x_j)} + + .. seealso:: + + **Original reference**: Godfrey, L. B., Gashler, M. S. (2015). + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 7th international joint conference on knowledge discovery, knowledge + engineering and knowledge management (IC3K), Vol. 1. + DOI: `arXiv preprint arXiv:1602.01321. + `_. + + **Original reference**: Jagtap, A. D., Karniadakis, G. E. (2020). + *Adaptive activation functions accelerate convergence in deep and + physics-informed neural networks*. + Journal of Computational Physics, 404. + DOI: `JCP 10.1016 `_. + """ + + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + """ + Initialization of the :class:`AdaptiveSoftmin` class. + + :param alpha: The output scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type alpha: int | float + :param beta: The input scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type beta: int | float + :param gamma: The input shifting parameter of the adaptive function. + If ``None``, it is initialized to ``0``. Default is ``None``. + :type gamma: int | float + :param fixed: The names of parameters to keep fixed during training. + These parameters will not be optimized and will have + ``requires_grad=False``. Available options are ``"alpha"``, + ``"beta"``, and ``"gamma"``. If ``None``, all parameters are + trainable. Default is ``None``. + :type fixed: str | list[str] + :raises ValueError: If alpha, when provided, is not a number. + :raises ValueError: If beta, when provided, is not a number. + :raises ValueError: If gamma, when provided, is not a number. + :raises ValueError: If fixed, when provided, is neither a string nor a + list of strings. + :raises ValueError: If fixed contains invalid parameter names. + """ + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.Softmin(dim=-1) diff --git a/pina/_src/adaptive_function/adaptive_tanh.py b/pina/_src/adaptive_function/adaptive_tanh.py new file mode 100644 index 000000000..513f4b3f0 --- /dev/null +++ b/pina/_src/adaptive_function/adaptive_tanh.py @@ -0,0 +1,72 @@ +"""Module for the Adaptive Tanh activation function.""" + +import torch +from pina._src.adaptive_function.base_adaptive_function import ( + BaseAdaptiveFunction, +) + + +class AdaptiveTanh(BaseAdaptiveFunction): + r""" + Adaptive, trainable variant of the :class:`~torch.nn.Tanh` activation. + + This module extends the standard Tanh by introducing learnable scaling + and shifting parameters applied to both the input and the output. + + Given the function :math:`\text{Tanh}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the corresponding adaptive activation + :math:`\text{Tanh}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` is + defined as: + + .. math:: + \text{Tanh}_{\text{adaptive}}({x})=\alpha\,\text{Tanh}(\beta{x}+\gamma), + + where :math:`\alpha`, :math:`\beta`, and :math:`\gamma` are trainable + parameters controlling output scaling, input scaling, and input shifting, + respectively. + + .. seealso:: + + **Original reference**: Godfrey, L. B., Gashler, M. S. (2015). + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 7th international joint conference on knowledge discovery, knowledge + engineering and knowledge management (IC3K), Vol. 1. + DOI: `arXiv preprint arXiv:1602.01321. + `_. + + **Original reference**: Jagtap, A. D., Karniadakis, G. E. (2020). + *Adaptive activation functions accelerate convergence in deep and + physics-informed neural networks*. + Journal of Computational Physics, 404. + DOI: `JCP 10.1016 `_. + """ + + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + """ + Initialization of the :class:`AdaptiveTanh` class. + + :param alpha: The output scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type alpha: int | float + :param beta: The input scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type beta: int | float + :param gamma: The input shifting parameter of the adaptive function. + If ``None``, it is initialized to ``0``. Default is ``None``. + :type gamma: int | float + :param fixed: The names of parameters to keep fixed during training. + These parameters will not be optimized and will have + ``requires_grad=False``. Available options are ``"alpha"``, + ``"beta"``, and ``"gamma"``. If ``None``, all parameters are + trainable. Default is ``None``. + :type fixed: str | list[str] + :raises ValueError: If alpha, when provided, is not a number. + :raises ValueError: If beta, when provided, is not a number. + :raises ValueError: If gamma, when provided, is not a number. + :raises ValueError: If fixed, when provided, is neither a string nor a + list of strings. + :raises ValueError: If fixed contains invalid parameter names. + """ + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.Tanh() diff --git a/pina/_src/adaptive_function/base_adaptive_function.py b/pina/_src/adaptive_function/base_adaptive_function.py new file mode 100644 index 000000000..c391d308a --- /dev/null +++ b/pina/_src/adaptive_function/base_adaptive_function.py @@ -0,0 +1,178 @@ +"""Module for the Adaptive Function base class.""" + +import torch +from pina._src.core.utils import check_consistency +from pina._src.adaptive_function.adaptive_function_interface import ( + AdaptiveFunctionInterface, +) + + +class BaseAdaptiveFunction(torch.nn.Module, AdaptiveFunctionInterface): + r""" + Base class for all adaptive functions, implementing common functionality. + + This class extends a standard :class:`torch.nn.Module` activation function + into a trainable adaptive form. It implements the common mechanism used to + scale and shift both the input and the output of a given activation + function. + + Given a function :math:`f:\mathbb{R}^n\rightarrow\mathbb{R}^m`, the adaptive + function :math:`f_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^m` + is defined as: + + .. math:: + f_{\text{adaptive}}(\mathbf{x}) = \alpha\,f(\beta\mathbf{x}+\gamma), + + where :math:`\alpha`, :math:`\beta`, and :math:`\gamma` are learnable + parameters controlling output scaling, input scaling, and input shifting, + respectively. + + All specific adaptive functions should inherit from this class and implement + the abstract methods declared in the interface. + + This class is not meant to be instantiated directly. + + .. seealso:: + + **Original reference**: Godfrey, L. B., Gashler, M. S. (2015). + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 7th international joint conference on knowledge discovery, knowledge + engineering and knowledge management (IC3K), Vol. 1. + DOI: `arXiv preprint arXiv:1602.01321. + `_. + + **Original reference**: Jagtap, A. D., Karniadakis, G. E. (2020). + *Adaptive activation functions accelerate convergence in deep and + physics-informed neural networks*. + Journal of Computational Physics, 404. + DOI: `JCP 10.1016 `_. + """ + + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + """ + Initialization of the :class:`BaseAdaptiveFunction` class. + + :param alpha: The output scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type alpha: int | float + :param beta: The input scaling parameter of the adaptive function. + If ``None``, it is initialized to ``1``. Default is ``None``. + :type beta: int | float + :param gamma: The input shifting parameter of the adaptive function. + If ``None``, it is initialized to ``0``. Default is ``None``. + :type gamma: int | float + :param fixed: The names of parameters to keep fixed during training. + These parameters will not be optimized and will have + ``requires_grad=False``. Available options are ``"alpha"``, + ``"beta"``, and ``"gamma"``. If ``None``, all parameters are + trainable. Default is ``None``. + :type fixed: str | list[str] + :raises ValueError: If alpha, when provided, is not a number. + :raises ValueError: If beta, when provided, is not a number. + :raises ValueError: If gamma, when provided, is not a number. + :raises ValueError: If fixed, when provided, is neither a string nor a + list of strings. + :raises ValueError: If fixed contains invalid parameter names. + """ + super().__init__() + + # Set default values for alpha, beta, gamma if they are None + alpha = 1.0 if alpha is None else alpha + beta = 1.0 if beta is None else beta + gamma = 0.0 if gamma is None else gamma + + # Check consistency + check_consistency(alpha, (int, float)) + check_consistency(beta, (int, float)) + check_consistency(gamma, (int, float)) + + # Process fixed parameters + if fixed is not None: + check_consistency(fixed, str) + fixed = {fixed} if isinstance(fixed, str) else set(fixed) + else: + fixed = set() + + # Validate fixed parameter names + invalid_names = fixed - {"alpha", "beta", "gamma"} + if invalid_names: + raise ValueError( + f"Invalid fixed parameter name(s): {sorted(invalid_names)}. " + "Available options are 'alpha', 'beta', and 'gamma'." + ) + + # Register either a trainable parameter or a fixed buffer + def _register_adaptive_param(name, value): + """ + Helper function to register an adaptive parameter as either a + trainable parameter or a fixed buffer, depending on whether it is + specified in the ``fixed`` argument. + """ + # Convert value to tensor + tensor = torch.tensor(value, dtype=torch.float32) + + # Register as buffer if fixed, otherwise as parameter + if name in fixed: + self.register_buffer(f"_{name}", tensor) + else: + setattr(self, f"_{name}", torch.nn.Parameter(tensor)) + + # Register parameters + _register_adaptive_param("alpha", alpha) + _register_adaptive_param("beta", beta) + _register_adaptive_param("gamma", gamma) + + # Initialize the adaptive function to None, to be set by subclasses + self._func = None + + def forward(self, x): + """ + Compute the transformation of the adaptive function on the input. + + :param x: The input tensor to evaluate the adaptive function. + :type x: torch.Tensor | LabelTensor + :raises RuntimeError: If the adaptive function has not been set. + :raises RuntimeError: If the adaptive function is not callable. + :return: The output of the adaptive function. + :rtype: torch.Tensor | LabelTensor + """ + # Raise an error if the adaptive function has not been set + if self._func is None: + raise RuntimeError("The adaptive function has not been set.") + + # Raise an error if the adaptive function is not callable + if not callable(self._func): + raise RuntimeError("The adaptive function is not callable.") + + return self.alpha * (self._func(self.beta * x + self.gamma)) + + @property + def alpha(self): + """ + The output scaling parameter of the adaptive function. + + :return: The alpha parameter. + :rtype: torch.nn.Parameter | torch.Tensor + """ + return self._alpha + + @property + def beta(self): + """ + The input scaling parameter of the adaptive function. + + :return: The beta parameter. + :rtype: torch.nn.Parameter | torch.Tensor + """ + return self._beta + + @property + def gamma(self): + """ + The input shifting parameter of the adaptive function. + + :return: The gamma parameter. + :rtype: torch.nn.Parameter | torch.Tensor + """ + return self._gamma diff --git a/pina/adaptive_function/__init__.py b/pina/adaptive_function/__init__.py index 9047be94a..d41f25ccd 100644 --- a/pina/adaptive_function/__init__.py +++ b/pina/adaptive_function/__init__.py @@ -7,35 +7,37 @@ """ __all__ = [ - "AdaptiveActivationFunctionInterface", + "AdaptiveFunctionInterface", + "BaseAdaptiveFunction", + "AdaptiveCELU", + "AdaptiveELU", + "AdaptiveExp", + "AdaptiveGELU", + "AdaptiveMish", "AdaptiveReLU", "AdaptiveSigmoid", - "AdaptiveTanh", "AdaptiveSiLU", - "AdaptiveMish", - "AdaptiveELU", - "AdaptiveCELU", - "AdaptiveGELU", - "AdaptiveSoftmin", - "AdaptiveSoftmax", "AdaptiveSIREN", - "AdaptiveExp", + "AdaptiveSoftmax", + "AdaptiveSoftmin", + "AdaptiveTanh", ] -from pina._src.adaptive_function.adaptive_function import ( - AdaptiveReLU, - AdaptiveSigmoid, - AdaptiveTanh, - AdaptiveSiLU, - AdaptiveMish, - AdaptiveELU, - AdaptiveCELU, - AdaptiveGELU, - AdaptiveSoftmin, - AdaptiveSoftmax, - AdaptiveSIREN, - AdaptiveExp, -) from pina._src.adaptive_function.adaptive_function_interface import ( - AdaptiveActivationFunctionInterface, + AdaptiveFunctionInterface, +) +from pina._src.adaptive_function.base_adaptive_function import ( + BaseAdaptiveFunction, ) +from pina._src.adaptive_function.adaptive_celu import AdaptiveCELU +from pina._src.adaptive_function.adaptive_elu import AdaptiveELU +from pina._src.adaptive_function.adaptive_exp import AdaptiveExp +from pina._src.adaptive_function.adaptive_gelu import AdaptiveGELU +from pina._src.adaptive_function.adaptive_mish import AdaptiveMish +from pina._src.adaptive_function.adaptive_relu import AdaptiveReLU +from pina._src.adaptive_function.adaptive_sigmoid import AdaptiveSigmoid +from pina._src.adaptive_function.adaptive_silu import AdaptiveSiLU +from pina._src.adaptive_function.adaptive_siren import AdaptiveSIREN +from pina._src.adaptive_function.adaptive_softmax import AdaptiveSoftmax +from pina._src.adaptive_function.adaptive_softmin import AdaptiveSoftmin +from pina._src.adaptive_function.adaptive_tanh import AdaptiveTanh diff --git a/tests/test_adaptive_function.py b/tests/test_adaptive_function.py deleted file mode 100644 index fae547ffb..000000000 --- a/tests/test_adaptive_function.py +++ /dev/null @@ -1,84 +0,0 @@ -import torch -import pytest - -from pina.adaptive_function import ( - AdaptiveReLU, - AdaptiveSigmoid, - AdaptiveTanh, - AdaptiveSiLU, - AdaptiveMish, - AdaptiveELU, - AdaptiveCELU, - AdaptiveGELU, - AdaptiveSoftmin, - AdaptiveSoftmax, - AdaptiveSIREN, - AdaptiveExp, -) - -adaptive_function = ( - AdaptiveReLU, - AdaptiveSigmoid, - AdaptiveTanh, - AdaptiveSiLU, - AdaptiveMish, - AdaptiveELU, - AdaptiveCELU, - AdaptiveGELU, - AdaptiveSoftmin, - AdaptiveSoftmax, - AdaptiveSIREN, - AdaptiveExp, -) -x = torch.rand(10, requires_grad=True) - - -@pytest.mark.parametrize("Func", adaptive_function) -def test_constructor(Func): - if Func.__name__ == "AdaptiveExp": - # simple - Func() - # setting values - af = Func(alpha=1.0, beta=2.0) - assert af.alpha.requires_grad - assert af.beta.requires_grad - assert af.alpha == 1.0 - assert af.beta == 2.0 - else: - # simple - Func() - # setting values - af = Func(alpha=1.0, beta=2.0, gamma=3.0) - assert af.alpha.requires_grad - assert af.beta.requires_grad - assert af.gamma.requires_grad - assert af.alpha == 1.0 - assert af.beta == 2.0 - assert af.gamma == 3.0 - - # fixed variables - af = Func(alpha=1.0, beta=2.0, fixed=["alpha"]) - assert af.alpha.requires_grad is False - assert af.beta.requires_grad - assert af.alpha == 1.0 - assert af.beta == 2.0 - - with pytest.raises(TypeError): - Func(alpha=1.0, beta=2.0, fixed=["delta"]) - - with pytest.raises(ValueError): - Func(alpha="s") - Func(alpha=1) - - -@pytest.mark.parametrize("Func", adaptive_function) -def test_forward(Func): - af = Func() - af(x) - - -@pytest.mark.parametrize("Func", adaptive_function) -def test_backward(Func): - af = Func() - y = af(x) - y.mean().backward() diff --git a/tests/test_adaptive_function/test_adaptive_celu.py b/tests/test_adaptive_function/test_adaptive_celu.py new file mode 100644 index 000000000..171845d61 --- /dev/null +++ b/tests/test_adaptive_function/test_adaptive_celu.py @@ -0,0 +1,89 @@ +import torch +import pytest +from pina import LabelTensor +from pina.adaptive_function import AdaptiveCELU + + +x = LabelTensor(torch.rand(10, 1), labels=["x"]) +fixed_variables = [None, "alpha", ["beta", "gamma"], ["alpha", "beta", "gamma"]] +param_names = ["alpha", "beta", "gamma"] + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_constructor(alpha, beta, gamma, fixed): + + # Construct the adaptive activation function with the specified parameters + activation = AdaptiveCELU(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + + # Build expected values for parameters + expected = {"alpha": alpha, "beta": beta, "gamma": gamma} + if fixed is None: + fixed_set = set() + else: + fixed_set = {fixed} if isinstance(fixed, str) else set(fixed) + + # Verify fixed parameters consistency + for v in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is False + assert param.item() == expected[v] + + # Verify trainable parameters consistency + for v in param_names: + if v not in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is True + assert param.item() == expected[v] + + # Should fail if alpha is not a number + with pytest.raises(ValueError): + AdaptiveCELU(alpha="s") + + # Should fail if beta is not a number + with pytest.raises(ValueError): + AdaptiveCELU(beta="s") + + # Should fail if gamma is not a number + with pytest.raises(ValueError): + AdaptiveCELU(gamma="s") + + # Should fail if fixed is not a string or list of strings + with pytest.raises(ValueError): + AdaptiveCELU(fixed=123) + + # Should fail if fixed contains invalid parameter names + with pytest.raises(ValueError): + AdaptiveCELU(fixed="delta") + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_forward(alpha, beta, gamma, fixed): + + # Compute the output + activation = AdaptiveCELU(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x) + + # Verify the output shape is the same as the input shape + assert output_.shape == x.shape + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_backward(alpha, beta, gamma, fixed): + + # Compute the output and perform backpropagation + activation = AdaptiveCELU(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x.requires_grad_()) + loss = torch.mean(output_) + loss.backward() + + # Verify that the gradients shape is the same as the input shape + assert x.grad.shape == x.shape diff --git a/tests/test_adaptive_function/test_adaptive_elu.py b/tests/test_adaptive_function/test_adaptive_elu.py new file mode 100644 index 000000000..7c231d8da --- /dev/null +++ b/tests/test_adaptive_function/test_adaptive_elu.py @@ -0,0 +1,89 @@ +import torch +import pytest +from pina import LabelTensor +from pina.adaptive_function import AdaptiveELU + + +x = LabelTensor(torch.rand(10, 1), labels=["x"]) +fixed_variables = [None, "alpha", ["beta", "gamma"], ["alpha", "beta", "gamma"]] +param_names = ["alpha", "beta", "gamma"] + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_constructor(alpha, beta, gamma, fixed): + + # Construct the adaptive activation function with the specified parameters + activation = AdaptiveELU(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + + # Build expected values for parameters + expected = {"alpha": alpha, "beta": beta, "gamma": gamma} + if fixed is None: + fixed_set = set() + else: + fixed_set = {fixed} if isinstance(fixed, str) else set(fixed) + + # Verify fixed parameters consistency + for v in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is False + assert param.item() == expected[v] + + # Verify trainable parameters consistency + for v in param_names: + if v not in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is True + assert param.item() == expected[v] + + # Should fail if alpha is not a number + with pytest.raises(ValueError): + AdaptiveELU(alpha="s") + + # Should fail if beta is not a number + with pytest.raises(ValueError): + AdaptiveELU(beta="s") + + # Should fail if gamma is not a number + with pytest.raises(ValueError): + AdaptiveELU(gamma="s") + + # Should fail if fixed is not a string or list of strings + with pytest.raises(ValueError): + AdaptiveELU(fixed=123) + + # Should fail if fixed contains invalid parameter names + with pytest.raises(ValueError): + AdaptiveELU(fixed="delta") + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_forward(alpha, beta, gamma, fixed): + + # Compute the output + activation = AdaptiveELU(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x) + + # Verify the output shape is the same as the input shape + assert output_.shape == x.shape + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_backward(alpha, beta, gamma, fixed): + + # Compute the output and perform backpropagation + activation = AdaptiveELU(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x.requires_grad_()) + loss = torch.mean(output_) + loss.backward() + + # Verify that the gradients shape is the same as the input shape + assert x.grad.shape == x.shape diff --git a/tests/test_adaptive_function/test_adaptive_exp.py b/tests/test_adaptive_function/test_adaptive_exp.py new file mode 100644 index 000000000..5bad49b00 --- /dev/null +++ b/tests/test_adaptive_function/test_adaptive_exp.py @@ -0,0 +1,89 @@ +import torch +import pytest +from pina import LabelTensor +from pina.adaptive_function import AdaptiveExp + + +x = LabelTensor(torch.rand(10, 1), labels=["x"]) +fixed_variables = [None, "alpha", ["beta", "gamma"], ["alpha", "beta", "gamma"]] +param_names = ["alpha", "beta", "gamma"] + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_constructor(alpha, beta, gamma, fixed): + + # Construct the adaptive activation function with the specified parameters + activation = AdaptiveExp(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + + # Build expected values for parameters + expected = {"alpha": alpha, "beta": beta, "gamma": gamma} + if fixed is None: + fixed_set = set() + else: + fixed_set = {fixed} if isinstance(fixed, str) else set(fixed) + + # Verify fixed parameters consistency + for v in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is False + assert param.item() == expected[v] + + # Verify trainable parameters consistency + for v in param_names: + if v not in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is True + assert param.item() == expected[v] + + # Should fail if alpha is not a number + with pytest.raises(ValueError): + AdaptiveExp(alpha="s") + + # Should fail if beta is not a number + with pytest.raises(ValueError): + AdaptiveExp(beta="s") + + # Should fail if gamma is not a number + with pytest.raises(ValueError): + AdaptiveExp(gamma="s") + + # Should fail if fixed is not a string or list of strings + with pytest.raises(ValueError): + AdaptiveExp(fixed=123) + + # Should fail if fixed contains invalid parameter names + with pytest.raises(ValueError): + AdaptiveExp(fixed="delta") + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_forward(alpha, beta, gamma, fixed): + + # Compute the output + activation = AdaptiveExp(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x) + + # Verify the output shape is the same as the input shape + assert output_.shape == x.shape + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_backward(alpha, beta, gamma, fixed): + + # Compute the output and perform backpropagation + activation = AdaptiveExp(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x.requires_grad_()) + loss = torch.mean(output_) + loss.backward() + + # Verify that the gradients shape is the same as the input shape + assert x.grad.shape == x.shape diff --git a/tests/test_adaptive_function/test_adaptive_gelu.py b/tests/test_adaptive_function/test_adaptive_gelu.py new file mode 100644 index 000000000..53c4af8d1 --- /dev/null +++ b/tests/test_adaptive_function/test_adaptive_gelu.py @@ -0,0 +1,89 @@ +import torch +import pytest +from pina import LabelTensor +from pina.adaptive_function import AdaptiveGELU + + +x = LabelTensor(torch.rand(10, 1), labels=["x"]) +fixed_variables = [None, "alpha", ["beta", "gamma"], ["alpha", "beta", "gamma"]] +param_names = ["alpha", "beta", "gamma"] + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_constructor(alpha, beta, gamma, fixed): + + # Construct the adaptive activation function with the specified parameters + activation = AdaptiveGELU(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + + # Build expected values for parameters + expected = {"alpha": alpha, "beta": beta, "gamma": gamma} + if fixed is None: + fixed_set = set() + else: + fixed_set = {fixed} if isinstance(fixed, str) else set(fixed) + + # Verify fixed parameters consistency + for v in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is False + assert param.item() == expected[v] + + # Verify trainable parameters consistency + for v in param_names: + if v not in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is True + assert param.item() == expected[v] + + # Should fail if alpha is not a number + with pytest.raises(ValueError): + AdaptiveGELU(alpha="s") + + # Should fail if beta is not a number + with pytest.raises(ValueError): + AdaptiveGELU(beta="s") + + # Should fail if gamma is not a number + with pytest.raises(ValueError): + AdaptiveGELU(gamma="s") + + # Should fail if fixed is not a string or list of strings + with pytest.raises(ValueError): + AdaptiveGELU(fixed=123) + + # Should fail if fixed contains invalid parameter names + with pytest.raises(ValueError): + AdaptiveGELU(fixed="delta") + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_forward(alpha, beta, gamma, fixed): + + # Compute the output + activation = AdaptiveGELU(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x) + + # Verify the output shape is the same as the input shape + assert output_.shape == x.shape + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_backward(alpha, beta, gamma, fixed): + + # Compute the output and perform backpropagation + activation = AdaptiveGELU(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x.requires_grad_()) + loss = torch.mean(output_) + loss.backward() + + # Verify that the gradients shape is the same as the input shape + assert x.grad.shape == x.shape diff --git a/tests/test_adaptive_function/test_adaptive_mish.py b/tests/test_adaptive_function/test_adaptive_mish.py new file mode 100644 index 000000000..218438337 --- /dev/null +++ b/tests/test_adaptive_function/test_adaptive_mish.py @@ -0,0 +1,89 @@ +import torch +import pytest +from pina import LabelTensor +from pina.adaptive_function import AdaptiveMish + + +x = LabelTensor(torch.rand(10, 1), labels=["x"]) +fixed_variables = [None, "alpha", ["beta", "gamma"], ["alpha", "beta", "gamma"]] +param_names = ["alpha", "beta", "gamma"] + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_constructor(alpha, beta, gamma, fixed): + + # Construct the adaptive activation function with the specified parameters + activation = AdaptiveMish(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + + # Build expected values for parameters + expected = {"alpha": alpha, "beta": beta, "gamma": gamma} + if fixed is None: + fixed_set = set() + else: + fixed_set = {fixed} if isinstance(fixed, str) else set(fixed) + + # Verify fixed parameters consistency + for v in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is False + assert param.item() == expected[v] + + # Verify trainable parameters consistency + for v in param_names: + if v not in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is True + assert param.item() == expected[v] + + # Should fail if alpha is not a number + with pytest.raises(ValueError): + AdaptiveMish(alpha="s") + + # Should fail if beta is not a number + with pytest.raises(ValueError): + AdaptiveMish(beta="s") + + # Should fail if gamma is not a number + with pytest.raises(ValueError): + AdaptiveMish(gamma="s") + + # Should fail if fixed is not a string or list of strings + with pytest.raises(ValueError): + AdaptiveMish(fixed=123) + + # Should fail if fixed contains invalid parameter names + with pytest.raises(ValueError): + AdaptiveMish(fixed="delta") + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_forward(alpha, beta, gamma, fixed): + + # Compute the output + activation = AdaptiveMish(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x) + + # Verify the output shape is the same as the input shape + assert output_.shape == x.shape + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_backward(alpha, beta, gamma, fixed): + + # Compute the output and perform backpropagation + activation = AdaptiveMish(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x.requires_grad_()) + loss = torch.mean(output_) + loss.backward() + + # Verify that the gradients shape is the same as the input shape + assert x.grad.shape == x.shape diff --git a/tests/test_adaptive_function/test_adaptive_relu.py b/tests/test_adaptive_function/test_adaptive_relu.py new file mode 100644 index 000000000..02f8c7ff5 --- /dev/null +++ b/tests/test_adaptive_function/test_adaptive_relu.py @@ -0,0 +1,89 @@ +import torch +import pytest +from pina import LabelTensor +from pina.adaptive_function import AdaptiveReLU + + +x = LabelTensor(torch.rand(10, 1), labels=["x"]) +fixed_variables = [None, "alpha", ["beta", "gamma"], ["alpha", "beta", "gamma"]] +param_names = ["alpha", "beta", "gamma"] + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_constructor(alpha, beta, gamma, fixed): + + # Construct the adaptive activation function with the specified parameters + activation = AdaptiveReLU(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + + # Build expected values for parameters + expected = {"alpha": alpha, "beta": beta, "gamma": gamma} + if fixed is None: + fixed_set = set() + else: + fixed_set = {fixed} if isinstance(fixed, str) else set(fixed) + + # Verify fixed parameters consistency + for v in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is False + assert param.item() == expected[v] + + # Verify trainable parameters consistency + for v in param_names: + if v not in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is True + assert param.item() == expected[v] + + # Should fail if alpha is not a number + with pytest.raises(ValueError): + AdaptiveReLU(alpha="s") + + # Should fail if beta is not a number + with pytest.raises(ValueError): + AdaptiveReLU(beta="s") + + # Should fail if gamma is not a number + with pytest.raises(ValueError): + AdaptiveReLU(gamma="s") + + # Should fail if fixed is not a string or list of strings + with pytest.raises(ValueError): + AdaptiveReLU(fixed=123) + + # Should fail if fixed contains invalid parameter names + with pytest.raises(ValueError): + AdaptiveReLU(fixed="delta") + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_forward(alpha, beta, gamma, fixed): + + # Compute the output + activation = AdaptiveReLU(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x) + + # Verify the output shape is the same as the input shape + assert output_.shape == x.shape + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_backward(alpha, beta, gamma, fixed): + + # Compute the output and perform backpropagation + activation = AdaptiveReLU(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x.requires_grad_()) + loss = torch.mean(output_) + loss.backward() + + # Verify that the gradients shape is the same as the input shape + assert x.grad.shape == x.shape diff --git a/tests/test_adaptive_function/test_adaptive_sigmoid.py b/tests/test_adaptive_function/test_adaptive_sigmoid.py new file mode 100644 index 000000000..0d5bb7c08 --- /dev/null +++ b/tests/test_adaptive_function/test_adaptive_sigmoid.py @@ -0,0 +1,95 @@ +import torch +import pytest +from pina import LabelTensor +from pina.adaptive_function import AdaptiveSigmoid + + +x = LabelTensor(torch.rand(10, 1), labels=["x"]) +fixed_variables = [None, "alpha", ["beta", "gamma"], ["alpha", "beta", "gamma"]] +param_names = ["alpha", "beta", "gamma"] + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_constructor(alpha, beta, gamma, fixed): + + # Construct the adaptive activation function with the specified parameters + activation = AdaptiveSigmoid( + alpha=alpha, beta=beta, gamma=gamma, fixed=fixed + ) + + # Build expected values for parameters + expected = {"alpha": alpha, "beta": beta, "gamma": gamma} + if fixed is None: + fixed_set = set() + else: + fixed_set = {fixed} if isinstance(fixed, str) else set(fixed) + + # Verify fixed parameters consistency + for v in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is False + assert param.item() == expected[v] + + # Verify trainable parameters consistency + for v in param_names: + if v not in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is True + assert param.item() == expected[v] + + # Should fail if alpha is not a number + with pytest.raises(ValueError): + AdaptiveSigmoid(alpha="s") + + # Should fail if beta is not a number + with pytest.raises(ValueError): + AdaptiveSigmoid(beta="s") + + # Should fail if gamma is not a number + with pytest.raises(ValueError): + AdaptiveSigmoid(gamma="s") + + # Should fail if fixed is not a string or list of strings + with pytest.raises(ValueError): + AdaptiveSigmoid(fixed=123) + + # Should fail if fixed contains invalid parameter names + with pytest.raises(ValueError): + AdaptiveSigmoid(fixed="delta") + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_forward(alpha, beta, gamma, fixed): + + # Compute the output + activation = AdaptiveSigmoid( + alpha=alpha, beta=beta, gamma=gamma, fixed=fixed + ) + output_ = activation(x) + + # Verify the output shape is the same as the input shape + assert output_.shape == x.shape + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_backward(alpha, beta, gamma, fixed): + + # Compute the output and perform backpropagation + activation = AdaptiveSigmoid( + alpha=alpha, beta=beta, gamma=gamma, fixed=fixed + ) + output_ = activation(x.requires_grad_()) + loss = torch.mean(output_) + loss.backward() + + # Verify that the gradients shape is the same as the input shape + assert x.grad.shape == x.shape diff --git a/tests/test_adaptive_function/test_adaptive_silu.py b/tests/test_adaptive_function/test_adaptive_silu.py new file mode 100644 index 000000000..56ed77cbb --- /dev/null +++ b/tests/test_adaptive_function/test_adaptive_silu.py @@ -0,0 +1,89 @@ +import torch +import pytest +from pina import LabelTensor +from pina.adaptive_function import AdaptiveSiLU + + +x = LabelTensor(torch.rand(10, 1), labels=["x"]) +fixed_variables = [None, "alpha", ["beta", "gamma"], ["alpha", "beta", "gamma"]] +param_names = ["alpha", "beta", "gamma"] + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_constructor(alpha, beta, gamma, fixed): + + # Construct the adaptive activation function with the specified parameters + activation = AdaptiveSiLU(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + + # Build expected values for parameters + expected = {"alpha": alpha, "beta": beta, "gamma": gamma} + if fixed is None: + fixed_set = set() + else: + fixed_set = {fixed} if isinstance(fixed, str) else set(fixed) + + # Verify fixed parameters consistency + for v in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is False + assert param.item() == expected[v] + + # Verify trainable parameters consistency + for v in param_names: + if v not in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is True + assert param.item() == expected[v] + + # Should fail if alpha is not a number + with pytest.raises(ValueError): + AdaptiveSiLU(alpha="s") + + # Should fail if beta is not a number + with pytest.raises(ValueError): + AdaptiveSiLU(beta="s") + + # Should fail if gamma is not a number + with pytest.raises(ValueError): + AdaptiveSiLU(gamma="s") + + # Should fail if fixed is not a string or list of strings + with pytest.raises(ValueError): + AdaptiveSiLU(fixed=123) + + # Should fail if fixed contains invalid parameter names + with pytest.raises(ValueError): + AdaptiveSiLU(fixed="delta") + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_forward(alpha, beta, gamma, fixed): + + # Compute the output + activation = AdaptiveSiLU(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x) + + # Verify the output shape is the same as the input shape + assert output_.shape == x.shape + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_backward(alpha, beta, gamma, fixed): + + # Compute the output and perform backpropagation + activation = AdaptiveSiLU(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x.requires_grad_()) + loss = torch.mean(output_) + loss.backward() + + # Verify that the gradients shape is the same as the input shape + assert x.grad.shape == x.shape diff --git a/tests/test_adaptive_function/test_adaptive_siren.py b/tests/test_adaptive_function/test_adaptive_siren.py new file mode 100644 index 000000000..5211ed290 --- /dev/null +++ b/tests/test_adaptive_function/test_adaptive_siren.py @@ -0,0 +1,89 @@ +import torch +import pytest +from pina import LabelTensor +from pina.adaptive_function import AdaptiveSIREN + + +x = LabelTensor(torch.rand(10, 1), labels=["x"]) +fixed_variables = [None, "alpha", ["beta", "gamma"], ["alpha", "beta", "gamma"]] +param_names = ["alpha", "beta", "gamma"] + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_constructor(alpha, beta, gamma, fixed): + + # Construct the adaptive activation function with the specified parameters + activation = AdaptiveSIREN(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + + # Build expected values for parameters + expected = {"alpha": alpha, "beta": beta, "gamma": gamma} + if fixed is None: + fixed_set = set() + else: + fixed_set = {fixed} if isinstance(fixed, str) else set(fixed) + + # Verify fixed parameters consistency + for v in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is False + assert param.item() == expected[v] + + # Verify trainable parameters consistency + for v in param_names: + if v not in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is True + assert param.item() == expected[v] + + # Should fail if alpha is not a number + with pytest.raises(ValueError): + AdaptiveSIREN(alpha="s") + + # Should fail if beta is not a number + with pytest.raises(ValueError): + AdaptiveSIREN(beta="s") + + # Should fail if gamma is not a number + with pytest.raises(ValueError): + AdaptiveSIREN(gamma="s") + + # Should fail if fixed is not a string or list of strings + with pytest.raises(ValueError): + AdaptiveSIREN(fixed=123) + + # Should fail if fixed contains invalid parameter names + with pytest.raises(ValueError): + AdaptiveSIREN(fixed="delta") + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_forward(alpha, beta, gamma, fixed): + + # Compute the output + activation = AdaptiveSIREN(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x) + + # Verify the output shape is the same as the input shape + assert output_.shape == x.shape + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_backward(alpha, beta, gamma, fixed): + + # Compute the output and perform backpropagation + activation = AdaptiveSIREN(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x.requires_grad_()) + loss = torch.mean(output_) + loss.backward() + + # Verify that the gradients shape is the same as the input shape + assert x.grad.shape == x.shape diff --git a/tests/test_adaptive_function/test_adaptive_softmax.py b/tests/test_adaptive_function/test_adaptive_softmax.py new file mode 100644 index 000000000..af53fcca9 --- /dev/null +++ b/tests/test_adaptive_function/test_adaptive_softmax.py @@ -0,0 +1,95 @@ +import torch +import pytest +from pina import LabelTensor +from pina.adaptive_function import AdaptiveSoftmax + + +x = LabelTensor(torch.rand(10, 1), labels=["x"]) +fixed_variables = [None, "alpha", ["beta", "gamma"], ["alpha", "beta", "gamma"]] +param_names = ["alpha", "beta", "gamma"] + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_constructor(alpha, beta, gamma, fixed): + + # Construct the adaptive activation function with the specified parameters + activation = AdaptiveSoftmax( + alpha=alpha, beta=beta, gamma=gamma, fixed=fixed + ) + + # Build expected values for parameters + expected = {"alpha": alpha, "beta": beta, "gamma": gamma} + if fixed is None: + fixed_set = set() + else: + fixed_set = {fixed} if isinstance(fixed, str) else set(fixed) + + # Verify fixed parameters consistency + for v in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is False + assert param.item() == expected[v] + + # Verify trainable parameters consistency + for v in param_names: + if v not in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is True + assert param.item() == expected[v] + + # Should fail if alpha is not a number + with pytest.raises(ValueError): + AdaptiveSoftmax(alpha="s") + + # Should fail if beta is not a number + with pytest.raises(ValueError): + AdaptiveSoftmax(beta="s") + + # Should fail if gamma is not a number + with pytest.raises(ValueError): + AdaptiveSoftmax(gamma="s") + + # Should fail if fixed is not a string or list of strings + with pytest.raises(ValueError): + AdaptiveSoftmax(fixed=123) + + # Should fail if fixed contains invalid parameter names + with pytest.raises(ValueError): + AdaptiveSoftmax(fixed="delta") + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_forward(alpha, beta, gamma, fixed): + + # Compute the output + activation = AdaptiveSoftmax( + alpha=alpha, beta=beta, gamma=gamma, fixed=fixed + ) + output_ = activation(x) + + # Verify the output shape is the same as the input shape + assert output_.shape == x.shape + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_backward(alpha, beta, gamma, fixed): + + # Compute the output and perform backpropagation + activation = AdaptiveSoftmax( + alpha=alpha, beta=beta, gamma=gamma, fixed=fixed + ) + output_ = activation(x.requires_grad_()) + loss = torch.mean(output_) + loss.backward() + + # Verify that the gradients shape is the same as the input shape + assert x.grad.shape == x.shape diff --git a/tests/test_adaptive_function/test_adaptive_softmin.py b/tests/test_adaptive_function/test_adaptive_softmin.py new file mode 100644 index 000000000..ef1968930 --- /dev/null +++ b/tests/test_adaptive_function/test_adaptive_softmin.py @@ -0,0 +1,95 @@ +import torch +import pytest +from pina import LabelTensor +from pina.adaptive_function import AdaptiveSoftmin + + +x = LabelTensor(torch.rand(10, 1), labels=["x"]) +fixed_variables = [None, "alpha", ["beta", "gamma"], ["alpha", "beta", "gamma"]] +param_names = ["alpha", "beta", "gamma"] + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_constructor(alpha, beta, gamma, fixed): + + # Construct the adaptive activation function with the specified parameters + activation = AdaptiveSoftmin( + alpha=alpha, beta=beta, gamma=gamma, fixed=fixed + ) + + # Build expected values for parameters + expected = {"alpha": alpha, "beta": beta, "gamma": gamma} + if fixed is None: + fixed_set = set() + else: + fixed_set = {fixed} if isinstance(fixed, str) else set(fixed) + + # Verify fixed parameters consistency + for v in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is False + assert param.item() == expected[v] + + # Verify trainable parameters consistency + for v in param_names: + if v not in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is True + assert param.item() == expected[v] + + # Should fail if alpha is not a number + with pytest.raises(ValueError): + AdaptiveSoftmin(alpha="s") + + # Should fail if beta is not a number + with pytest.raises(ValueError): + AdaptiveSoftmin(beta="s") + + # Should fail if gamma is not a number + with pytest.raises(ValueError): + AdaptiveSoftmin(gamma="s") + + # Should fail if fixed is not a string or list of strings + with pytest.raises(ValueError): + AdaptiveSoftmin(fixed=123) + + # Should fail if fixed contains invalid parameter names + with pytest.raises(ValueError): + AdaptiveSoftmin(fixed="delta") + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_forward(alpha, beta, gamma, fixed): + + # Compute the output + activation = AdaptiveSoftmin( + alpha=alpha, beta=beta, gamma=gamma, fixed=fixed + ) + output_ = activation(x) + + # Verify the output shape is the same as the input shape + assert output_.shape == x.shape + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_backward(alpha, beta, gamma, fixed): + + # Compute the output and perform backpropagation + activation = AdaptiveSoftmin( + alpha=alpha, beta=beta, gamma=gamma, fixed=fixed + ) + output_ = activation(x.requires_grad_()) + loss = torch.mean(output_) + loss.backward() + + # Verify that the gradients shape is the same as the input shape + assert x.grad.shape == x.shape diff --git a/tests/test_adaptive_function/test_adaptive_tanh.py b/tests/test_adaptive_function/test_adaptive_tanh.py new file mode 100644 index 000000000..4ece0b8eb --- /dev/null +++ b/tests/test_adaptive_function/test_adaptive_tanh.py @@ -0,0 +1,89 @@ +import torch +import pytest +from pina import LabelTensor +from pina.adaptive_function import AdaptiveTanh + + +x = LabelTensor(torch.rand(10, 1), labels=["x"]) +fixed_variables = [None, "alpha", ["beta", "gamma"], ["alpha", "beta", "gamma"]] +param_names = ["alpha", "beta", "gamma"] + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_constructor(alpha, beta, gamma, fixed): + + # Construct the adaptive activation function with the specified parameters + activation = AdaptiveTanh(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + + # Build expected values for parameters + expected = {"alpha": alpha, "beta": beta, "gamma": gamma} + if fixed is None: + fixed_set = set() + else: + fixed_set = {fixed} if isinstance(fixed, str) else set(fixed) + + # Verify fixed parameters consistency + for v in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is False + assert param.item() == expected[v] + + # Verify trainable parameters consistency + for v in param_names: + if v not in fixed_set: + param = getattr(activation, v) + assert param.requires_grad is True + assert param.item() == expected[v] + + # Should fail if alpha is not a number + with pytest.raises(ValueError): + AdaptiveTanh(alpha="s") + + # Should fail if beta is not a number + with pytest.raises(ValueError): + AdaptiveTanh(beta="s") + + # Should fail if gamma is not a number + with pytest.raises(ValueError): + AdaptiveTanh(gamma="s") + + # Should fail if fixed is not a string or list of strings + with pytest.raises(ValueError): + AdaptiveTanh(fixed=123) + + # Should fail if fixed contains invalid parameter names + with pytest.raises(ValueError): + AdaptiveTanh(fixed="delta") + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_forward(alpha, beta, gamma, fixed): + + # Compute the output + activation = AdaptiveTanh(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x) + + # Verify the output shape is the same as the input shape + assert output_.shape == x.shape + + +@pytest.mark.parametrize("fixed", fixed_variables) +@pytest.mark.parametrize("alpha", [1, 2.5]) +@pytest.mark.parametrize("beta", [1, 2.5]) +@pytest.mark.parametrize("gamma", [1, 2.5]) +def test_backward(alpha, beta, gamma, fixed): + + # Compute the output and perform backpropagation + activation = AdaptiveTanh(alpha=alpha, beta=beta, gamma=gamma, fixed=fixed) + output_ = activation(x.requires_grad_()) + loss = torch.mean(output_) + loss.backward() + + # Verify that the gradients shape is the same as the input shape + assert x.grad.shape == x.shape