\begin{thebibliography}{10}

\bibitem{apicella2021survey}
Andrea Apicella, Francesco Donnarumma, Francesco Isgr{\`o}, and Roberto
  Prevete.
\newblock A survey on modern trainable activation functions.
\newblock {\em Neural Networks}, 138:14--32, 2021.

\bibitem{bhumbra2018deep}
Gardave~S Bhumbra.
\newblock Deep learning improved by biological activation functions.
\newblock {\em arXiv preprint arXiv:1804.11237}, 2018.

\bibitem{bingham2020evolutionary}
Garrett Bingham, William Macke, and Risto Miikkulainen.
\newblock Evolutionary optimization of deep learning activation functions.
\newblock In {\em Proceedings of the 2020 Genetic and Evolutionary Computation
  Conference}, pages 289--296, 2020.

\bibitem{datta2020survey}
Leonid Datta.
\newblock A survey on activation functions and their relation with xavier and
  he normal initialization.
\newblock {\em arXiv preprint arXiv:2004.06632}, 2020.

\bibitem{dubey2019comparative}
Arun~Kumar Dubey and Vanita Jain.
\newblock Comparative study of convolution neural network’s relu and
  leaky-relu activation functions.
\newblock In {\em Applications of Computing, Automation and Wireless Systems in
  Electrical Engineering}, pages 873--880. Springer, 2019.

\bibitem{dubey2021comprehensive}
Shiv~Ram Dubey, Satish~Kumar Singh, and Bidyut~Baran Chaudhuri.
\newblock A comprehensive survey and performance analysis of activation
  functions in deep learning.
\newblock {\em arXiv preprint arXiv:2109.14545}, 2021.

\bibitem{noel2021growing}
Mathew~Mithra Noel, Advait Trivedi, Praneet Dutta, et~al.
\newblock Growing cosine unit: A novel oscillatory activation function that can
  speedup training and reduce parameters in convolutional neural networks.
\newblock {\em arXiv preprint arXiv:2108.12943}, 2021.

\bibitem{noel2021biologically}
Matthew~Mithra Noel, Shubham Bharadwaj, Venkataraman Muthiah-Nakarajan, Praneet
  Dutta, and Geraldine~Bessie Amali.
\newblock Biologically inspired oscillating activation functions can bridge the
  performance gap between biological and artificial neurons.
\newblock {\em arXiv preprint arXiv:2111.04020}, 2021.

\bibitem{nwankpa2018activation}
Chigozie Nwankpa, Winifred Ijomah, Anthony Gachagan, and Stephen Marshall.
\newblock Activation functions: Comparison of trends in practice and research
  for deep learning.
\newblock {\em arXiv preprint arXiv:1811.03378}, 2018.

\bibitem{szandala2021review}
Tomasz Szanda{\l}a.
\newblock Review and comparison of commonly used activation functions for deep
  neural networks.
\newblock In {\em Bio-inspired neurocomputing}, pages 203--224. Springer, 2021.

\end{thebibliography}