Unordered, variable-sized inputs arise in many settings across multiple
fields. The ability for set- and multiset- oriented neural networks to handle
this type of input has been the focus of much work in recent years. We propose
to represent multisets using complex-weighted multiset automata and show how
the multiset representations of certain existing neural architectures can be
viewed as special cases of ours. Namely, (1) we provide a new theoretical and
intuitive justification for the Transformer model's representation of positions
using sinusoidal functions, and (2) we extend the DeepSets model to use complex
numbers, enabling it to outperform the existing model on an extension of one of
their tasks.
Description
[2001.00610] Representing Unordered Data Using Multiset Automata and Complex Numbers
%0 Journal Article
%1 debenedetto2020representing
%A DeBenedetto, Justin
%A Chiang, David
%D 2020
%K automata deep-learning readings sets
%T Representing Unordered Data Using Multiset Automata and Complex Numbers
%U http://arxiv.org/abs/2001.00610
%X Unordered, variable-sized inputs arise in many settings across multiple
fields. The ability for set- and multiset- oriented neural networks to handle
this type of input has been the focus of much work in recent years. We propose
to represent multisets using complex-weighted multiset automata and show how
the multiset representations of certain existing neural architectures can be
viewed as special cases of ours. Namely, (1) we provide a new theoretical and
intuitive justification for the Transformer model's representation of positions
using sinusoidal functions, and (2) we extend the DeepSets model to use complex
numbers, enabling it to outperform the existing model on an extension of one of
their tasks.
@article{debenedetto2020representing,
abstract = {Unordered, variable-sized inputs arise in many settings across multiple
fields. The ability for set- and multiset- oriented neural networks to handle
this type of input has been the focus of much work in recent years. We propose
to represent multisets using complex-weighted multiset automata and show how
the multiset representations of certain existing neural architectures can be
viewed as special cases of ours. Namely, (1) we provide a new theoretical and
intuitive justification for the Transformer model's representation of positions
using sinusoidal functions, and (2) we extend the DeepSets model to use complex
numbers, enabling it to outperform the existing model on an extension of one of
their tasks.},
added-at = {2020-06-03T09:48:00.000+0200},
author = {DeBenedetto, Justin and Chiang, David},
biburl = {https://www.bibsonomy.org/bibtex/2c118166142b6eff34709fab2fc6eefd8/kirk86},
description = {[2001.00610] Representing Unordered Data Using Multiset Automata and Complex Numbers},
interhash = {d3ed31cdef814b3786fcb1061156f3d0},
intrahash = {c118166142b6eff34709fab2fc6eefd8},
keywords = {automata deep-learning readings sets},
note = {cite arxiv:2001.00610},
timestamp = {2020-06-03T09:48:00.000+0200},
title = {Representing Unordered Data Using Multiset Automata and Complex Numbers},
url = {http://arxiv.org/abs/2001.00610},
year = 2020
}