`__
で導入された。 実装は簡単である。
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def softmax(X):
X_exp = d2l.exp(X)
partition = d2l.reduce_sum(X_exp, 1, keepdims=True)
return X_exp / partition # ここではブロードキャスト機構が適用される
任意の入力 ``X`` に対して、各要素を非負の数に変換する。
各行の和は1になる。 これは確率として必要な性質である。注意:
上のコードは、非常に大きい値や非常に小さい値に対しては\ *頑健ではない*\ 。何が起きているかを示すには十分であるが、実用上の目的でこのコードをそのまま使うべきではない。深層学習フレームワークにはこのような保護機構が組み込まれており、今後は組み込みのソフトマックスを使う。
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
X = d2l.rand((2, 5))
X_prob = softmax(X)
X_prob, d2l.reduce_sum(X_prob, 1)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
(tensor([[0.1481, 0.2050, 0.2577, 0.1232, 0.2661],
[0.2167, 0.2010, 0.2468, 0.1128, 0.2227]]),
tensor([1.0000, 1.0000]))
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
X = d2l.rand(2, 5)
X_prob = softmax(X)
X_prob, d2l.reduce_sum(X_prob, 1)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
(array([[0.17777154, 0.1857739 , 0.20995119, 0.23887765, 0.18762572],
[0.24042214, 0.1757977 , 0.23786479, 0.15572716, 0.19018826]]),
array([1., 1.]))
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
X = jax.random.uniform(jax.random.PRNGKey(d2l.get_seed()), (2, 5))
X_prob = softmax(X)
X_prob, d2l.reduce_sum(X_prob, 1)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
(Array([[0.16229036, 0.13327661, 0.22257593, 0.1909562 , 0.2909009 ],
[0.30952108, 0.17390645, 0.18135622, 0.19830671, 0.13690951]], dtype=float32),
Array([1., 1.], dtype=float32))
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
X = d2l.rand((2, 5))
X_prob = softmax(X)
X_prob, d2l.reduce_sum(X_prob, 1)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
(,
)
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
class SoftmaxRegressionScratch(d2l.Classifier):
def __init__(self, num_inputs, num_outputs, lr, sigma=0.01):
super().__init__()
self.save_hyperparameters()
self.W = torch.normal(0, sigma, size=(num_inputs, num_outputs),
requires_grad=True)
self.b = torch.zeros(num_outputs, requires_grad=True)
def parameters(self):
return [self.W, self.b]
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
class SoftmaxRegressionScratch(d2l.Classifier):
def __init__(self, num_inputs, num_outputs, lr, sigma=0.01):
super().__init__()
self.save_hyperparameters()
self.W = np.random.normal(0, sigma, (num_inputs, num_outputs))
self.b = np.zeros(num_outputs)
self.W.attach_grad()
self.b.attach_grad()
def collect_params(self):
return [self.W, self.b]
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
class SoftmaxRegressionScratch(d2l.Classifier):
num_inputs: int
num_outputs: int
lr: float
sigma: float = 0.01
def setup(self):
self.W = self.param('W', nn.initializers.normal(self.sigma),
(self.num_inputs, self.num_outputs))
self.b = self.param('b', nn.initializers.zeros, self.num_outputs)
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
class SoftmaxRegressionScratch(d2l.Classifier):
def __init__(self, num_inputs, num_outputs, lr, sigma=0.01):
super().__init__()
self.save_hyperparameters()
self.W = tf.random.normal((num_inputs, num_outputs), 0, sigma)
self.b = tf.zeros(num_outputs)
self.W = tf.Variable(self.W)
self.b = tf.Variable(self.b)
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
y = d2l.tensor([0, 2])
y_hat = d2l.tensor([[0.1, 0.3, 0.6], [0.3, 0.2, 0.5]])
y_hat[[0, 1], y]
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
tensor([0.1000, 0.5000])
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
y = d2l.tensor([0, 2])
y_hat = d2l.tensor([[0.1, 0.3, 0.6], [0.3, 0.2, 0.5]])
y_hat[[0, 1], y]
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
array([0.1, 0.5])
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
y = d2l.tensor([0, 2])
y_hat = d2l.tensor([[0.1, 0.3, 0.6], [0.3, 0.2, 0.5]])
y_hat[[0, 1], y]
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
Array([0.1, 0.5], dtype=float32)
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
y_hat = tf.constant([[0.1, 0.3, 0.6], [0.3, 0.2, 0.5]])
y = tf.constant([0, 2])
tf.boolean_mask(y_hat, tf.one_hot(y, depth=y_hat.shape[-1]))
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def cross_entropy(y_hat, y):
return -d2l.reduce_mean(d2l.log(y_hat[list(range(len(y_hat))), y]))
cross_entropy(y_hat, y)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
tensor(1.4979)
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def cross_entropy(y_hat, y):
return -d2l.reduce_mean(d2l.log(y_hat[list(range(len(y_hat))), y]))
cross_entropy(y_hat, y)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
array(1.4978662)
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def cross_entropy(y_hat, y):
return -d2l.reduce_mean(d2l.log(y_hat[list(range(len(y_hat))), y]))
cross_entropy(y_hat, y)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
Array(1.4978662, dtype=float32)
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
def cross_entropy(y_hat, y):
return -tf.reduce_mean(tf.math.log(tf.boolean_mask(
y_hat, tf.one_hot(y, depth=y_hat.shape[-1]))))
cross_entropy(y_hat, y)
.. raw:: latex
\diilbookstyleoutputcell
.. parsed-literal::
:class: output
.. raw:: html
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
@d2l.add_to_class(SoftmaxRegressionScratch)
def loss(self, y_hat, y):
return cross_entropy(y_hat, y)
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
@d2l.add_to_class(SoftmaxRegressionScratch)
def loss(self, y_hat, y):
return cross_entropy(y_hat, y)
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
@d2l.add_to_class(SoftmaxRegressionScratch)
@partial(jax.jit, static_argnums=(0))
def loss(self, params, X, y, state):
def cross_entropy(y_hat, y):
return -d2l.reduce_mean(d2l.log(y_hat[list(range(len(y_hat))), y]))
y_hat = state.apply_fn({'params': params}, *X)
# 返される空の辞書は補助データのプレースホルダであり、
# 後で（たとえば batch norm のために）使われます
return cross_entropy(y_hat, y), {}
.. raw:: html
.. raw:: html
.. raw:: latex
\diilbookstyleinputcell
.. code:: python
@d2l.add_to_class(SoftmaxRegressionScratch)
def loss(self, y_hat, y):
return cross_entropy(y_hat, y)
.. raw:: html
.. raw:: html