From f3da57dedffec2936dd62803e3031ec04f98c79f Mon Sep 17 00:00:00 2001 From: pks Date: Tue, 3 Oct 2023 12:14:06 +0200 Subject: tensorflow/transformer-attention1.py --- tensorflow/transformer-attention1.py | 54 ++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tensorflow/transformer-attention1.py (limited to 'tensorflow') diff --git a/tensorflow/transformer-attention1.py b/tensorflow/transformer-attention1.py new file mode 100644 index 0000000..32fe739 --- /dev/null +++ b/tensorflow/transformer-attention1.py @@ -0,0 +1,54 @@ +import numpy as np +import math + +dmodel = 32 +num_heads = 2 +embedding_dim = dmodel #dmodel // num_heads +nwords = 4 + +#assert(dmodel/num_heads == embedding_dim) + +states = np.array([np.ones(shape=[embedding_dim])*(i*0.1) for i in range(nwords)]) # num. words x embedding dim + +Wqs = [] +Wks = [] +Wvs = [] +scores = [] + + +def softmax(m): + return np.exp(m) / np.sum(np.exp(m), axis=1) + +for h in range(num_heads): + Wq = np.random.rand(embedding_dim, int(dmodel/num_heads)) + Wk = np.random.rand(embedding_dim, int(dmodel/num_heads)) + Wv = np.random.rand(embedding_dim, int(dmodel/num_heads)) + + queries = np.matmul(states, Wq) + keys = np.matmul(states, Wk) + print(states.shape) + values = np.matmul(states, Wv) + print(values.shape) + exit() + + out = np.matmul(queries, np.transpose(keys)) + out = out/math.sqrt(dmodel) + + # manual + #out_max = [] + #for i in range(out.shape[0]): + # out_max.append(softmax(out[i])) + #out = np.array(out_max) + + out = softmax(out) + out = np.matmul(out, values) + + Wqs.append(Wq) + Wks.append(Wk) + Wvs.append(Wv) + scores.append(out) + +out = np.concatenate(scores, axis=0) +out = np.matmul(np.random.rand(nwords, out.shape[0]), out) + + -- cgit v1.2.3