4 files changed, 153 insertions, 0 deletions
diff --git a/tensorflow/scatter-nd-add.py b/tensorflow/scatter-nd-add.py
new file mode 100644
index 0000000..7194d8b
--- /dev/null
+++ b/tensorflow/scatter-nd-add.py
@@ -0,0 +1,57 @@
+import numpy as np
+import tensorflow as tf
+
+sess = tf.Session()
+
+#idx = tf.constant([[0,2],[1,2]])
+
+# 4 x 2 | 40K x 256
+m = tf.Variable([[1,2],
+                 [0,0],
+                 [0,0],
+                 [0,0]], dtype=tf.float32)
+# -> 2 x 4 | 256 x 40K
+m_transposed = tf.transpose(m)
+# -> AttributeError: 'Tensor' object has no attribute '_lazy_read'
+m_new = tf.Variable([[1., 0., 0., 0.],
+                     [2., 0., 0., 0.]], dtype=tf.float32)
+
+# 1 x 3 | 1 x Y
+idx = tf.constant([1,2,3], dtype=tf.int32)
+idx = sess.run(idx)
+_idx = []
+for j in idx:
+    for i in range(0,m_new.shape[0]):
+        _idx.append([i,j])
+#idx_new = tf.constant(_idx, dtype=tf.int32)
+idx_new = np.full(fill_value=_idx, shape=[6,2], dtype=np.int32)
+
+# 2 x 2
+up = tf.constant([[1,1],[1,1],[1,1]], dtype=tf.float32)
+# 1 x 4
+up_new = tf.reshape(up, [tf.size(up)])
+
+sess.run(tf.global_variables_initializer())
+
+print("m")
+print(sess.run(m))
+print("m_new")
+print(sess.run(m_new))
+print("m_transposed")
+print(sess.run(m_transposed))
+print("idx")
+print(idx)
+print("idx_new")
+#print(sess.run(idx_new))
+print(idx_new)
+print("up")
+print(sess.run(up))
+print("up_new")
+print(sess.run(up_new))
+
+print()
+print("scatter_nd_add")
+print(sess.run(tf.scatter_nd_add(m_new, indices=idx_new, updates=up_new)))
+
+print()
+
diff --git a/tensorflow/tf-cond.py b/tensorflow/tf-cond.py
new file mode 100644
index 0000000..13c1bc3
--- /dev/null
+++ b/tensorflow/tf-cond.py
@@ -0,0 +1,9 @@
+import tensorflow as tf
+
+sess = tf.Session()
+x = tf.constant(0)
+a = sess.run(
+  [tf.cond(tf.equal(tf.count_nonzero(x), 0), lambda: tf.constant(True, name="b"), lambda: tf.constant(False,name="a"))]
+)
+
+print(a)
diff --git a/tensorflow/transformer-attention.py b/tensorflow/transformer-attention.py
new file mode 100644
index 0000000..6f82549
--- /dev/null
+++ b/tensorflow/transformer-attention.py
@@ -0,0 +1,51 @@
+import numpy as np
+import math
+
+dmodel = 32
+embedding_dim = 8
+nwords = 3
+num_heads = 4
+
+assert(dmodel/num_heads == embedding_dim)
+
+states = np.array([np.ones(shape=[embedding_dim])*(i+1) for i in range(nwords)]) # num. words x embedding dim
+
+Wqs = []
+Wks = []
+Wvs = []
+scores = []
+
+def softmax(m):
+    return np.exp(m) / np.sum(np.exp(m), axis=1)
+
+for h in range(num_heads):
+    Wq = np.random.rand(embedding_dim, int(dmodel/num_heads))
+    Wk = np.random.rand(embedding_dim, int(dmodel/num_heads))
+    Wv = np.random.rand(embedding_dim, int(dmodel/num_heads))
+
+    queries = np.matmul(states, Wq)
+    keys    = np.matmul(states, Wk)
+    values  = np.matmul(states, Wv)
+
+    out = np.matmul(queries, np.transpose(keys))
+    out = out/math.sqrt(dmodel)
+
+    # manual
+    #out_max = []
+    #for i in range(out.shape[0]):
+    #    out_max.append(softmax(out[i]))
+    #out = np.array(out_max)
+
+    out = softmax(out)
+    out = np.matmul(out, values)
+
+    Wqs.append(Wq)
+    Wks.append(Wk)
+    Wvs.append(Wv)
+    scores.append(out)
+
+out = np.concatenate(scores, axis=0)
+out = np.matmul(np.random.rand(nwords,out.shape[0]), out)
+print(out.shape)
+print(out)
+
diff --git a/tensorflow/transformer-attention2.py b/tensorflow/transformer-attention2.py
new file mode 100644
index 0000000..c214934
--- /dev/null
+++ b/tensorflow/transformer-attention2.py
@@ -0,0 +1,36 @@
+import numpy as np
+import math
+
+dmodel = 32
+embedding_dim = 8
+nwords = 3
+num_heads = 4
+
+assert(dmodel/num_heads == embedding_dim)
+
+states = np.array([np.random.rand(embedding_dim) for i in range(nwords)]) # num. words x embedding dim
+
+def softmax(m):
+    return np.exp(m) / np.sum(np.exp(m), axis=1)
+
+Wqs = np.random.rand(embedding_dim, dmodel)
+Wks = np.random.rand(embedding_dim, dmodel)
+Wvs = np.random.rand(embedding_dim, dmodel)
+
+queries = np.matmul(states, Wqs)
+keys    = np.matmul(states, Wks)
+values  = np.matmul(states, Wvs)
+
+print(values)
+
+out = np.matmul(queries, np.transpose(keys))
+out = out/math.sqrt(dmodel/float(num_heads))
+
+out = softmax(out)
+print(out)
+out = np.matmul(out, values)
+
+out = np.matmul(np.random.rand(nwords,out.shape[0]), out)
+print(out.shape)
+print(out)
+