tensorflow 模型搭建及使用 常用小技巧

针对基于tensorflow的深度学习实验中，如何自定义损失函数 激活函数 注意力层 等问题，记录常用自定义模型的技巧和方法

1. 自定义激活函数

自定义gelu激活函数

# gelu激活函数
def gelu(x):
    return 0.5 * x * (1 + tf.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))
模型使用自定义激活函数

inputs1 = Input(shape=(2048,))
fe1 = Dropout(0.5)(inputs1)
fe2 = Dense(256, activation=gelu)(fe1)
inputs2 = Input(shape=(max_length1,))
se1 = Embedding(vocab_size, embedding_dim, mask_zero=True)(inputs2)
se2 = Dropout(0.5)(se1)
x = LSTM(128, return_sequences=True,activation=gelu)(se2)
x = Dropout(0.5)(x)
x = LSTM(256)(x)
加载自定义激活函数的模型

from keras.utils import to_categorical, get_custom_objects
from keras.models import load_model
from keras.layers import Activation
model = load_model(\'saved_model/inceptionV3_LSTM2_200.h5\',
                   custom_objects=get_custom_objects().update({\'gelu\': Activation(gelu)}))

2. 自定义Class的模型搭建
自定义class

# TPA注意力
class CalculateScoreMatrix(Layer):
    def __init__(self, output_dim=None, **kwargs):
        self.output_dim = output_dim
        super(CalculateScoreMatrix, self).__init__(**kwargs)
    def get_config(self):
        config = super().get_config().copy()
        config.update({\'output_dim\': self.output_dim})
        return config
    def build(self, input_shape):
        self.kernel = self.add_weight(name=\'kernel\',
                                      shape=(input_shape[-1], self.output_dim),
                                      initializer=\'uniform\',
                                      trainable=True)
        super(CalculateScoreMatrix, self).build(input_shape)
    def call(self, x):
        res = K.dot(x, self.kernel)
        return res
模型搭建

# ATP-LSTM
fe1 = Dropout(0.5)(inputs1)
fe2 = Dense(256, activation=tf_swish)(fe1)
inputs2 = Input(shape=(max_length1,))
se1 = Embedding(vocab_size, embedding_dim, mask_zero=True)(inputs2)
se2 = Dropout(0.5)(se1)
x = LSTM(64, return_sequences=True)(se2)
se2 = Dropout(0.5)(se1)
# get the 1~t-1 and t hidden state
H = Lambda(lambda x: x[:, :-1, :])(x)
ht = Lambda(lambda x: x[:, -1, :])(x)
ht = Reshape((64, 1))(ht)
# get the HC by 1*1 convolution
HC = Lambda(lambda x: K.permute_dimensions(x, [0, 2, 1]))(H)
score_mat = CalculateScoreMatrix(64)(HC)
score_mat = Lambda(lambda x: K.batch_dot(x[0], x[1]))([score_mat, ht])
# get the attn matrix
score_mat = Activation("sigmoid")(score_mat)
attn_mat = Multiply()([HC, score_mat])
attn_vec = Lambda(lambda x: K.sum(x, axis=-1))(attn_mat)
wvt = Dense(units=64 * 4, activation=None)(attn_vec)
wht = Dense(units=64 * 4, activation=None)(Flatten()(ht))
yht = Add()([wht, wvt])
加载自定义class的模型

from keras.utils import to_categorical, get_custom_objects
from keras.models import load_model
from keras.layers import Activation
model = load_model(\'saved_model/model_inception_TPA_lstm1_30.h5\',custom_objects=get_custom_objects().update({\'CalculateScoreMatrix\': CalculateScoreMatrix}))

3.花式学习率设置和损失函数自定义过几天再弄