fxnn_model.py

import numpy as np
import sys, os
if len(sys.argv) > 1 and sys.argv[1] == "gpu":
    os.environ["THEANO_FLAGS"] = "device=gpu2, "
    os.environ["THEANO_FLAGS"] += "dnn.enabled=True, "
    os.environ["THEANO_FLAGS"] += "lib.cnmem=0.95"
from keras.layers import Input, Convolution2D, MaxPooling2D, AveragePooling2D
from keras.layers import Flatten, BatchNormalization, Activation
from keras.regularizers import l2
from keras.models import Model

def get_kernels(M):
    W = [[[1, 2, 1],
          [2, 4, 1],
          [1, 2, 1]],
         [[1, 2, 1],
          [0, 0, 0],
          [-1,-2,-1]],
         [[1, 0, -1],
          [2, 0, -1],
          [1, 0, -1]],
         [[1, 0, -1],
          [0, 0, 0],
          [-1, 0, 1]],
         [[0, 0, 0],
          [0, 1, 0],
          [0, 0, 0]]]
    W = np.array(W, dtype="float32")
    W = np.expand_dims(W, 1)
    W = np.repeat(W, M, axis=1)
    W = np.repeat(W, M, axis=0)
    b = np.zeros((W.shape[0],), dtype="float32")
    return [W, b]

def cbr_block(M, N, prefix, r=0.0, act="relu"):
    def f(input_):
        W, b = get_kernels(M)
        y1 = Convolution2D(5*M, 3, 3, activation="relu", name=prefix + "_conv",
                           border_mode="same", weights=[W, b])(input_)
        y2 = Convolution2D(N, 1, 1, activation="linear", border_mode="same",
                           W_regularizer=l2(r), b_regularizer=l2(r),
                           name=prefix + "_project")(y1)
        y3 = BatchNormalization(axis=1)(y2)
        y4 = Activation(act)(y3)
        return y4
    return f

def get_model(inshape=(3, 32, 32), r=0.0):
    x = Input(inshape)
    y = cbr_block(3,  16, "fxnn1", r=r)(x)
    y = cbr_block(16, 16, "fxnn2", r=r)(y)
    y = MaxPooling2D((3, 3), (2, 2))(y)
    y = cbr_block(16, 32, "fxnn3", r=r)(y)
    y = cbr_block(32, 32, "fxnn4", r=r)(y)
    y = MaxPooling2D((3, 3), (2, 2))(y)
    y = cbr_block(32, 64, "fxnn5", r=r)(y)
    y = cbr_block(64, 64, "fxnn6", r=r)(y)
    y = MaxPooling2D((3, 3), (2, 2))(y)
    y = cbr_block(64, 128,"fxnn7", r=r)(y)
    y = cbr_block(128,128,"fxnn8", r=r)(y)
    y = Activation("relu")(y)

    y = Convolution2D(100, 1, 1)(y)
    y = AveragePooling2D((3, 3))(y)
    y = Flatten()(y)
    y = Activation("softmax")(y)
    m = Model(x, y)
    for l in m.layers:
        if "_conv" in l.name:
            l.trainable = False
    return m

if __name__ == "__main__":
    from keras.datasets import cifar100
    from keras.utils.np_utils import to_categorical
    (trX, trY), (teX, teY) = cifar100.load_data(label_mode="fine")

    trY = to_categorical(trY)
    teY = to_categorical(teY)

    trX = trX.astype("float32") - 128.0
    teX = teX.astype("float32") - 128.0

    m = get_model(r=1e-5)
    m.compile("rmsprop", "categorical_crossentropy", metrics=["accuracy"])
    m.fit(trX, trY, validation_data=(teX, teY))

It seems your kernels are not same as my implementation.