向Keras CNN提供额外参数

Question

我必须用Keras做图像识别。然而，我的图片属于有限数量的类，我知道他们提前（如“狗”，“鱼”…） 所以我想提供这个额外的信息，使识别更容易。然后CNN应该确定亚类（比如德国牧羊犬，金枪鱼…） 我的想法是给CNN提供一张图片，并在分类器中添加一个热编码的类名

问题是，什么是确切的语法

我的想法是：

1. 创建CNN：

例如

def createSmallerVggNet(nDrop, nL2):
    print("2: createSmallerVggNet(", nDrop, ", ", nL2, ")")
    model = Sequential()
    model.add(Conv2D(32, (3, 3), padding="same", input_shape=input_shape))
    model.add(Activation("relu"))
    model.add(BatchNormalization()) #axis=chanDim)) # Note: related to OCV to TF channelsLast

    model.add(Conv2D(32, (3, 3), padding="same", kernel_regularizer=regularizers.l2(nL2)))
    model.add(Activation("relu"))
    model.add(BatchNormalization())

    model.add(Conv2D(32, (3, 3), padding="same", kernel_regularizer=regularizers.l2(nL2)))
    model.add(Activation("relu"))
    model.add(BatchNormalization())

    model.add(MaxPooling2D(pool_size=(3, 3)))
    model.add(Dropout(nDrop))

    model.add(Conv2D(64, (3, 3), padding="same", kernel_regularizer=regularizers.l2(nL2)))
    model.add(Activation("relu"))
    model.add(BatchNormalization())

    model.add(Conv2D(64, (3, 3), padding="same", kernel_regularizer=regularizers.l2(nL2)))
    model.add(Activation("relu"))
    model.add(BatchNormalization())

    model.add(Conv2D(64, (3, 3), padding="same", kernel_regularizer=regularizers.l2(nL2)))
    model.add(Activation("relu"))
    model.add(BatchNormalization())

    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(nDrop))

    model.add(Conv2D(128, (3, 3), padding="same", kernel_regularizer=regularizers.l2(nL2)))
    model.add(Activation("relu"))
    model.add(BatchNormalization())

    model.add(Conv2D(128, (3, 3), padding="same", kernel_regularizer=regularizers.l2(nL2)))
    model.add(Activation("relu"))
    model.add(BatchNormalization())

    model.add(Conv2D(128, (3, 3), padding="same", kernel_regularizer=regularizers.l2(nL2)))
    model.add(Activation("relu"))
    model.add(BatchNormalization())

    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(nDrop))  

    model.add(Conv2D(256, (3, 3), padding="same", kernel_regularizer=regularizers.l2(nL2)))
    model.add(Activation("relu"))
    model.add(BatchNormalization())

    model.add(Conv2D(256, (3, 3), padding="same", kernel_regularizer=regularizers.l2(nL2)))
    model.add(Activation("relu"))
    model.add(BatchNormalization())

    model.add(Conv2D(256, (3, 3), padding="same", kernel_regularizer=regularizers.l2(nL2)))
    model.add(Activation("relu"))
    model.add(BatchNormalization())

    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(nDrop))  

    model.add(Flatten())

    model.add(Dense(512, kernel_regularizer=regularizers.l2(nL2)))

    model.add(Activation("relu"))
    model.add(BatchNormalization())

    model.add(Dropout(nDrop))

    return(model)

1. 添加带有额外输入的分类器：

例如

def createSmallerVggNetWithExtraInputs(nDrop, nL2):

    base_model = createSmallerVggNet(nDrop, nL2)

    x = base_model.output
    x = Dense(128, kernel_regularizer=regularizers.l2(nL2))(x)
    x = Activation('relu')(x)
    x = Dropout(nL2)(x)

    predictions = Dense(len(mlb.classes_), activation='softmax', kernel_regularizer=regularizers.l2(nL2))(x)

    model = Model(inputs = [base_model.input, >>> extra inputs here, but how? <<<], outputs = predictions)

    opt = SGD(lr=0.001, momentum=0.9)

    model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

    return model

我认为，数据生成器可能是这样的（同样，不确定确切的语法）：

def gen(bIsTrain):

  while True:
    arrImages = []
    arrImageClasses = []
    arrImageExtras = []

    arrClasses = np.random.choice(arrLabels, size = BATCH_SIZE)

    for cls in arrClasses:
      nImagesPerClass = len(cls['ImageNames'])
      nTrainingImages = int(0.6 * nImagesPerClass)
      nValidationImages = int(0.3 * nImagesPerClass)

      if(bIsTrain):
        nImageIdx = random.randint(0, nTrainingImages)
      else:
        nImageIdx = random.randint(nTrainingImages, nTrainingImages + nValidationImages)

      img = loadImage(cls, nImageIdx, datagen, 0) #bIsTrain)

      #img = cv2.blur(img, (5,5))
      #img = np.matrix.transpose(img)
      #img = np.expand_dims(img, axis=2)

      arrImages.append(img)
      arrImageClasses.append(cls['LabelsOneHot'])
      arrImageExtras.append(cls['FirstLevelOneHot'])      

    # yield np.array(arrImages), np.array(arrImageClasses)

    yield ({'images': np.array(arrImages), 'extras': arrImageExtras}, np.array(arrImageClasses))

在本例中，网络创建看起来像（语法？）：

def createSmallerVggNetWithExtraInputs(nDrop, nL2):

  #images_input = Input(input_shape, name='images')
  extras_input = Input(NUM_CLASSES, name='extras')

  base_model = createSmallerVggNet(nDrop, nL2)

  x = base_model.output
  x = Dense(128, kernel_regularizer=regularizers.l2(nL2))(x)
  x = Activation('relu')(x)
  x = Dropout(nL2)(x)

  predictions = Dense(len(mlb.classes_), activation='softmax', kernel_regularizer=regularizers.l2(nL2))(x)

  model = Model(inputs = [base_model.input, extras_input], outputs = predictions)

  opt = SGD(lr=0.001, momentum=0.9)

  model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])

  return model

非常感谢你的帮助。 谢谢你