我有一个工作模型,当在一个小数据集上使用fit时,我可以定期进行训练。但是,由于数据集的总大小很大,我想切换到生成器。你知道吗
我的模型是一个变分自动编码器,有3个输入层。首先,我使用正则拟合方法使模型工作。当我按以下方式运行模型时,它可以工作:
history = bvae.fit([data_class, data_regress, n_particles],
epochs = epochs,
verbose = 1,
batch_size = batch_size,
validation_split = 0.2,
callbacks = callback_list)
当我得到这个工作,我想切换到一个发电机,以便我可以使用更多的训练样本。我尝试实现我的生成器,如下所示:
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, filename, sample_size, batch_size):
self.filename = filename
self.sample_size = sample_size
self.batch_size = batch_size # batch size
self.channel = h5py.File(self.filename, 'r')
self.list_class_arg = range(3, 67, 4)
self.list_regress_arg = [x for x in range(1, 67) if x not in self.list_class_arg]
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(self.sample_size / self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
# selects data
events = self.channel['SM'][index*self.batch_size:(index+1)*self.batch_size]
# splits events into class and regress
events_class = self.class_norm_to_int(events[:,self.list_class_arg])
events_regress = events[:,self.list_regress_arg]
# count number of particles in event
n_particles = self.count_particles(events)
print(type(events_class))
print(type(events_regress))
print(type(n_particles))
return events_class, events_regress, n_particles
def count_particles(self, data):
'Counts the number of particles per events and returns that number in an array'
n_particles = np.zeros(len(data), dtype='int32')
for i in range(len(data)):
for j in range(3,67,4):
if data[i,j] != 0:
n_particles[i] += 1
else:
break
return n_particles
def class_norm_to_int(self, data_class):
'Transform the particle type to an interger number'
class_shape = np.shape(data_class)
data_class = np.ndarray.flatten(data_class)
data_class = [(x*2.1659782700803696)+5.2390088017131005 if x!=0 else x for x in data_class]
data_class = np.reshape(data_class, newshape=class_shape)
return data_class
但是我得到了一个错误:
Using TensorFlow backend.
Epoch 1/1
<class 'numpy.ndarray'>
<class 'numpy.ndarray'>
<class 'numpy.ndarray'>
Traceback (most recent call last):
File "bvae_train_generator.py", line 96, in <module>
callbacks = callback_list)
File "/home/usr/venv/lib/python3.6/site-packages/keras/legacy/interfaces.py", line 91, in wrapper
return func(*args, **kwargs)
File "/home/usr/venv/lib/python3.6/site-packages/keras/engine/training.py", line 1658, in fit_generator
initial_epoch=initial_epoch)
File "/home/usr/venv/lib/python3.6/site-packages/keras/engine/training_generator.py", line 215, in fit_generator
class_weight=class_weight)
File "/home/usr/venv/lib/python3.6/site-packages/keras/engine/training.py", line 1443, in train_on_batch
class_weight=class_weight)
File "/home/usr/venv/lib/python3.6/site-packages/keras/engine/training.py", line 757, in _standardize_user_data
exception_prefix='input')
File "/home/usr/venv/lib/python3.6/site-packages/keras/engine/training_utils.py", line 105, in standardize_input_data
str(len(data)) + ' arrays: ' + str(data)[:200] + '...')
ValueError: Error when checking model input: the list of Numpy arrays that you are passing to your model is not the size the model expected. Expected to see 3 array(s), but instead got the following list of 1 arrays: [array([[5., 5., 6., ..., 0., 0., 0.],
[5., 0., 0., ..., 0., 0., 0.],
[5., 7., 7., ..., 0., 0., 0.],
...,
[5., 5., 0., ..., 0., 0., 0.],
[5., 5., 0., ..., 0., 0., 0....
似乎认为我只返回一个带有一个数组的列表,但正如您在print语句的结果中所看到的,所有三个给定的变量都是这种类型的numpy.ndarray公司你知道吗
我已尝试将\uu getitem\uuuu中的返回值更改为:
return [events_class, events_regress, n_particles]
或者
return (events_class, events_regress, n_particles)
然而,这似乎并不能解决问题。欢迎任何帮助!你知道吗
编辑:
我的模型有以下形式:
def initialize_bvae(beta, n_particles, latent, activation_func, extra_layer = False):
'Network parameters'
input_shape_class = (n_particles, )
input_shape_regress = (n_particles*3 + 2, )
hidden_layer_dim = 512
latent_dim = n_particles*latent
'Build encoder'
inputs_class = Input(shape=input_shape_class, name='input_1') # Input for classification
inputs_class_t0 = Lambda(lambda i: i)(inputs_class)
inputs_class_t1 = Lambda(lambda x: K.one_hot(K.cast(x, 'int32'), 8))(inputs_class_t0) # Onehot encode classification input
inputs_class_t2 = Reshape((n_particles*8,))(inputs_class_t1)
inputs_regress = Input(shape=input_shape_regress, name='input_2') # Input for regression
inputs = Concatenate()([inputs_class_t2, inputs_regress]) # Combine inputs
x = Dense(hidden_layer_dim, activation=activation_func)(inputs)
if extra_layer:
x = Dense(int(hidden_layer_dim), activation=activation_func)(x)
x = Dense(int(hidden_layer_dim/2), activation=activation_func)(x)
x = Dense(int(hidden_layer_dim/2), activation=activation_func)(x)
x = Dense(int(hidden_layer_dim/4), activation=activation_func)(x)
'Latent space'
z_mean = Dense(latent_dim, activation='linear', name='z_mean')(x)
z_log_var = Dense(latent_dim, activation='linear', name='z_log_var')(x)
z = Lambda(sampling, output_shape=(latent_dim,), name='z')([z_mean, z_log_var])
encoder = Model([inputs_class, inputs_regress], [z_mean, z_log_var, z], name='encoder')
'Build decoder'
inputs_latent = Input(shape=(latent_dim,), name='z_sampling')
inputs_particles = Input(shape=(1,), name='input_3') # Input for particles
# Implement mask
mask_fn = lambda n: K.concatenate((K.ones(shape=latent*n, dtype='float32'), K.zeros(shape=latent_dim-(latent*n), dtype='float32')))
mask_layer = Lambda(lambda x: K.map_fn(mask_fn, K.cast(x, 'int32'), dtype='float32'))(inputs_particles)
masked_latent = Lambda(lambda x: x[0]*x[1])([inputs_latent, mask_layer])
x = Dense(int(hidden_layer_dim/4), activation=activation_func)(masked_latent)
x = Dense(int(hidden_layer_dim/2), activation=activation_func)(x)
if extra_layer:
x = Dense(int(hidden_layer_dim/2), activation=activation_func)(x)
x = Dense(int(hidden_layer_dim), activation=activation_func)(x)
x = Dense(hidden_layer_dim, activation=activation_func)(x)
outputs_class_t = Dense(n_particles*8, activation='elu')(x)
outputs_class_t2 = Reshape((n_particles, 8))(outputs_class_t)
outputs_class = Conv1D(8, 1, activation='softmax')(outputs_class_t2) # Classification output
outputs_regress = Dense(n_particles*3+2, activation='linear')(x) #Regression output
decoder = Model([inputs_latent, inputs_particles], [outputs_class, outputs_regress], name='decoder')
'Build autoencoder'
outputs = decoder([encoder([inputs_class, inputs_regress])[2], inputs_particles])
bvae = Model([inputs_class, inputs_regress, inputs_particles], outputs, name='bvae')
'Calculate loss'
classification_loss = categorical_crossentropy(inputs_class_t1, outputs[0])
regression_loss = mse(inputs_regress, outputs[1])
reconstruction_loss = 20*K.mean(classification_loss) + K.mean(regression_loss)
kl_loss = -0.5 * K.sum((1+z_log_var-K.square(z_mean)-K.exp(z_log_var)), axis=-1)
bvae_loss = K.mean((1-beta) * reconstruction_loss + beta * kl_loss)
bvae.add_loss(bvae_loss)
bvae.compile(optimizer = keras.optimizers.Adam(lr=1e-3, beta_1=0.9, beta_2=0.999))
return bvae, encoder, decoder
编辑2:
下面是我如何称呼fit_generator
:
history = bvae.fit_generator(load_data.DataGenerator(file_training, 1e6, 50000),
epochs = epochs,
verbose = 1,
workers=0,
validation_data = load_data.DataGenerator(file_validation, 1e5, 50000),
callbacks = callback_list)
您正在模型input\u regression、input\u class、input\u潜伏和input\u particles中定义多个输入。所有这些都应该在拟合模型时输入数据,而您只是从数据生成器返回(None,n)维数组。如果输入所有输入,由于尺寸不匹配而导致的错误将消失。你知道吗
相关问题 更多 >
编程相关推荐