我有一个文件和数据集来生成矩阵(drug),数据集不是那么大,只有9000x行,但我在使用算法库时遇到了MemoryError。有没有在循环中运行样本的方法(使用迭代器)或者解决这个问题的方法。你知道吗
我试着阅读文档,但没有完全理解。任何帮助都是有价值的。你知道吗
def getParamter(real_matrix, multiple_matrix, testPosition):
creator.create("FitnessMax", base.Fitness, weights=(1.0,))
creator.create("Individual", array.array, typecode='d',
fitness=creator.FitnessMax)
toolbox = base.Toolbox()
# Attribute generator
toolbox.register("attr_float", random.uniform, 0, 1)
# Structure initializers
variable_num = len(multiple_matrix)
toolbox.register("individual", tools.initRepeat,
creator.Individual, toolbox.attr_float, variable_num)
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
#################################################################################################
real_labels = []
for i in range(0, len(testPosition)):
real_labels.append(real_matrix[testPosition[i][0], testPosition[i][1]])
multiple_prediction = []
for i in range(0, len(multiple_matrix)):
predicted_probability = []
predict_matrix = multiple_matrix[i]
for j in range(0, len(testPosition)):
predicted_probability.append(
predict_matrix[testPosition[j][0], testPosition[j][1]])
normalize = MinMaxScaler()
predicted_probability = np.array(predicted_probability).reshape(-1, 1)
predicted_probability = normalize.fit_transform(predicted_probability)
multiple_prediction.append(predicted_probability)
#################################################################################################
print(len(real_labels), len(multiple_prediction))
# real_labels = real_labels[0:1000]
toolbox.register("evaluate", fitFunction,
parameter1=real_labels, parameter2=multiple_prediction)
toolbox.register("mate", tools.cxTwoPoint)
toolbox.register("mutate", tools.mutFlipBit, indpb=0.05)
toolbox.register("select", tools.selTournament, tournsize=3)
random.seed(0)
pop = toolbox.population(n=100)
hof = tools.HallOfFame(1)
stats = tools.Statistics(lambda ind: ind.fitness.values)
stats.register("avg", numpy.mean)
stats.register("std", numpy.std)
stats.register("min", numpy.min)
stats.register("max", numpy.max)
# Below line is causing MemoryError
pop, log = algorithms.eaSimple(pop, toolbox, cxpb=0.5, mutpb=0.2, ngen=50,
stats=stats, halloffame=hof, verbose=True)
pop.sort(key=lambda ind: ind.fitness, reverse=True)
print(pop[0])
return pop[0]
我认为问题可能出在嵌套for循环中:应该避免使用for循环,而更喜欢numpy广播。你知道吗
我试着复制你在这里所做的:
这就是我想到的(我可能误判了你的意见):
也许有一种方法可以避免两个转置(the
.T
),我没试太久。你知道吗相关问题 更多 >
编程相关推荐