SGE集群上的python Java最大堆大小
我有一个用python编写的管道,用Java调用一些进程。管道以两种可能的模式运行,本地模式(单个节点上)或SGE集群上
当我将选项设置为集群模式时,日志中的错误消息如下所示
Invalid maximum heap size: -Xmx4g -jar
Error: Could not create the Java Virtual Machine.
Error: A fatal exception has occurred. Program will exit.
在本地模式下,没有错误,也没有问题
我的问题是什么会导致这样的错误
我在本地或集群上运行作业的类如下
class LocalJobManager(JobManager):
def __init__(self):
self.cmd_strs = []
def add_job(self, cmd, cmd_args, **kwargs):
cmd_str = ' '.join([cmd, ] + [str(x) for x in cmd_args])
self.cmd_strs.append(cmd_str)
def run_job(self, cmd, cmd_args, **kwargs):
cmd_str = ' '.join([cmd, ] + [str(x) for x in cmd_args])
self._run_cmd(cmd_str)
def wait(self):
for cmd_str in self.cmd_strs:
self._run_cmd(cmd_str)
def _run_cmd(self, cmd_str):
'''
Throw exception if run command fails
'''
process = subprocess.Popen(cmd_str, stdin=subprocess.PIPE, shell=True)
process.stdin.close()
sts = os.waitpid(process.pid, 0)
if sts[1] != 0:
raise Exception('Failed to run {0}\n'.format(cmd_str))
class ClusterJobManager(JobManager):
def __init__(self, log_dir=None):
import drmaa
self._drmaa = drmaa
self.log_dir = log_dir
if self.log_dir is not None:
make_directory(self.log_dir)
self.session = self._drmaa.Session()
self.session.initialize()
self.job_ids = Queue()
self._lock = threading.Lock()
def add_job(self, cmd, cmd_args, mem=4, max_mem=10, num_cpus=1):
job_id = self._run_job(cmd, cmd_args, mem, max_mem, num_cpus)
self.job_ids.put(job_id)
def run_job(self, cmd, cmd_args, mem=4, max_mem=10, num_cpus=1):
job_id = self._run_job(cmd, cmd_args, mem, max_mem, num_cpus)
self._check_exit_status(job_id)
def wait(self):
self._lock.acquire()
job_ids = []
while not self.job_ids.empty():
job_ids.append(self.job_ids.get())
self.session.synchronize(job_ids, self._drmaa.Session.TIMEOUT_WAIT_FOREVER, False)
self._lock.release()
for job_id in job_ids:
self._check_exit_status(job_id)
def close(self):
self.session.control(self._drmaa.Session.JOB_IDS_SESSION_ALL, self._drmaa.JobControlAction.TERMINATE)
self.session.exit()
def _run_job(self, cmd, cmd_args, mem, max_mem, num_cpus):
job_template = self._init_job_template(cmd, cmd_args, mem, max_mem, num_cpus)
job_id = self.session.runJob(job_template)
self.session.deleteJobTemplate(job_template)
return job_id
def _init_job_template(self, cmd, cmd_args, mem, max_mem, num_cpus):
job_template = self.session.createJobTemplate()
job_template.remoteCommand = cmd
job_template.args = [str(x) for x in cmd_args]
job_template.workingDirectory = os.getcwd()
if self.log_dir is not None:
job_template.errorPath = ':' + self.log_dir
job_template.outputPath = ':' + self.log_dir
job_template.nativeSpecification = '-l mem_free={mem}G,mem_token={mem}G,h_vmem={max_mem}G -V -w n -pe ncpus {num_cpus}'.format(**locals())
return job_template
def _check_exit_status(self, job_id):
return_value = self.session.wait(job_id, self._drmaa.Session.TIMEOUT_WAIT_FOREVER)
if return_value.exitStatus != 0:
raise Exception('Job {0} failed with exit status {1}.'.format(return_value.jobId,
return_value.exitStatus))
通常无法创建Java虚拟机(正如我在某个论坛上读到的)是由语法错误引起的,即使调用的命令是正确的并且在本地工作,除了上面显示的在集群上运行作业的类之外,还运行除Java以外的所有对象
谢谢
# 1 楼答案
我在SGE上碰到过这个。您可能将默认的硬盘内存限制设置为4GB左右,而Java在初始化过程中使用的内存似乎比您在-Xmx4g参数中设置的4GB多一点。您是否可以查看管理员是否设置了硬内存限制?通常,您将使用以下方法设置或覆盖默认限制:
尝试通过该参数提供远远超过所需的内存,看看这是否解决了问题,然后在不崩溃的情况下尽可能降低h_vmem