上传大文件到S3

2024-06-20 15:13:07 发布

您现在位置:Python中文网/ 问答频道 /正文

我的机器上有python2.7、pip和boto。你知道吗

我需要上传一个约400 MB的zip文件到S3存储桶。你知道吗

上载可以在小文件上工作,但在上载所需的400 MB文件时失败并出现此错误。 BotoServerError:400错误请求 我正在使用下面的代码。有什么想法吗?你知道吗

    def UploadFileToS3(self, zippedfile, aws_access_key_id, aws_secret_access_key, bucket, awsfolder, callback=None, md5=None, reduced_redundancy=False, content_type=None):

    #def UploadFileToS3(self, zippedfile, callback=None, md5=None, reduced_redundancy=False, content_type=None):

    try:
        file = open(zippedfile, 'r+')

        key = file.name


        try:
            size = os.fstat(file.fileno()).st_size
        except Exception as ex:
            # Not all file objects implement fileno(),
            # so we fall back on this            
            file.seek(0, os.SEEK_END)
            size = file.tell()

        conn = boto.connect_s3(aws_access_key_id, aws_secret_access_key)
        bucket = conn.get_bucket(bucket, validate=True)
        k = Key(bucket)
        #k.key = key


        filename = os.path.basename(zippedfile)
        k.key = awsfolder + "/" + filename

        if content_type:
            k.set_metadata('Content-Type', content_type)
        sent = k.set_contents_from_file(file, cb=callback, md5=md5, reduced_redundancy=reduced_redundancy, rewind=True)

        # Rewind for later use
        file.seek(0)
    except Exception as ex:
        print "Error encountered while uploading to S3. " + unicode(ex)

    if sent != None:
        if sent == size:
            return True

    return False
#enddef  

 Stacktrace:
 Traceback (most recent call last):
  File "C:\AJ\scripts\GISSTdbS3ExtractFunctions.py", line 286, in       UploadFileToS3
    sent = k.set_contents_from_file(file, cb=callback, md5=md5,     reduced_redundancy=reduced_redundancy, rewind=True)
  File "C:\Python27\python2714\Lib\site-packages\boto\s3\key.py", line 1309,  in set_contents_from_file
    chunked_transfer=chunked_transfer, size=size)
  File "C:\Python27\python2714\Lib\site-packages\boto\s3\key.py", line 762,  in send_file
    chunked_transfer=chunked_transfer, size=size)
  File "C:\Python27\python2714\Lib\site-packages\boto\s3\key.py", line 963, in _ send_file_internal
    query_args=query_args
  File "C:\Python27\python2714\Lib\site-packages\boto\s3\connection.py", line 671, in make_request
    retry_handler=retry_handler
  File "C:\Python27\python2714\Lib\site-packages\boto\connection.py", line   1071, in make_request
  retry_handler=retry_handler)
  File "C:\Python27\python2714\Lib\site-packages\boto\connection.py"        raise BotoServerError(response.status, response.reason, body)
  BotoServerError: BotoServerError: 400 Bad Request

  Error encountered while uploading to S3. BotoServerError: 400 Bad Request

Tags: keyinpynonesizeliblinemd5