Question

我需要做这样的事情,但在下午:

dd if=/dev/sdb | gzip -c | curl ftp upload

我可以用平线使用整个指挥系统,因为:

I need non-blocking operation
I need progress information (tried looping through proc.stderr to no avail)

另一大事是,在上载之前,我可以制作一个压抑的焦炭文档,用于记忆或磁盘。

因此,这是我想要说明如何做的,而“gzip_stream_of_strings(投入)”则不是:

import os, pycurl
filename =  /path/to/super/large/file.img 
filesize = os.path.getsize(filename)

def progress(dl_left, dl_completed, ul_left, ul_completed):
    return (ul_completed/filesize)*100

def main():
    c = pycurl.Curl()
    c.setopt(c.URL,  ftp://IP/save_as.img.gz )
    c.setopt(pycurl.NOPROGRESS, 0)
    c.setopt(pycurl.PROGRESSFUNCTION, progress)
    c.setopt(pycurl.UPLOAD, 1)
    c.setopt(pycurl.INFILESIZE, filesize)
    c.setopt(pycurl.USERPWD,  user:passwd )
    with open(filename) as input:
        c.setopt(pycurl.READFUNCTION, gzip_stream_of_stings(input))
        c.perform()
        c.close()

Any help is greatly appreciated!

EDIT: Here s the solution:

from gzip import GzipFile
from StringIO import StringIO

CHUNCK_SIZE = 1024

class GZipPipe(StringIO):
    """This class implements a compression pipe suitable for asynchronous 
    process.
    Credit to cdvddt @ http://snippets.dzone.com/posts/show/5644

    @param source: this is the input file to compress
    @param name: this is stored as the name in the gzip header
    @function read: call this to read(size) chunks from the gzip stream        
    """
    def __init__(self, source = None, name = "data"):
        StringIO.__init__(self)

        self.source = source
        self.source_eof = False
        self.buffer = ""
        self.zipfile = GzipFile(name,  wb , 9, self)

    def write(self, data):
        self.buffer += data

    def read(self, size = -1):
        while ((len(self.buffer) < size) or (size == -1)) and not self.source_eof:
            if self.source == None: 
                break
            chunk = self.source.read(CHUNCK_SIZE)
            self.zipfile.write(chunk)
            if (len(chunk) < CHUNCK_SIZE) :
                self.source_eof = True
                self.zipfile.flush()
                self.zipfile.close()
                break

        if size == 0:
            result = ""
        if size >= 1:
            result = self.buffer[0:size]
            self.buffer = self.buffer[size:]
        else:
            result = self.buffer
            self.buffer = ""

        return result

Used like so:

with open(filename) as input:
    c.setopt(pycurl.READFUNCTION, GZipPipe(input).read)

Answer 1

http://docs.python.org/library/zlib.html“rel=“nofollow noreferer”>zlib 。图书馆允许使用任何档案类物体,其中包括文字流。

import os, pycurl, zlib
from cStringIO import StringIO
filename =  /path/to/super/large/file.img 
filesize = os.path.getsize(filename)

def progress(dl_left, dl_completed, ul_left, ul_completed):
    return (ul_completed/filesize)*100

def main():
    c = pycurl.Curl()
    c.setopt(c.URL,  ftp://IP/save_as.img.gz )
    c.setopt(pycurl.NOPROGRESS, 0)
    c.setopt(pycurl.PROGRESSFUNCTION, progress)
    c.setopt(pycurl.UPLOAD, 1)
    c.setopt(pycurl.INFILESIZE, filesize)
    c.setopt(pycurl.USERPWD,  user:passwd )
    with open(filename) as input:
        s = StringIO()
        c.setopt(pycurl.READFUNCTION, s.write(zlib.compress(input.readlines())))
        c.perform()
        c.close()

我没有对此进行测试。 See for addition info.

友情链接